<commit_before>from pprint import pprint

from django.shortcuts import render
from django.db import connections
from django.views import View

from fo2.models import rows_to_dict_list_lower

import lotes.models

import cd.forms

def index(request):
    context = {}
    return render(request, 'cd/index.html', context)

class LotelLocal(View):
    Form_class = cd.forms.LoteForm
    template_name = 'cd/lote_local.html'
    title_name = 'Inventariar 63'

    def mount_context(self, cursor, form):
        endereco = form.cleaned_data['endereco']
        lote = form.cleaned_data['lote']
        identificado = form.cleaned_data['identificado']

        context = {'endereco': endereco}

        try:
            lote_rec = lotes.models.Lote.objects.get(lote=lote)
        except lotes.models.Lote.DoesNotExist:
            context.update({'erro': 'Lote não encontrado'})
            return context
        context.update({
            'op': lote_rec.op,
            'referencia': lote_rec.referencia,
            'cor': lote_rec.cor,
            'tamanho': lote_rec.tamanho,
            'qtd_produzir': lote_rec.qtd_produzir,
            'local': lote_rec.local,
            })

        # print('identificado={}'.format(identificado))
        if identificado:
            # print('if identificado')
            form.data['identificado'] = None
            form.data['lote'] = None
            if lote != identificado:
                context.update({
                    'erro': 'A confirmação é bipando o mesmo lote. '
                            'Identifique o lote novamente.'})
                return context

            try:
                lote_rec = lotes.models.Lote.objects.get(lote=lote)
            except lotes.models.Lote.DoesNotExist:
                context.update({
                    'erro': 'Lote não encontrado no banco de dados'})
                return context

            lote_rec.local = endereco
            lote_rec.save()

            context['identificado'] = identificado
        else:
            # print('if not identificado')
            context['lote'] = lote
            # periodo = lote[:4]
            # ordem_confeccao = lote[-5:]

            # data = lotes.models.posicao_get_op(
            #     cursor, periodo, ordem_confeccao)
            # if len(data) == 0:
            #     context.update({'erro': 'Lote não encontrado'})
            #     return context
            # row = data[0]
            # op = row['OP']

            # print('lote_rec.local = "{}" endereco = "{}"'.format(
            #     lote_rec.local, endereco))
            if lote_rec.local != endereco:
                context['confirma'] = True
                form.data['identificado'] = form.data['lote']
            form.data['lote'] = None

        # lotes_rec = lotes.models.Lote.objects.get(local=endereco)
        # data = rows_to_dict_list_lower(lotes_rec)
        # pprint(data)

        return context

    def get(self, request, *args, **kwargs):
        if 'lote' in kwargs:
            return self.post(request, *args, **kwargs)
        else:
            context = {'titulo': self.title_name}
            form = self.Form_class()
            context['form'] = form
            return render(request, self.template_name, context)

    def post(self, request, *args, **kwargs):
        context = {'titulo': self.title_name}
        form = self.Form_class(request.POST)
        if 'lote' in kwargs:
            form.data['lote'] = kwargs['lote']
        if form.is_valid():
            # pprint(request.POST)
            cursor = connections['so'].cursor()
            data = self.mount_context(cursor, form)
            context.update(data)
        context['form'] = form
        return render(request, self.template_name, context)

class Estoque(View):
    Form_class = cd.forms.EstoqueForm
    template_name = 'cd/estoque.html'
    title_name = 'Inventariar 63'

    def mount_context(self, cursor, form):
        endereco = form.cleaned_data['endereco']
        lote = form.cleaned_data['lote']
        op = form.cleaned_data['op']
        ref = form.cleaned_data['ref']
        tam = form.cleaned_data['tam']
        cor = form.cleaned_data['cor']

        context = {'endereco': endereco,
                   'lote': lote,
                   'op': op,
                   'ref': ref,
                   'tam': tam,
                   'cor': cor,
                   }

        data_rec = lotes.models.Lote.objects
        if endereco:
            data_rec = data_rec.filter(local=endereco)
        else:
            # data_rec = data_rec.filter(local__isnull=False)
            data_rec = data_rec.exclude(
                local__isnull=True
            ).exclude(
                local__exact='')
        if lote:
            data_rec = data_rec.filter(lote=lote)
        if op:
            data_rec = data_rec.filter(op=op)
        if ref:
            data_rec = data_rec.filter(referencia=ref)
        if tam:
            data_rec = data_rec.filter(tamanho=tam)
        if cor:
            data_rec = data_rec.filter(cor=cor)
        data = data_rec.values()
        # pprint(data[0])
        # pprint(list(data_rec.values()[:3]))
        # print('len ->>>>>>>>>', len(data_rec.values()[:3]))
        # for row in data_rec[:3]:
        #     pprint(row.lote)
        context.update({
            'headers': ('Endereço', 'OP', 'Lote',
                        'Referência', 'Tamanho', 'Cor', 'Quant'),
            'fields': ('local', 'op', 'lote',
                       'referencia', 'tamanho', 'cor', 'qtd_produzir'),
            'data': data,
        })

        return context

    def get(self, request, *args, **kwargs):
        if 'lote' in kwargs:
            return self.post(request, *args, **kwargs)
        else:
            context = {'titulo': self.title_name}
            form = self.Form_class()
            context['form'] = form
            return render(request, self.template_name, context)

    def post(self, request, *args, **kwargs):
        context = {'titulo': self.title_name}
        form = self.Form_class(request.POST)
        if 'lote' in kwargs:
            form.data['lote'] = kwargs['lote']
        if form.is_valid():
            cursor = connections['so'].cursor()
            data = self.mount_context(cursor, form)
            context.update(data)
        context['form'] = form
        return render(request, self.template_name, context)
<commit_msg>permission_required<commit_after>from pprint import pprint

from django.shortcuts import render
from django.db import connections
from django.views import View
from django.contrib.auth.mixins import PermissionRequiredMixin

from fo2.models import rows_to_dict_list_lower

import lotes.models

import cd.forms

def index(request):
    context = {}
    return render(request, 'cd/index.html', context)

class LotelLocal(PermissionRequiredMixin, View):
    permission_required = 'lotes.can_inventorize_lote'
    Form_class = cd.forms.LoteForm
    template_name = 'cd/lote_local.html'
    title_name = 'Inventariar 63'

    def mount_context(self, cursor, form):
        endereco = form.cleaned_data['endereco']
        lote = form.cleaned_data['lote']
        identificado = form.cleaned_data['identificado']

        context = {'endereco': endereco}

        try:
            lote_rec = lotes.models.Lote.objects.get(lote=lote)
        except lotes.models.Lote.DoesNotExist:
            context.update({'erro': 'Lote não encontrado'})
            return context
        context.update({
            'op': lote_rec.op,
            'referencia': lote_rec.referencia,
            'cor': lote_rec.cor,
            'tamanho': lote_rec.tamanho,
            'qtd_produzir': lote_rec.qtd_produzir,
            'local': lote_rec.local,
            })

        # print('identificado={}'.format(identificado))
        if identificado:
            # print('if identificado')
            form.data['identificado'] = None
            form.data['lote'] = None
            if lote != identificado:
                context.update({
                    'erro': 'A confirmação é bipando o mesmo lote. '
                            'Identifique o lote novamente.'})
                return context

            try:
                lote_rec = lotes.models.Lote.objects.get(lote=lote)
            except lotes.models.Lote.DoesNotExist:
                context.update({
                    'erro': 'Lote não encontrado no banco de dados'})
                return context

            lote_rec.local = endereco
            lote_rec.save()

            context['identificado'] = identificado
        else:
            # print('if not identificado')
            context['lote'] = lote
            # periodo = lote[:4]
            # ordem_confeccao = lote[-5:]

            # data = lotes.models.posicao_get_op(
            #     cursor, periodo, ordem_confeccao)
            # if len(data) == 0:
            #     context.update({'erro': 'Lote não encontrado'})
            #     return context
            # row = data[0]
            # op = row['OP']

            # print('lote_rec.local = "{}" endereco = "{}"'.format(
            #     lote_rec.local, endereco))
            if lote_rec.local != endereco:
                context['confirma'] = True
                form.data['identificado'] = form.data['lote']
            form.data['lote'] = None

        # lotes_rec = lotes.models.Lote.objects.get(local=endereco)
        # data = rows_to_dict_list_lower(lotes_rec)
        # pprint(data)

        return context

    def get(self, request, *args, **kwargs):
        if 'lote' in kwargs:
            return self.post(request, *args, **kwargs)
        else:
            context = {'titulo': self.title_name}
            form = self.Form_class()
            context['form'] = form
            return render(request, self.template_name, context)

    def post(self, request, *args, **kwargs):
        context = {'titulo': self.title_name}
        form = self.Form_class(request.POST)
        if 'lote' in kwargs:
            form.data['lote'] = kwargs['lote']
        if form.is_valid():
            # pprint(request.POST)
            cursor = connections['so'].cursor()
            data = self.mount_context(cursor, form)
            context.update(data)
        context['form'] = form
        return render(request, self.template_name, context)

class Estoque(View):
    Form_class = cd.forms.EstoqueForm
    template_name = 'cd/estoque.html'
    title_name = 'Inventariar 63'

    def mount_context(self, cursor, form):
        endereco = form.cleaned_data['endereco']
        lote = form.cleaned_data['lote']
        op = form.cleaned_data['op']
        ref = form.cleaned_data['ref']
        tam = form.cleaned_data['tam']
        cor = form.cleaned_data['cor']

        context = {'endereco': endereco,
                   'lote': lote,
                   'op': op,
                   'ref': ref,
                   'tam': tam,
                   'cor': cor,
                   }

        data_rec = lotes.models.Lote.objects
        if endereco:
            data_rec = data_rec.filter(local=endereco)
        else:
            # data_rec = data_rec.filter(local__isnull=False)
            data_rec = data_rec.exclude(
                local__isnull=True
            ).exclude(
                local__exact='')
        if lote:
            data_rec = data_rec.filter(lote=lote)
        if op:
            data_rec = data_rec.filter(op=op)
        if ref:
            data_rec = data_rec.filter(referencia=ref)
        if tam:
            data_rec = data_rec.filter(tamanho=tam)
        if cor:
            data_rec = data_rec.filter(cor=cor)
        data = data_rec.values()
        # pprint(data[0])
        # pprint(list(data_rec.values()[:3]))
        # print('len ->>>>>>>>>', len(data_rec.values()[:3]))
        # for row in data_rec[:3]:
        #     pprint(row.lote)
        context.update({
            'headers': ('Endereço', 'OP', 'Lote',
                        'Referência', 'Tamanho', 'Cor', 'Quant'),
            'fields': ('local', 'op', 'lote',
                       'referencia', 'tamanho', 'cor', 'qtd_produzir'),
            'data': data,
        })

        return context

    def get(self, request, *args, **kwargs):
        if 'lote' in kwargs:
            return self.post(request, *args, **kwargs)
        else:
            context = {'titulo': self.title_name}
            form = self.Form_class()
            context['form'] = form
            return render(request, self.template_name, context)

    def post(self, request, *args, **kwargs):
        context = {'titulo': self.title_name}
        form = self.Form_class(request.POST)
        if 'lote' in kwargs:
            form.data['lote'] = kwargs['lote']
        if form.is_valid():
            cursor = connections['so'].cursor()
            data = self.mount_context(cursor, form)
            context.update(data)
        context['form'] = form
        return render(request, self.template_name, context)
<|endoftext|>


<commit_before>__version__ = "1.3.0"
<commit_msg>Update version.py<commit_after>__version__ = "1.3.1"
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import numpy as np
from numpy import ma
from scipy.interpolate import griddata

from oceansdb.utils import dbsource

def get_depth(lat, lon, cfg):
    """

    ATTENTION, conceptual error on the data near by Greenwich.
    url='http://opendap.ccst.inpe.br/Climatologies/ETOPO/etopo5.cdf'

    If I ever need to get depth from multiple points, check the history
      of this file. One day it was like that.
    """
    # This assert fails if it is a np.float64. Re-think this assert anyways.
    #assert type(lat) in [int, float]
    #assert type(lon) in [int, float]

    # if lat.shape != lon.shape:
    #            print "lat and lon must have the same size"

    try:
        try:
            etopo = netCDF4.Dataset(expanduser(cfg['file']))
        except:
            # FIXME, It must have a time limit defined here, otherwise it can
            #   get stuck trying to open the file.
            etopo = netCDF4.Dataset(expanduser(cfg['url']))
        x = etopo.variables['ETOPO05_X'][:]
        y = etopo.variables['ETOPO05_Y'][:]
    except:
        etopo = open_url(cfg['url']).ROSE
        x = etopo.ETOPO05_X[:]
        y = etopo.ETOPO05_Y[:]

    if lon < 0:
        lon += 360

    iini = (abs(lon - x)).argmin() - 2
    ifin = (abs(lon - x)).argmin() + 2
    jini = (abs(lat - y)).argmin() - 2
    jfin = (abs(lat - y)).argmin() + 2

    assert (iini >= 0) or (iini <= len(x)) or \
        (jini >= 0) or (jini <= len(y)), \
        "Sorry not ready to handle too close to boundaries"

    try:
        z = etopo.variables['ROSE'][jini:jfin, iini:ifin]
    except:
        z = etopo.ROSE[jini:jfin, iini:ifin]

    interpolator = RectBivariateSpline(x[iini:ifin], y[jini:jfin], z.T)
    return interpolator(lon, lat)[0][0]

class ETOPO_var_nc(object):
    """
    """
    def __init__(self, source):
        import netCDF4

        self.ncs = []
        for s in source:
            self.ncs.append(netCDF4.Dataset(s, 'r'))

        self.load_dims(dims=['lat', 'lon'])

    def keys(self):
        return ['elevation']

    def __getitem__(self, item):
        # elevation
        return self.data[item]

    def load_dims(self):
        dims = ['lat', 'lon']
        self.dims = {}
        for d in dims:
            self.dims[d] = self.ncs[0].variables[d][:]
            for nc in self.ncs[1:]:
                assert (self.dims[d] == nc.variables[d][:]).all()

    def subset(self, lat, lon, var):
        dims = {}

        yn = slice(
                np.nonzero(self.dims['lat'] <= lat.min())[0].max(),
                np.nonzero(self.dims['lat'] >= lat.max())[0].min() + 1)
        dims['lat'] = np.atleast_1d(self.dims['lat'][yn])

        lon_ext = np.array(
                (self.dims['lon'] - 2*360).tolist() +
                (self.dims['lon'] - 360).tolist() +
                self.dims['lon'].tolist() +
                (self.dims['lon'] + 360).tolist())
        xn_ext = np.array(4 * list(range(self.dims['lon'].shape[0])))
        xn_start = np.nonzero(lon_ext <= lon.min())[0].max()
        xn_end = np.nonzero(lon_ext >= lon.max())[0].min()
        xn = xn_ext[xn_start:xn_end+1]
        dims['lon'] = np.atleast_1d(lon_ext[xn_start:xn_end+1])

        #varin = []
        #for v in var:
        #    if v in self.keys():
        #        varin.append(v)

        subset = {}
        #for v, vin in zip(var, varin):
        #    subset[v] = ma.asanyarray(
        #            [self.ncs[0][vin][yn, xn]])
        subset = {'elevation': self.ncs[0].variables['z'][yn, xn]}

        return subset, dims

    def interpolate(self, lat, lon, var):
        """ Interpolate each var on the coordinates requested

        """

        subset, dims = self.subset(lat, lon, var)

        if np.all([y in dims['lat'] for y in lat]) & \
                np.all([x in dims['lon'] for x in lon]):
                    yn = np.nonzero([y in lat for y in dims['lat']])[0]
                    xn = np.nonzero([x in lon for x in dims['lon']])[0]
                    output = {}
                    for v in subset:
                        # output[v] = subset[v][dn, zn, yn, xn]
                        # Seriously that this is the way to do it?!!??
                        output[v] = subset[v][:, xn][yn]
                    return output

        # The output coordinates shall be created only once.
        points_out = []
        for latn in lat:
            for lonn in lon:
                points_out.append([latn, lonn])
        points_out = np.array(points_out)

        output = {}
        for v in var:
            output[v] = ma.masked_all(
                    (lat.size, lon.size),
                    dtype=subset[v].dtype)

            # The valid data
            idx = np.nonzero(~ma.getmaskarray(subset[v]))

            if idx[0].size > 0:
                points = np.array([
                    dims['lat'][idx[0]], dims['lon'][idx[1]]]).T
                values = subset[v][idx]

                # Interpolate along the dimensions that have more than one
                #   position, otherwise it means that the output is exactly
                #   on that coordinate.
                ind = np.array(
                        [np.unique(points[:, i]).size > 1 for i in
                            range(points.shape[1])])
                assert ind.any()

                values_out = griddata(
                        np.atleast_1d(np.squeeze(points[:, ind])),
                        values,
                        np.atleast_1d(np.squeeze(points_out[:, ind]))
                        )

                # Remap the interpolated value back into a 4D array
                idx = np.isfinite(values_out)
                for [y, x], out in zip(points_out[idx], values_out[idx]):
                    output[v][y==lat, x==lon] = out

        return output

    def extract(self, mode=None, **kwargs):
        """

            Possible scenarios:
              - Point:   lat{1},lon{1}
              - Section: [lat{1},lon{n} | lat{n},lon{1}]

              - Track:   lat{n},lon{n}
        """
        for k in kwargs:
            assert k in ['var', 'lat', 'lon'], \
                    "Wrong dimension to extract, check the manual"

        if 'var' in kwargs:
            var = np.atleast_1d(kwargs['var'])
        else:
            var = np.asanyarray(self.keys())

        lat = np.atleast_1d(kwargs['lat'])
        lon = np.atleast_1d(kwargs['lon'])

        try:
            if mode == 'nearest':
                output = self.closest(lat, lon, var)
            else:
                output = self.interpolate(lat, lon, var)
                for v in output:
                    output[v] = np.atleast_1d(np.squeeze(output[v]))
        except:
            print("Sorry, I was not able to extract the climatology.")
            return

        return output

class ETOPO(ETOPO_var_nc):
    """
    """
    def __init__(self, dbname='ETOPO'):
        #super(ETOPO, self).__init__(source=dbsource(dbname, 'elevation'))
        self.dbname = dbname
        self.data = {'elevation': None}

    def keys(self):
        return self.data.keys()

    def __getitem__(self, item):
        if self.data[item] is None:
            self.data[item] = ETOPO_var_nc(source=dbsource(self.dbname, item))

        return self.data[item]
<commit_msg>Cosmetics, relative import.<commit_after># -*- coding: utf-8 -*-

import numpy as np
from numpy import ma
from scipy.interpolate import griddata

from .utils import dbsource

def get_depth(lat, lon, cfg):
    """

    ATTENTION, conceptual error on the data near by Greenwich.
    url='http://opendap.ccst.inpe.br/Climatologies/ETOPO/etopo5.cdf'

    If I ever need to get depth from multiple points, check the history
      of this file. One day it was like that.
    """
    # This assert fails if it is a np.float64. Re-think this assert anyways.
    #assert type(lat) in [int, float]
    #assert type(lon) in [int, float]

    # if lat.shape != lon.shape:
    #            print "lat and lon must have the same size"

    try:
        try:
            etopo = netCDF4.Dataset(expanduser(cfg['file']))
        except:
            # FIXME, It must have a time limit defined here, otherwise it can
            #   get stuck trying to open the file.
            etopo = netCDF4.Dataset(expanduser(cfg['url']))
        x = etopo.variables['ETOPO05_X'][:]
        y = etopo.variables['ETOPO05_Y'][:]
    except:
        etopo = open_url(cfg['url']).ROSE
        x = etopo.ETOPO05_X[:]
        y = etopo.ETOPO05_Y[:]

    if lon < 0:
        lon += 360

    iini = (abs(lon - x)).argmin() - 2
    ifin = (abs(lon - x)).argmin() + 2
    jini = (abs(lat - y)).argmin() - 2
    jfin = (abs(lat - y)).argmin() + 2

    assert (iini >= 0) or (iini <= len(x)) or \
        (jini >= 0) or (jini <= len(y)), \
        "Sorry not ready to handle too close to boundaries"

    try:
        z = etopo.variables['ROSE'][jini:jfin, iini:ifin]
    except:
        z = etopo.ROSE[jini:jfin, iini:ifin]

    interpolator = RectBivariateSpline(x[iini:ifin], y[jini:jfin], z.T)
    return interpolator(lon, lat)[0][0]

class ETOPO_var_nc(object):
    """
    """
    def __init__(self, source):
        import netCDF4

        self.ncs = []
        for s in source:
            self.ncs.append(netCDF4.Dataset(s, 'r'))

        self.load_dims(dims=['lat', 'lon'])

    def keys(self):
        return ['elevation']

    def __getitem__(self, item):
        # elevation
        return self.data[item]

    def load_dims(self):
        dims = ['lat', 'lon']
        self.dims = {}
        for d in dims:
            self.dims[d] = self.ncs[0].variables[d][:]
            for nc in self.ncs[1:]:
                assert (self.dims[d] == nc.variables[d][:]).all()

    def subset(self, lat, lon, var):
        dims = {}

        yn = slice(
                np.nonzero(self.dims['lat'] <= lat.min())[0].max(),
                np.nonzero(self.dims['lat'] >= lat.max())[0].min() + 1)
        dims['lat'] = np.atleast_1d(self.dims['lat'][yn])

        lon_ext = np.array(
                (self.dims['lon'] - 2*360).tolist() +
                (self.dims['lon'] - 360).tolist() +
                self.dims['lon'].tolist() +
                (self.dims['lon'] + 360).tolist())
        xn_ext = np.array(4 * list(range(self.dims['lon'].shape[0])))
        xn_start = np.nonzero(lon_ext <= lon.min())[0].max()
        xn_end = np.nonzero(lon_ext >= lon.max())[0].min()
        xn = xn_ext[xn_start:xn_end+1]
        dims['lon'] = np.atleast_1d(lon_ext[xn_start:xn_end+1])

        #varin = []
        #for v in var:
        #    if v in self.keys():
        #        varin.append(v)

        subset = {}
        #for v, vin in zip(var, varin):
        #    subset[v] = ma.asanyarray(
        #            [self.ncs[0][vin][yn, xn]])
        subset = {'elevation': self.ncs[0].variables['z'][yn, xn]}

        return subset, dims

    def interpolate(self, lat, lon, var):
        """ Interpolate each var on the coordinates requested

        """

        subset, dims = self.subset(lat, lon, var)

        if np.all([y in dims['lat'] for y in lat]) & \
                np.all([x in dims['lon'] for x in lon]):
                    yn = np.nonzero([y in lat for y in dims['lat']])[0]
                    xn = np.nonzero([x in lon for x in dims['lon']])[0]
                    output = {}
                    for v in subset:
                        # output[v] = subset[v][dn, zn, yn, xn]
                        # Seriously that this is the way to do it?!!??
                        output[v] = subset[v][:, xn][yn]
                    return output

        # The output coordinates shall be created only once.
        points_out = []
        for latn in lat:
            for lonn in lon:
                points_out.append([latn, lonn])
        points_out = np.array(points_out)

        output = {}
        for v in var:
            output[v] = ma.masked_all(
                    (lat.size, lon.size),
                    dtype=subset[v].dtype)

            # The valid data
            idx = np.nonzero(~ma.getmaskarray(subset[v]))

            if idx[0].size > 0:
                points = np.array([
                    dims['lat'][idx[0]], dims['lon'][idx[1]]]).T
                values = subset[v][idx]

                # Interpolate along the dimensions that have more than one
                #   position, otherwise it means that the output is exactly
                #   on that coordinate.
                ind = np.array(
                        [np.unique(points[:, i]).size > 1 for i in
                            range(points.shape[1])])
                assert ind.any()

                values_out = griddata(
                        np.atleast_1d(np.squeeze(points[:, ind])),
                        values,
                        np.atleast_1d(np.squeeze(points_out[:, ind]))
                        )

                # Remap the interpolated value back into a 4D array
                idx = np.isfinite(values_out)
                for [y, x], out in zip(points_out[idx], values_out[idx]):
                    output[v][y==lat, x==lon] = out

        return output

    def extract(self, mode=None, **kwargs):
        """

            Possible scenarios:
              - Point:   lat{1},lon{1}
              - Section: [lat{1},lon{n} | lat{n},lon{1}]

              - Track:   lat{n},lon{n}
        """
        for k in kwargs:
            assert k in ['var', 'lat', 'lon'], \
                    "Wrong dimension to extract, check the manual"

        if 'var' in kwargs:
            var = np.atleast_1d(kwargs['var'])
        else:
            var = np.asanyarray(self.keys())

        lat = np.atleast_1d(kwargs['lat'])
        lon = np.atleast_1d(kwargs['lon'])

        try:
            if mode == 'nearest':
                output = self.closest(lat, lon, var)
            else:
                output = self.interpolate(lat, lon, var)
                for v in output:
                    output[v] = np.atleast_1d(np.squeeze(output[v]))
        except:
            print("Sorry, I was not able to extract the climatology.")
            return

        return output

class ETOPO(ETOPO_var_nc):
    """
    """
    def __init__(self, dbname='ETOPO'):
        #super(ETOPO, self).__init__(source=dbsource(dbname, 'elevation'))
        self.dbname = dbname
        self.data = {'elevation': None}

    def keys(self):
        return self.data.keys()

    def __getitem__(self, item):
        if self.data[item] is None:
            self.data[item] = ETOPO_var_nc(source=dbsource(self.dbname, item))

        return self.data[item]
<|endoftext|>


<commit_before>from __future__ import absolute_import

"""
    mesh
    ~~~~

    This module contains the Mesh, MeshData, and Material classes.
    This documentation was auto-generated from the mesh.py file.
"""
import numpy as np
from pyglet import gl
from .utils import gl as ugl
from .texture import MockTexture
from . import mixins
from . import shader

class MeshData(object):

    def __init__(self, vertices, face_indices, normals, texcoords):
        """
        Collects all vertex data for rendering in 3D graphics packages.

        Args:
            vertices (list): Nx3 vertex array
            face_indices (list): Nx3 Face index array (0-indexed)
            normals (list): Nx3 normals array
            texture_uv (list): Nx2 texture_uv array

        Returns:
            MeshData object
        """
        # CPU Data
        self.vertices = np.array(vertices, dtype=float).reshape((-1, 3))
        self.face_indices = np.array(face_indices, dtype=int)
        self.normals = np.array(normals, dtype=float).reshape((-1, 3))
        self.texcoords = np.array(texcoords, dtype=float).reshape((-1, 2))

class Material(object):

    def __init__(self, diffuse=[.8, .8, .8], spec_weight=0., specular=[0., 0., 0.],
                 ambient=[0., 0., 0.], opacity=1., flat_shading=False):
        self.diffuse = diffuse
        self.spec_weight = spec_weight
        self.specular = specular
        self.ambient = ambient
        self.opacity = opacity
        self.flat_shading = flat_shading

class MeshLoader(object):

    def __init__(self, name, meshdata, material=None):
        """Creates various types of Meshes from MeshData and Material objects."""

        self.name = name
        self.meshdata = meshdata
        self.material = material

    def load_mesh(self, **kwargs):
        from collections import Iterable

        """Construct a Mesh object"""
        uniforms = []
        if self.material:
            for key, val in self.material.__dict__.items():
                if not isinstance(val, Iterable):
                    val = int(val) if isinstance(val, bool) else val
                    val = [val]
                uniforms.append(shader.Uniform(key, *val))

        return Mesh(self.name, self.meshdata.vertices, self.meshdata.normals, self.meshdata.texcoords, uniforms=uniforms, **kwargs)

class EmptyMesh(mixins.PhysicalNode):

    def __init__(self, *args, **kwargs):
        super(EmptyMesh, self).__init__(*args, **kwargs)

    def _draw(self, shader=None):
        pass

class Mesh(EmptyMesh, mixins.Picklable):

    drawstyle = {'fill': gl.GL_TRIANGLES, 'line': gl.GL_LINE_LOOP, 'point': gl.GL_POINTS}

    def __init__(self, name, vertices, normals, texcoords, uniforms=list(), drawstyle='fill', visible=True, point_size=4,
                 **kwargs):
        """
        Returns a Mesh object, containing the position, rotation, and color info of an OpenGL Mesh.

        Meshes have two coordinate system, the "local" and "world" systems, on which the transforms are performed
        sequentially.  This allows them to be placed in the scene while maintaining a relative position to one another.

        .. note:: Meshes are not usually instantiated directly, but from a 3D file, like the WavefrontReader .obj and .mtl files.

        Args:
            mesh_data (MeshData): MeshData object containing the vertex data to be displayed.
            uniforms (list): List of Uniform or UniformGroup instances, containing data to send to shader when drawing the Mesh.
            drawstyle (str): 'point': only vertices, 'line': points and edges, 'fill': points, edges, and faces (full)
            visible (bool): whether the Mesh is available to be rendered.  To make hidden (invisible), set to False.

        Attributes:
            local (:py:class:`.Physical`): The local position and rotation of the Mesh.  This shift is done first, so should be used for general positioning.  If *centered* is set to False, the local position will be set as the mean vertex coordinates from :py:class:`.MeshData`, so Mesh will appear in the same location as the Mesh's original position in its file.
            world (:py:class:`.Physical`): The world position and rotation of the Mesh.  World position is used for moving an object about another point, so that multiple objects can be moved at once and retain their inter-object distances and orientations.  By default, set to the origin.
        Returns:
            Mesh instance
        """
        super(Mesh, self).__init__(**kwargs)

        self.name = name

        # Mesh Data
        self.vertices = np.array(vertices, dtype=float)
        self.normals = np.array(normals, dtype=float)
        self.texcoords = np.array(texcoords, dtype=float)

        # Convert Mean position into Global Coordinates. If "centered" is True, though, simply leave global position to 0
        vertex_mean = np.mean(self.vertices, axis=0)
        self.vertices -= vertex_mean
        #: :py:class:`.Physical`, World Mesh coordinates
        #: Local Mesh coordinates (Physical type)
        self.uniforms = uniforms

        #: Pyglet texture object for mapping an image file to the vertices (set using Mesh.load_texture())
        self.texture = MockTexture()
        self.drawstyle = drawstyle
        self.point_size = point_size

        #: Bool: if the Mesh is visible for rendering. If false, will not be rendered.
        self.visible = visible
        self.vao = None

    def _draw(self, shader=None, *args, **kwargs):
        super(Mesh, self)._draw(*args, **kwargs)

        if not self.vao:
            self.vao = ugl.VAO(self.vertices, self.normals, self.texcoords)

        self.update()

        if self.visible:

            # Change Material to Mesh's
            for uniform in self.uniforms:
                uniform.send_to(shader)

            # Send Model and Normal Matrix to shader.
            shader.uniform_matrixf('model_matrix', self.model_matrix_global.T.ravel())
            shader.uniform_matrixf('normal_matrix', self.normal_matrix_global.T.ravel())

            # Set Point Size, if drawing a point cloud
            if self.drawstyle == 'point':
                gl.glPointSize(int(self.point_size))

            # Bind the VAO and Texture, and draw.
            with self.vao, self.texture as texture:
                texture.send_to(shader)
                gl.glDrawArrays(Mesh.drawstyle[self.drawstyle], 0, self.vertices.size)<commit_msg>reinstated Mesh centering--it's an important option for controlling your scene layout.<commit_after>from __future__ import absolute_import

"""
    mesh
    ~~~~

    This module contains the Mesh, MeshData, and Material classes.
    This documentation was auto-generated from the mesh.py file.
"""
import numpy as np
from pyglet import gl
from .utils import gl as ugl
from .texture import MockTexture
from . import mixins
from . import shader

class MeshData(object):

    def __init__(self, vertices, face_indices, normals, texcoords):
        """
        Collects all vertex data for rendering in 3D graphics packages.

        Args:
            vertices (list): Nx3 vertex array
            face_indices (list): Nx3 Face index array (0-indexed)
            normals (list): Nx3 normals array
            texture_uv (list): Nx2 texture_uv array

        Returns:
            MeshData object
        """
        # CPU Data
        self.vertices = np.array(vertices, dtype=float).reshape((-1, 3))
        self.face_indices = np.array(face_indices, dtype=int)
        self.normals = np.array(normals, dtype=float).reshape((-1, 3))
        self.texcoords = np.array(texcoords, dtype=float).reshape((-1, 2))

class Material(object):

    def __init__(self, diffuse=[.8, .8, .8], spec_weight=0., specular=[0., 0., 0.],
                 ambient=[0., 0., 0.], opacity=1., flat_shading=False):
        self.diffuse = diffuse
        self.spec_weight = spec_weight
        self.specular = specular
        self.ambient = ambient
        self.opacity = opacity
        self.flat_shading = flat_shading

class MeshLoader(object):

    def __init__(self, name, meshdata, material=None):
        """Creates various types of Meshes from MeshData and Material objects."""

        self.name = name
        self.meshdata = meshdata
        self.material = material

    def load_mesh(self, **kwargs):
        from collections import Iterable

        """Construct a Mesh object"""
        uniforms = []
        if self.material:
            for key, val in self.material.__dict__.items():
                if not isinstance(val, Iterable):
                    val = int(val) if isinstance(val, bool) else val
                    val = [val]
                uniforms.append(shader.Uniform(key, *val))

        return Mesh(self.name, self.meshdata.vertices, self.meshdata.normals, self.meshdata.texcoords, uniforms=uniforms, **kwargs)

class EmptyMesh(mixins.PhysicalNode):

    def __init__(self, *args, **kwargs):
        super(EmptyMesh, self).__init__(*args, **kwargs)

    def _draw(self, shader=None):
        pass

class Mesh(EmptyMesh, mixins.Picklable):

    drawstyle = {'fill': gl.GL_TRIANGLES, 'line': gl.GL_LINE_LOOP, 'point': gl.GL_POINTS}

    def __init__(self, name, vertices, normals, texcoords, uniforms=list(), drawstyle='fill', visible=True, point_size=4,
                 centered=False, **kwargs):
        """
        Returns a Mesh object, containing the position, rotation, and color info of an OpenGL Mesh.

        Meshes have two coordinate system, the "local" and "world" systems, on which the transforms are performed
        sequentially.  This allows them to be placed in the scene while maintaining a relative position to one another.

        .. note:: Meshes are not usually instantiated directly, but from a 3D file, like the WavefrontReader .obj and .mtl files.

        Args:
            mesh_data (MeshData): MeshData object containing the vertex data to be displayed.
            uniforms (list): List of Uniform or UniformGroup instances, containing data to send to shader when drawing the Mesh.
            drawstyle (str): 'point': only vertices, 'line': points and edges, 'fill': points, edges, and faces (full)
            visible (bool): whether the Mesh is available to be rendered.  To make hidden (invisible), set to False.

        Attributes:
            local (:py:class:`.Physical`): The local position and rotation of the Mesh.  This shift is done first, so should be used for general positioning.  If *centered* is set to False, the local position will be set as the mean vertex coordinates from :py:class:`.MeshData`, so Mesh will appear in the same location as the Mesh's original position in its file.
            world (:py:class:`.Physical`): The world position and rotation of the Mesh.  World position is used for moving an object about another point, so that multiple objects can be moved at once and retain their inter-object distances and orientations.  By default, set to the origin.
        Returns:
            Mesh instance
        """
        super(Mesh, self).__init__(**kwargs)

        self.name = name

        # Mesh Data
        self.vertices = np.array(vertices, dtype=float)
        self.normals = np.array(normals, dtype=float)
        self.texcoords = np.array(texcoords, dtype=float)

        # Convert Mean position into Global Coordinates. If "centered" is True, though, simply leave global position to 0
        vertex_mean = np.mean(self.vertices, axis=0)
        self.vertices -= vertex_mean
        self.position = (0., 0., 0.) if centered else vertex_mean
        #: :py:class:`.Physical`, World Mesh coordinates
        #: Local Mesh coordinates (Physical type)
        self.uniforms = uniforms

        #: Pyglet texture object for mapping an image file to the vertices (set using Mesh.load_texture())
        self.texture = MockTexture()
        self.drawstyle = drawstyle
        self.point_size = point_size

        #: Bool: if the Mesh is visible for rendering. If false, will not be rendered.
        self.visible = visible
        self.vao = None

    def _draw(self, shader=None, *args, **kwargs):
        super(Mesh, self)._draw(*args, **kwargs)

        if not self.vao:
            self.vao = ugl.VAO(self.vertices, self.normals, self.texcoords)

        self.update()

        if self.visible:

            # Change Material to Mesh's
            for uniform in self.uniforms:
                uniform.send_to(shader)

            # Send Model and Normal Matrix to shader.
            shader.uniform_matrixf('model_matrix', self.model_matrix_global.T.ravel())
            shader.uniform_matrixf('normal_matrix', self.normal_matrix_global.T.ravel())

            # Set Point Size, if drawing a point cloud
            if self.drawstyle == 'point':
                gl.glPointSize(int(self.point_size))

            # Bind the VAO and Texture, and draw.
            with self.vao, self.texture as texture:
                texture.send_to(shader)
                gl.glDrawArrays(Mesh.drawstyle[self.drawstyle], 0, self.vertices.size)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

""" License

    Copyright (C) 2013 YunoHost

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published
    by the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program; if not, see http://www.gnu.org/licenses

"""

""" yunohost_backup.py

    Manage backups
"""
import os
import re
import sys
import json
import errno
import time
import shutil
import tarfile

from moulinette.core import MoulinetteError
from moulinette.utils.log import getActionLogger

backup_path   = '/home/yunohost.backup'
archives_path = '%s/archives' % backup_path

logger = getActionLogger('yunohost.backup')

def backup_create(name=None, description=None, output_directory=None,
                  no_compress=False, hooks=[], ignore_apps=False):
    """
    Create a backup local archive

    Keyword arguments:
        name -- Name of the backup archive
        description -- Short description of the backup
        output_directory -- Output directory for the backup
        no_compress -- Do not create an archive file
        hooks -- List of backup hooks names to execute
        ignore_apps -- Do not backup apps

    """
    # TODO: Add a 'clean' argument to clean output directory
    from yunohost.hook import hook_add
    from yunohost.hook import hook_callback

    tmp_dir = None

    # Validate and define backup name
    timestamp = int(time.time())
    if not name:
        name = str(timestamp)
    if name in backup_list()['archives']:
        raise MoulinetteError(errno.EINVAL,
                              m18n.n('backup_archive_name_exists'))

    # Validate additional arguments
    if no_compress and not output_directory:
        raise MoulinetteError(errno.EINVAL,
                              m18n.n('backup_output_directory_required'))
    if output_directory:
        output_directory = os.path.abspath(output_directory)

        # Check for forbidden folders
        if output_directory.startswith(archives_path) or \
           re.match(r'^/(|(bin|boot|dev|etc|lib|root|run|sbin|sys|usr|var)(|/.*))$',
                    output_directory):
            logger.error("forbidden output directory '%'", output_directory)
            raise MoulinetteError(errno.EINVAL,
                                  m18n.n('backup_output_directory_forbidden'))

        # Create the output directory
        if not os.path.isdir(output_directory):
            logger.info("creating output directory '%s'", output_directory)
            os.makedirs(output_directory, 0750)
        # Check that output directory is empty
        elif no_compress and os.listdir(output_directory):
            logger.error("not empty output directory '%'", output_directory)
            raise MoulinetteError(errno.EIO,
                                  m18n.n('backup_output_directory_not_empty'))

        # Define temporary directory
        if no_compress:
            tmp_dir = output_directory
    else:
        output_directory = archives_path

    # Create temporary directory
    if not tmp_dir:
        tmp_dir = "%s/tmp/%s" % (backup_path, name)
        if os.path.isdir(tmp_dir):
            logger.warning("temporary directory for backup '%s' already exists",
                           tmp_dir)
            os.system('rm -rf %s' % tmp_dir)
        try:
            os.mkdir(tmp_dir, 0750)
        except OSError:
            # Create temporary directory recursively
            os.makedirs(tmp_dir, 0750)
            os.system('chown -hR admin: %s' % backup_path)
        else:
            os.system('chown -hR admin: %s' % tmp_dir)

    # Initialize backup info
    info = {
        'description': description or '',
        'created_at': timestamp,
        'apps': {},
    }

    # Add apps backup hook
    if not ignore_apps:
        from yunohost.app import app_info
        try:
            for app_id in os.listdir('/etc/yunohost/apps'):
                hook = '/etc/yunohost/apps/%s/scripts/backup' % app_id
                if os.path.isfile(hook):
                    hook_add(app_id, hook)

                    # Add app info
                    i = app_info(app_id)
                    info['apps'][app_id] = {
                        'version': i['version'],
                    }
                else:
                    logger.warning("unable to find app's backup hook '%s'",
                                   hook)
                    msignals.display(m18n.n('unbackup_app', app_id),
                                     'warning')
        except IOError as e:
            logger.info("unable to add apps backup hook: %s", str(e))

    # Run hooks
    msignals.display(m18n.n('backup_running_hooks'))
    hook_callback('backup', hooks, args=[tmp_dir])

    # Create backup info file
    with open("%s/info.json" % tmp_dir, 'w') as f:
        f.write(json.dumps(info))

    # Create the archive
    if not no_compress:
        msignals.display(m18n.n('backup_creating_archive'))
        archive_file = "%s/%s.tar.gz" % (output_directory, name)
        try:
            tar = tarfile.open(archive_file, "w:gz")
        except:
            tar = None

            # Create the archives directory and retry
            if not os.path.isdir(archives_path):
                os.mkdir(archives_path, 0750)
                try:
                    tar = tarfile.open(archive_file, "w:gz")
                except:
                    logger.exception("unable to open the archive '%s' for writing "
                                     "after creating directory '%s'",
                                     archive_file, archives_path)
                    tar = None
            else:
                logger.exception("unable to open the archive '%s' for writing",
                                 archive_file)
            if tar is None:
                raise MoulinetteError(errno.EIO,
                                      m18n.n('backup_archive_open_failed'))
        tar.add(tmp_dir, arcname='')
        tar.close()

        # Copy info file
        os.system('mv %s/info.json %s/%s.info.json' %
                  (tmp_dir, archives_path, name))

    # Clean temporary directory
    if tmp_dir != output_directory:
        os.system('rm -rf %s' % tmp_dir)

    msignals.display(m18n.n('backup_complete'), 'success')

def backup_restore(name, hooks=[], ignore_apps=False, force=False):
    """
    Restore from a local backup archive

    Keyword argument:
        name -- Name of the local backup archive
        hooks -- List of restoration hooks names to execute
        ignore_apps -- Do not restore apps
        force -- Force restauration on an already installed system

    """
    from yunohost.hook import hook_add
    from yunohost.hook import hook_callback

    # Retrieve and open the archive
    archive_file = backup_info(name)['path']
    try:
        tar = tarfile.open(archive_file, "r:gz")
    except:
        logger.exception("unable to open the archive '%s' for reading",
                         archive_file)
        raise MoulinetteError(errno.EIO, m18n.n('backup_archive_open_failed'))

    # Check temporary directory
    tmp_dir = "%s/tmp/%s" % (backup_path, name)
    if os.path.isdir(tmp_dir):
        logger.warning("temporary directory for restoration '%s' already exists",
                       tmp_dir)
        os.system('rm -rf %s' % tmp_dir)

    # Extract the tarball
    msignals.display(m18n.n('backup_extracting_archive'))
    tar.extractall(tmp_dir)
    tar.close()

    # Retrieve backup info
    try:
        with open("%s/info.json" % tmp_dir, 'r') as f:
            info = json.load(f)
    except IOError:
        logger.error("unable to retrieve backup info from '%s/info.json'",
                     tmp_dir)
        raise MoulinetteError(errno.EIO, m18n.n('backup_invalid_archive'))
    else:
        logger.info("restoring from backup '%s' created on %s", name,
                    time.ctime(info['created_at']))

    # Retrieve domain from the backup
    try:
        with open("%s/yunohost/current_host" % tmp_dir, 'r') as f:
            domain = f.readline().rstrip()
    except IOError:
        logger.error("unable to retrieve domain from '%s/yunohost/current_host'",
                     tmp_dir)
        raise MoulinetteError(errno.EIO, m18n.n('backup_invalid_archive'))

    # Check if YunoHost is installed
    if os.path.isfile('/etc/yunohost/installed'):
        msignals.display(m18n.n('yunohost_already_installed'), 'warning')
        if not force:
            try:
                # Ask confirmation for restoring
                i = msignals.prompt(m18n.n('restore_confirm_yunohost_installed',
                                           answers='y/N'))
            except NotImplemented:
                pass
            else:
                if i == 'y' or i == 'Y':
                    force = True
            if not force:
                raise MoulinetteError(errno.EEXIST, m18n.n('restore_failed'))
    else:
        from yunohost.tools import tools_postinstall
        logger.info("executing the post-install...")
        tools_postinstall(domain, 'yunohost', True)

    # Add apps restore hook
    if not ignore_apps:
        for app_id in info['apps'].keys():
            hook = "/etc/yunohost/apps/%s/scripts/restore" % app_id
            if os.path.isfile(hook):
                hook_add(app_id, hook)
                logger.info("app '%s' will be restored", app_id)
            else:
                msignals.display(m18n.n('unrestore_app', app_id), 'warning')

    # Run hooks
    msignals.display(m18n.n('restore_running_hooks'))
    hook_callback('restore', hooks, args=[tmp_dir])

    # Remove temporary directory
    os.system('rm -rf %s' % tmp_dir)

    msignals.display(m18n.n('restore_complete'), 'success')

def backup_list():
    """
    List available local backup archives

    """
    result = []

    try:
        # Retrieve local archives
        archives = os.listdir(archives_path)
    except IOError as e:
        logging.info("unable to iterate over local archives: %s", str(e))
    else:
        # Iterate over local archives
        for f in archives:
            try:
                name = f[:f.rindex('.tar.gz')]
            except ValueError:
                continue
            result.append(name)

    return { 'archives': result }

def backup_info(name):
    """
    Get info about a local backup archive

    Keyword arguments:
        name -- Name of the local backup archive

    """
    archive_file = '%s/%s.tar.gz' % (archives_path, name)
    if not os.path.isfile(archive_file):
        logger.error("no local backup archive found at '%s'", archive_file)
        raise MoulinetteError(errno.EIO, m18n.n('backup_archive_name_unknown'))

    info_file = "%s/%s.info.json" % (archives_path, name)
    try:
        with open(info_file) as f:
            # Retrieve backup info
            info = json.load(f)
    except:
        # TODO: Attempt to extract backup info file from tarball
        logger.exception("unable to retrive backup info file '%s'",
                         info_file)
        raise MoulinetteError(errno.EIO, m18n.n('backup_invalid_archive'))

    return {
        'path': archive_file,
        'created_at': time.strftime(m18n.n('format_datetime_short'),
                                    time.gmtime(info['created_at'])),
        'description': info['description'],
        'apps': info['apps'],
    }
<commit_msg>[fix] Catch proper exception in backup_list (fix #65)<commit_after># -*- coding: utf-8 -*-

""" License

    Copyright (C) 2013 YunoHost

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published
    by the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program; if not, see http://www.gnu.org/licenses

"""

""" yunohost_backup.py

    Manage backups
"""
import os
import re
import sys
import json
import errno
import time
import shutil
import tarfile

from moulinette.core import MoulinetteError
from moulinette.utils.log import getActionLogger

backup_path   = '/home/yunohost.backup'
archives_path = '%s/archives' % backup_path

logger = getActionLogger('yunohost.backup')

def backup_create(name=None, description=None, output_directory=None,
                  no_compress=False, hooks=[], ignore_apps=False):
    """
    Create a backup local archive

    Keyword arguments:
        name -- Name of the backup archive
        description -- Short description of the backup
        output_directory -- Output directory for the backup
        no_compress -- Do not create an archive file
        hooks -- List of backup hooks names to execute
        ignore_apps -- Do not backup apps

    """
    # TODO: Add a 'clean' argument to clean output directory
    from yunohost.hook import hook_add
    from yunohost.hook import hook_callback

    tmp_dir = None

    # Validate and define backup name
    timestamp = int(time.time())
    if not name:
        name = str(timestamp)
    if name in backup_list()['archives']:
        raise MoulinetteError(errno.EINVAL,
                              m18n.n('backup_archive_name_exists'))

    # Validate additional arguments
    if no_compress and not output_directory:
        raise MoulinetteError(errno.EINVAL,
                              m18n.n('backup_output_directory_required'))
    if output_directory:
        output_directory = os.path.abspath(output_directory)

        # Check for forbidden folders
        if output_directory.startswith(archives_path) or \
           re.match(r'^/(|(bin|boot|dev|etc|lib|root|run|sbin|sys|usr|var)(|/.*))$',
                    output_directory):
            logger.error("forbidden output directory '%'", output_directory)
            raise MoulinetteError(errno.EINVAL,
                                  m18n.n('backup_output_directory_forbidden'))

        # Create the output directory
        if not os.path.isdir(output_directory):
            logger.info("creating output directory '%s'", output_directory)
            os.makedirs(output_directory, 0750)
        # Check that output directory is empty
        elif no_compress and os.listdir(output_directory):
            logger.error("not empty output directory '%'", output_directory)
            raise MoulinetteError(errno.EIO,
                                  m18n.n('backup_output_directory_not_empty'))

        # Define temporary directory
        if no_compress:
            tmp_dir = output_directory
    else:
        output_directory = archives_path

    # Create temporary directory
    if not tmp_dir:
        tmp_dir = "%s/tmp/%s" % (backup_path, name)
        if os.path.isdir(tmp_dir):
            logger.warning("temporary directory for backup '%s' already exists",
                           tmp_dir)
            os.system('rm -rf %s' % tmp_dir)
        try:
            os.mkdir(tmp_dir, 0750)
        except OSError:
            # Create temporary directory recursively
            os.makedirs(tmp_dir, 0750)
            os.system('chown -hR admin: %s' % backup_path)
        else:
            os.system('chown -hR admin: %s' % tmp_dir)

    # Initialize backup info
    info = {
        'description': description or '',
        'created_at': timestamp,
        'apps': {},
    }

    # Add apps backup hook
    if not ignore_apps:
        from yunohost.app import app_info
        try:
            for app_id in os.listdir('/etc/yunohost/apps'):
                hook = '/etc/yunohost/apps/%s/scripts/backup' % app_id
                if os.path.isfile(hook):
                    hook_add(app_id, hook)

                    # Add app info
                    i = app_info(app_id)
                    info['apps'][app_id] = {
                        'version': i['version'],
                    }
                else:
                    logger.warning("unable to find app's backup hook '%s'",
                                   hook)
                    msignals.display(m18n.n('unbackup_app', app_id),
                                     'warning')
        except IOError as e:
            logger.info("unable to add apps backup hook: %s", str(e))

    # Run hooks
    msignals.display(m18n.n('backup_running_hooks'))
    hook_callback('backup', hooks, args=[tmp_dir])

    # Create backup info file
    with open("%s/info.json" % tmp_dir, 'w') as f:
        f.write(json.dumps(info))

    # Create the archive
    if not no_compress:
        msignals.display(m18n.n('backup_creating_archive'))
        archive_file = "%s/%s.tar.gz" % (output_directory, name)
        try:
            tar = tarfile.open(archive_file, "w:gz")
        except:
            tar = None

            # Create the archives directory and retry
            if not os.path.isdir(archives_path):
                os.mkdir(archives_path, 0750)
                try:
                    tar = tarfile.open(archive_file, "w:gz")
                except:
                    logger.exception("unable to open the archive '%s' for writing "
                                     "after creating directory '%s'",
                                     archive_file, archives_path)
                    tar = None
            else:
                logger.exception("unable to open the archive '%s' for writing",
                                 archive_file)
            if tar is None:
                raise MoulinetteError(errno.EIO,
                                      m18n.n('backup_archive_open_failed'))
        tar.add(tmp_dir, arcname='')
        tar.close()

        # Copy info file
        os.system('mv %s/info.json %s/%s.info.json' %
                  (tmp_dir, archives_path, name))

    # Clean temporary directory
    if tmp_dir != output_directory:
        os.system('rm -rf %s' % tmp_dir)

    msignals.display(m18n.n('backup_complete'), 'success')

def backup_restore(name, hooks=[], ignore_apps=False, force=False):
    """
    Restore from a local backup archive

    Keyword argument:
        name -- Name of the local backup archive
        hooks -- List of restoration hooks names to execute
        ignore_apps -- Do not restore apps
        force -- Force restauration on an already installed system

    """
    from yunohost.hook import hook_add
    from yunohost.hook import hook_callback

    # Retrieve and open the archive
    archive_file = backup_info(name)['path']
    try:
        tar = tarfile.open(archive_file, "r:gz")
    except:
        logger.exception("unable to open the archive '%s' for reading",
                         archive_file)
        raise MoulinetteError(errno.EIO, m18n.n('backup_archive_open_failed'))

    # Check temporary directory
    tmp_dir = "%s/tmp/%s" % (backup_path, name)
    if os.path.isdir(tmp_dir):
        logger.warning("temporary directory for restoration '%s' already exists",
                       tmp_dir)
        os.system('rm -rf %s' % tmp_dir)

    # Extract the tarball
    msignals.display(m18n.n('backup_extracting_archive'))
    tar.extractall(tmp_dir)
    tar.close()

    # Retrieve backup info
    try:
        with open("%s/info.json" % tmp_dir, 'r') as f:
            info = json.load(f)
    except IOError:
        logger.error("unable to retrieve backup info from '%s/info.json'",
                     tmp_dir)
        raise MoulinetteError(errno.EIO, m18n.n('backup_invalid_archive'))
    else:
        logger.info("restoring from backup '%s' created on %s", name,
                    time.ctime(info['created_at']))

    # Retrieve domain from the backup
    try:
        with open("%s/yunohost/current_host" % tmp_dir, 'r') as f:
            domain = f.readline().rstrip()
    except IOError:
        logger.error("unable to retrieve domain from '%s/yunohost/current_host'",
                     tmp_dir)
        raise MoulinetteError(errno.EIO, m18n.n('backup_invalid_archive'))

    # Check if YunoHost is installed
    if os.path.isfile('/etc/yunohost/installed'):
        msignals.display(m18n.n('yunohost_already_installed'), 'warning')
        if not force:
            try:
                # Ask confirmation for restoring
                i = msignals.prompt(m18n.n('restore_confirm_yunohost_installed',
                                           answers='y/N'))
            except NotImplemented:
                pass
            else:
                if i == 'y' or i == 'Y':
                    force = True
            if not force:
                raise MoulinetteError(errno.EEXIST, m18n.n('restore_failed'))
    else:
        from yunohost.tools import tools_postinstall
        logger.info("executing the post-install...")
        tools_postinstall(domain, 'yunohost', True)

    # Add apps restore hook
    if not ignore_apps:
        for app_id in info['apps'].keys():
            hook = "/etc/yunohost/apps/%s/scripts/restore" % app_id
            if os.path.isfile(hook):
                hook_add(app_id, hook)
                logger.info("app '%s' will be restored", app_id)
            else:
                msignals.display(m18n.n('unrestore_app', app_id), 'warning')

    # Run hooks
    msignals.display(m18n.n('restore_running_hooks'))
    hook_callback('restore', hooks, args=[tmp_dir])

    # Remove temporary directory
    os.system('rm -rf %s' % tmp_dir)

    msignals.display(m18n.n('restore_complete'), 'success')

def backup_list():
    """
    List available local backup archives

    """
    result = []

    try:
        # Retrieve local archives
        archives = os.listdir(archives_path)
    except OSError as e:
        logging.info("unable to iterate over local archives: %s", str(e))
    else:
        # Iterate over local archives
        for f in archives:
            try:
                name = f[:f.rindex('.tar.gz')]
            except ValueError:
                continue
            result.append(name)

    return { 'archives': result }

def backup_info(name):
    """
    Get info about a local backup archive

    Keyword arguments:
        name -- Name of the local backup archive

    """
    archive_file = '%s/%s.tar.gz' % (archives_path, name)
    if not os.path.isfile(archive_file):
        logger.error("no local backup archive found at '%s'", archive_file)
        raise MoulinetteError(errno.EIO, m18n.n('backup_archive_name_unknown'))

    info_file = "%s/%s.info.json" % (archives_path, name)
    try:
        with open(info_file) as f:
            # Retrieve backup info
            info = json.load(f)
    except:
        # TODO: Attempt to extract backup info file from tarball
        logger.exception("unable to retrive backup info file '%s'",
                         info_file)
        raise MoulinetteError(errno.EIO, m18n.n('backup_invalid_archive'))

    return {
        'path': archive_file,
        'created_at': time.strftime(m18n.n('format_datetime_short'),
                                    time.gmtime(info['created_at'])),
        'description': info['description'],
        'apps': info['apps'],
    }
<|endoftext|>


<commit_before># This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free Software
# Foundation; either version 2 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License along with
# this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
# copyright 2003-2010 Sylvain Thenault, all rights reserved.
# contact mailto:thenault@gmail.com
#
# This file is part of logilab-astng.
#
# logilab-astng is free software: you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by the
# Free Software Foundation, either version 2.1 of the License, or (at your
# option) any later version.
#
# logilab-astng is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
# for more details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with logilab-astng. If not, see <http://www.gnu.org/licenses/>.
"""this module contains a set of functions to create astng trees from scratch
(build_* functions) or from living object (object_build_* functions)
"""

__docformat__ = "restructuredtext en"

import sys
from os.path import abspath
from inspect import (getargspec, isdatadescriptor, isfunction, ismethod,
                     ismethoddescriptor, isclass, isbuiltin)

from logilab.astng import nodes
from logilab.astng.bases import Generator
from logilab.astng.manager import ASTNGManager
MANAGER = ASTNGManager()

def _attach_local_node(parent, node, name):
    node.name = name # needed by add_local_node
    parent.add_local_node(node)

_marker = object()

def attach_dummy_node(node, name, object=_marker):
    """create a dummy node and register it in the locals of the given
    node with the specified name
    """
    enode = nodes.EmptyNode()
    enode.object = object
    _attach_local_node(node, enode, name)

nodes.EmptyNode.has_underlying_object = lambda self: self.object is not _marker

def attach_const_node(node, name, value):
    """create a Const node and register it in the locals of the given
    node with the specified name
    """
    if not name in node.special_attributes:
        _attach_local_node(node, nodes.const_factory(value), name)

def attach_import_node(node, modname, membername):
    """create a From node and register it in the locals of the given
    node with the specified name
    """
    from_node = nodes.From(modname, [(membername, None)])
    _attach_local_node(node, from_node, membername)

def build_module(name, doc=None):
    """create and initialize a astng Module node"""
    node = nodes.Module(name, doc, pure_python=False)
    node.package = False
    node.parent = None
    return node

def build_class(name, basenames=(), doc=None):
    """create and initialize a astng Class node"""
    node = nodes.Class(name, doc)
    for base in basenames:
        basenode = nodes.Name()
        basenode.name = base
        node.bases.append(basenode)
        basenode.parent = node
    return node

def build_function(name, args=None, defaults=None, flag=0, doc=None):
    """create and initialize a astng Function node"""
    args, defaults = args or [], defaults or []
    # first argument is now a list of decorators
    func = nodes.Function(name, doc)
    func.args = argsnode = nodes.Arguments()
    argsnode.args = []
    for arg in args:
        argsnode.args.append(nodes.Name())
        argsnode.args[-1].name = arg
        argsnode.args[-1].parent = argsnode
    argsnode.defaults = []
    for default in defaults:
        argsnode.defaults.append(nodes.const_factory(default))
        argsnode.defaults[-1].parent = argsnode
    argsnode.kwarg = None
    argsnode.vararg = None
    argsnode.parent = func
    if args:
        register_arguments(func)
    return func

# def build_name_assign(name, value):
#     """create and initialize an astng Assign for a name assignment"""
#     return nodes.Assign([nodes.AssName(name, 'OP_ASSIGN')], nodes.Const(value))

# def build_attr_assign(name, value, attr='self'):
#     """create and initialize an astng Assign for an attribute assignment"""
#     return nodes.Assign([nodes.AssAttr(nodes.Name(attr), name, 'OP_ASSIGN')],
#                         nodes.Const(value))

def build_from_import(fromname, names):
    """create and initialize an astng From import statement"""
    return nodes.From(fromname, [(name, None) for name in names])

def register_arguments(func, args=None):
    """add given arguments to local

    args is a list that may contains nested lists
    (i.e. def func(a, (b, c, d)): ...)
    """
    if args is None:
        args = func.args.args
        if func.args.vararg:
            func.set_local(func.args.vararg, func.args)
        if func.args.kwarg:
            func.set_local(func.args.kwarg, func.args)
    for arg in args:
        if isinstance(arg, nodes.Name):
            func.set_local(arg.name, arg)
        else:
            register_arguments(func, arg.elts)

def object_build_class(node, member, localname):
    """create astng for a living class object"""
    basenames = [base.__name__ for base in member.__bases__]
    return _base_class_object_build(node, member, basenames,
                                    localname=localname)

def object_build_function(node, member, localname):
    """create astng for a living function object"""
    args, varargs, varkw, defaults = getargspec(member)
    if varargs is not None:
        args.append(varargs)
    if varkw is not None:
        args.append(varkw)
    func = build_function(getattr(member, '__name__', None) or localname, args,
                          defaults, member.func_code.co_flags, member.__doc__)
    node.add_local_node(func, localname)

def object_build_datadescriptor(node, member, name):
    """create astng for a living data descriptor object"""
    return _base_class_object_build(node, member, [], name)

def object_build_methoddescriptor(node, member, localname):
    """create astng for a living method descriptor object"""
    # FIXME get arguments ?
    func = build_function(getattr(member, '__name__', None) or localname,
                          doc=member.__doc__)
    # set node's arguments to None to notice that we have no information, not
    # and empty argument list
    func.args.args = None
    node.add_local_node(func, localname)

def _base_class_object_build(node, member, basenames, name=None, localname=None):
    """create astng for a living class object, with a given set of base names
    (e.g. ancestors)
    """
    klass = build_class(name or getattr(member, '__name__', None) or localname,
                        basenames, member.__doc__)
    klass._newstyle = isinstance(member, type)
    node.add_local_node(klass, localname)
    try:
        # limit the instantiation trick since it's too dangerous
        # (such as infinite test execution...)
        # this at least resolves common case such as Exception.args,
        # OSError.errno
        if issubclass(member, Exception):
            instdict = member().__dict__
        else:
            raise TypeError
    except:
        pass
    else:
        for name, obj in instdict.items():
            valnode = nodes.EmptyNode()
            valnode.object = obj
            valnode.parent = klass
            valnode.lineno = 1
            klass.instance_attrs[name] = [valnode]
    return klass

def imported_member(node, member, name):
    """consider a class/builtin member where __module__ != current module name

    check if it's sound valid and then add an import node, else use a dummy node
    """
    # /!\ some classes like ExtensionClass doesn't have a
    # __module__ attribute !
    member_module = getattr(member, '__module__', '__builtin__')
    try:
        getattr(sys.modules[member_module], name)
    except (KeyError, AttributeError):
        attach_dummy_node(node, name, member)
    else:
        attach_import_node(node, member_module, name)

class InspectBuilder(object):
    """class for building nodes from living object"""

    # astng from living objects ###############################################
    #
    # this is actually a really minimal representation, including only Module,
    # Function and Class nodes and some others as guessed

    def __init__(self):
        self._done = {}
        self._module = None

    def inspect_build(self, module, modname=None, path=None):
        """build astng from a living module (i.e. using inspect)
        this is used when there is no python source code available (either
        because it's a built-in module or because the .py is not available)
        """
        self._module = module
        if modname is None:
            modname = module.__name__
        node = build_module(modname, module.__doc__)
        node.file = node.path = path and abspath(path) or path
        MANAGER._cache[modname] = node
        node.package = hasattr(module, '__path__')
        self._done = {}
        self.object_build(node, module)
        return node

    def object_build(self, node, obj):
        """recursive method which create a partial ast from real objects
         (only function, class, and method are handled)
        """
        if obj in self._done:
            return self._done[obj]
        self._done[obj] = node
        for name in dir(obj):
            try:
                member = getattr(obj, name)
            except AttributeError:
                # damned ExtensionClass.Base, I know you're there !
                attach_dummy_node(node, name)
                continue
            if ismethod(member):
                member = member.im_func
            if isfunction(member):
                # verify this is not an imported function
                if member.func_code.co_filename != getattr(self._module, '__file__', None):
                    attach_dummy_node(node, name, member)
                    continue
                object_build_function(node, member, name)
            elif isbuiltin(member):
                if self.imported_member(node, member, name):
                    continue
                object_build_methoddescriptor(node, member, name)
            elif isclass(member):
                if self.imported_member(node, member, name):
                    continue
                if member in self._done:
                    class_node = self._done[member]
                    if not class_node in node.locals.get(name, ()):
                        node.add_local_node(class_node, name)
                else:
                    class_node = object_build_class(node, member, name)
                    # recursion
                    self.object_build(class_node, member)
                if name == '__class__' and class_node.parent is None:
                    class_node.parent = self._done[self._module]
            elif ismethoddescriptor(member):
                assert isinstance(member, object)
                object_build_methoddescriptor(node, member, name)
            elif isdatadescriptor(member):
                assert isinstance(member, object)
                object_build_datadescriptor(node, member, name)
            elif isinstance(member, (int, long, float, str, unicode)) or member is None:
                attach_const_node(node, name, member)
            else:
                # create an empty node so that the name is actually defined
                attach_dummy_node(node, name, member)

    def imported_member(self, node, member, name):
        """verify this is not an imported class or handle it"""
        modname = getattr(member, '__module__', None)
        if {'gtk': 'gtk._gtk'}.get(modname, modname) != self._module.__name__:
            imported_member(node, member, name)
            return True
        return False

### astng boot strapping ################################################### ###

_CONST_PROXY = {}
def astng_boot_strapping():
    """astng boot strapping the builtins module"""
    # this boot strapping is necessary since we need the Const nodes to
    # inspect_build builtins, and then we can proxy Const
    builder = InspectBuilder()
    from logilab.common.compat import builtins
    astng_builtin = builder.inspect_build(builtins)
    for cls in (bool, int, long, float, complex, str, unicode):
        _CONST_PROXY[cls] = astng_builtin.getattr(cls.__name__)[0] # XXX
    nonetype = build_class('NoneType')
    nonetype.parent = astng_builtin
    _CONST_PROXY[type(None)] = nonetype
    if sys.version_info >= (2, 6):
        _CONST_PROXY[bytes] = MANAGER.astng_from_class(bytes)

astng_boot_strapping()

# TODO : find a nicer way to handle this situation;
# However __proxied introduced an
# infinite recursion (see https://bugs.launchpad.net/pylint/+bug/456870)
def _set_proxied(const):
    return _CONST_PROXY[const.value.__class__]
nodes.Const._proxied = property(_set_proxied)

nodes.List._proxied = MANAGER.astng_from_class(list)
nodes.Tuple._proxied = MANAGER.astng_from_class(tuple)
nodes.Dict._proxied = MANAGER.astng_from_class(dict)
# FIXME : is it alright that Generator._proxied is not a astng node?
Generator._proxied = MANAGER.infer_astng_from_something(type(a for a in ()))

<commit_msg>don't depend on logilab.astng import<commit_after># This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free Software
# Foundation; either version 2 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License along with
# this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
# copyright 2003-2010 Sylvain Thenault, all rights reserved.
# contact mailto:thenault@gmail.com
#
# This file is part of logilab-astng.
#
# logilab-astng is free software: you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by the
# Free Software Foundation, either version 2.1 of the License, or (at your
# option) any later version.
#
# logilab-astng is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
# for more details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with logilab-astng. If not, see <http://www.gnu.org/licenses/>.
"""this module contains a set of functions to create astng trees from scratch
(build_* functions) or from living object (object_build_* functions)
"""

__docformat__ = "restructuredtext en"

import sys
from os.path import abspath
from inspect import (getargspec, isdatadescriptor, isfunction, ismethod,
                     ismethoddescriptor, isclass, isbuiltin)

from logilab.astng.nodes import (Module, Class, Const, const_factory, From,
    Function, EmptyNode, Name, Arguments, Dict, List, Set, Tuple)
from logilab.astng.bases import Generator
from logilab.astng.manager import ASTNGManager
MANAGER = ASTNGManager()

def _attach_local_node(parent, node, name):
    node.name = name # needed by add_local_node
    parent.add_local_node(node)

_marker = object()

def attach_dummy_node(node, name, object=_marker):
    """create a dummy node and register it in the locals of the given
    node with the specified name
    """
    enode = EmptyNode()
    enode.object = object
    _attach_local_node(node, enode, name)

EmptyNode.has_underlying_object = lambda self: self.object is not _marker

def attach_const_node(node, name, value):
    """create a Const node and register it in the locals of the given
    node with the specified name
    """
    if not name in node.special_attributes:
        _attach_local_node(node, const_factory(value), name)

def attach_import_node(node, modname, membername):
    """create a From node and register it in the locals of the given
    node with the specified name
    """
    from_node = From(modname, [(membername, None)])
    _attach_local_node(node, from_node, membername)

def build_module(name, doc=None):
    """create and initialize a astng Module node"""
    node = Module(name, doc, pure_python=False)
    node.package = False
    node.parent = None
    return node

def build_class(name, basenames=(), doc=None):
    """create and initialize a astng Class node"""
    node = Class(name, doc)
    for base in basenames:
        basenode = Name()
        basenode.name = base
        node.bases.append(basenode)
        basenode.parent = node
    return node

def build_function(name, args=None, defaults=None, flag=0, doc=None):
    """create and initialize a astng Function node"""
    args, defaults = args or [], defaults or []
    # first argument is now a list of decorators
    func = Function(name, doc)
    func.args = argsnode = Arguments()
    argsnode.args = []
    for arg in args:
        argsnode.args.append(Name())
        argsnode.args[-1].name = arg
        argsnode.args[-1].parent = argsnode
    argsnode.defaults = []
    for default in defaults:
        argsnode.defaults.append(const_factory(default))
        argsnode.defaults[-1].parent = argsnode
    argsnode.kwarg = None
    argsnode.vararg = None
    argsnode.parent = func
    if args:
        register_arguments(func)
    return func

def build_from_import(fromname, names):
    """create and initialize an astng From import statement"""
    return From(fromname, [(name, None) for name in names])

def register_arguments(func, args=None):
    """add given arguments to local

    args is a list that may contains nested lists
    (i.e. def func(a, (b, c, d)): ...)
    """
    if args is None:
        args = func.args.args
        if func.args.vararg:
            func.set_local(func.args.vararg, func.args)
        if func.args.kwarg:
            func.set_local(func.args.kwarg, func.args)
    for arg in args:
        if isinstance(arg, Name):
            func.set_local(arg.name, arg)
        else:
            register_arguments(func, arg.elts)

def object_build_class(node, member, localname):
    """create astng for a living class object"""
    basenames = [base.__name__ for base in member.__bases__]
    return _base_class_object_build(node, member, basenames,
                                    localname=localname)

def object_build_function(node, member, localname):
    """create astng for a living function object"""
    args, varargs, varkw, defaults = getargspec(member)
    if varargs is not None:
        args.append(varargs)
    if varkw is not None:
        args.append(varkw)
    func = build_function(getattr(member, '__name__', None) or localname, args,
                          defaults, member.func_code.co_flags, member.__doc__)
    node.add_local_node(func, localname)

def object_build_datadescriptor(node, member, name):
    """create astng for a living data descriptor object"""
    return _base_class_object_build(node, member, [], name)

def object_build_methoddescriptor(node, member, localname):
    """create astng for a living method descriptor object"""
    # FIXME get arguments ?
    func = build_function(getattr(member, '__name__', None) or localname,
                          doc=member.__doc__)
    # set node's arguments to None to notice that we have no information, not
    # and empty argument list
    func.args.args = None
    node.add_local_node(func, localname)

def _base_class_object_build(node, member, basenames, name=None, localname=None):
    """create astng for a living class object, with a given set of base names
    (e.g. ancestors)
    """
    klass = build_class(name or getattr(member, '__name__', None) or localname,
                        basenames, member.__doc__)
    klass._newstyle = isinstance(member, type)
    node.add_local_node(klass, localname)
    try:
        # limit the instantiation trick since it's too dangerous
        # (such as infinite test execution...)
        # this at least resolves common case such as Exception.args,
        # OSError.errno
        if issubclass(member, Exception):
            instdict = member().__dict__
        else:
            raise TypeError
    except:
        pass
    else:
        for name, obj in instdict.items():
            valnode = EmptyNode()
            valnode.object = obj
            valnode.parent = klass
            valnode.lineno = 1
            klass.instance_attrs[name] = [valnode]
    return klass

def imported_member(node, member, name):
    """consider a class/builtin member where __module__ != current module name

    check if it's sound valid and then add an import node, else use a dummy node
    """
    # /!\ some classes like ExtensionClass doesn't have a
    # __module__ attribute !
    member_module = getattr(member, '__module__', '__builtin__')
    try:
        getattr(sys.modules[member_module], name)
    except (KeyError, AttributeError):
        attach_dummy_node(node, name, member)
    else:
        attach_import_node(node, member_module, name)

class InspectBuilder(object):
    """class for building nodes from living object"""

    # astng from living objects ###############################################
    #
    # this is actually a really minimal representation, including only Module,
    # Function and Class nodes and some others as guessed

    def __init__(self):
        self._done = {}
        self._module = None

    def inspect_build(self, module, modname=None, path=None):
        """build astng from a living module (i.e. using inspect)
        this is used when there is no python source code available (either
        because it's a built-in module or because the .py is not available)
        """
        self._module = module
        if modname is None:
            modname = module.__name__
        node = build_module(modname, module.__doc__)
        node.file = node.path = path and abspath(path) or path
        MANAGER._cache[modname] = node
        node.package = hasattr(module, '__path__')
        self._done = {}
        self.object_build(node, module)
        return node

    def object_build(self, node, obj):
        """recursive method which create a partial ast from real objects
         (only function, class, and method are handled)
        """
        if obj in self._done:
            return self._done[obj]
        self._done[obj] = node
        for name in dir(obj):
            try:
                member = getattr(obj, name)
            except AttributeError:
                # damned ExtensionClass.Base, I know you're there !
                attach_dummy_node(node, name)
                continue
            if ismethod(member):
                member = member.im_func
            if isfunction(member):
                # verify this is not an imported function
                if member.func_code.co_filename != getattr(self._module, '__file__', None):
                    attach_dummy_node(node, name, member)
                    continue
                object_build_function(node, member, name)
            elif isbuiltin(member):
                if self.imported_member(node, member, name):
                    continue
                object_build_methoddescriptor(node, member, name)
            elif isclass(member):
                if self.imported_member(node, member, name):
                    continue
                if member in self._done:
                    class_node = self._done[member]
                    if not class_node in node.locals.get(name, ()):
                        node.add_local_node(class_node, name)
                else:
                    class_node = object_build_class(node, member, name)
                    # recursion
                    self.object_build(class_node, member)
                if name == '__class__' and class_node.parent is None:
                    class_node.parent = self._done[self._module]
            elif ismethoddescriptor(member):
                assert isinstance(member, object)
                object_build_methoddescriptor(node, member, name)
            elif isdatadescriptor(member):
                assert isinstance(member, object)
                object_build_datadescriptor(node, member, name)
            elif isinstance(member, (int, long, float, str, unicode)) or member is None:
                attach_const_node(node, name, member)
            else:
                # create an empty node so that the name is actually defined
                attach_dummy_node(node, name, member)

    def imported_member(self, node, member, name):
        """verify this is not an imported class or handle it"""
        modname = getattr(member, '__module__', None)
        if {'gtk': 'gtk._gtk'}.get(modname, modname) != self._module.__name__:
            imported_member(node, member, name)
            return True
        return False

### astng boot strapping ################################################### ###

_CONST_PROXY = {}
def astng_boot_strapping():
    """astng boot strapping the builtins module"""
    # this boot strapping is necessary since we need the Const nodes to
    # inspect_build builtins, and then we can proxy Const
    builder = InspectBuilder()
    from logilab.common.compat import builtins
    astng_builtin = builder.inspect_build(builtins)
    for cls in (bool, int, long, float, complex, str, unicode):
        _CONST_PROXY[cls] = astng_builtin.getattr(cls.__name__)[0] # XXX
    nonetype = build_class('NoneType')
    nonetype.parent = astng_builtin
    _CONST_PROXY[type(None)] = nonetype
    if sys.version_info >= (2, 6):
        _CONST_PROXY[bytes] = MANAGER.astng_from_class(bytes)

astng_boot_strapping()

# TODO : find a nicer way to handle this situation;
# However __proxied introduced an
# infinite recursion (see https://bugs.launchpad.net/pylint/+bug/456870)
def _set_proxied(const):
    return _CONST_PROXY[const.value.__class__]
Const._proxied = property(_set_proxied)

Dict._proxied = MANAGER.astng_from_class(dict)
List._proxied = MANAGER.astng_from_class(list)
Set._proxied =  MANAGER.astng_from_class(set) # will be needed for py3k
Tuple._proxied = MANAGER.astng_from_class(tuple)
# FIXME : is it alright that Generator._proxied is not a astng node?
Generator._proxied = MANAGER.infer_astng_from_something(type(a for a in ()))

<|endoftext|>


<commit_before>"""
The dataset class used in this package.
Datasets consists of data used for training, represented by a list of
(feature, label) tuples.
May be exported in different formats for application on other libraries.
"""

class Dataset(object):

    def __init__(self, X=[], y=[]):
        """Constructor with scikit-learn style (X, y) data
        """
        self.data = list(zip(X, y))

    def add(self, feature, label):
        """Add a (feature, label) entry into the dataset.
        A None label indicates an unlabeled entry.
        """
        self.data.append((feature, label))

    # TODO: remove, update

    def format_sklearn(self):
        """Returns dataset in (X, y) format for use in scikit-learn.
        Unlabeled entries are ignored.
        """
        l = list(zip(*[entry for entry in self.data if entry[1] is not None]))
        return (list(l[0]), list(l[1]))
<commit_msg>Dataset.update<commit_after>"""
The dataset class used in this package.
Datasets consists of data used for training, represented by a list of
(feature, label) tuples.
May be exported in different formats for application on other libraries.
"""

class Dataset(object):

    def __init__(self, X=[], y=[]):
        """Constructor with scikit-learn style (X, y) data"""
        self.data = list(zip(X, y))

    def add(self, feature, label):
        """Add a (feature, label) entry into the dataset.
        A None label indicates an unlabeled entry.
        Returns entry_id for updating labels.
        """
        self.data.append((feature, label))
        return len(self.data) - 1

    def update(self, entry_id, label):
        """Updates an entry at entry_id with the given new label"""
        self.data[entry_id] = (self.data[entry_id][0], label)

    def format_sklearn(self):
        """Returns dataset in (X, y) format for use in scikit-learn.
        Unlabeled entries are ignored.
        """
        l = list(zip(*[entry for entry in self.data if entry[1] is not None]))
        return (list(l[0]), list(l[1]))
<|endoftext|>


<commit_before>__copyright__ = "Copyright 2017 Birkbeck, University of London"
__author__ = "Martin Paul Eve & Andy Byers"
__license__ = "AGPL v3"
__maintainer__ = "Birkbeck Centre for Technology and Publishing"

from utils import setting_handler, models

def yield_homepage_element_context(request, homepage_elements):
    plugin = models.Plugin.objects.get(name='HTML')
    try:
        html_block_content = setting_handler.get_plugin_setting(plugin, 'html_block_content', request.journal).value
    except IndexError:
        html_block_content = '
This element has no content.
'

    return {'html_content': html_block_content}
<commit_msg>1111: HTML plugin not to break the site if translation is missing<commit_after>__copyright__ = "Copyright 2017 Birkbeck, University of London"
__author__ = "Martin Paul Eve & Andy Byers"
__license__ = "AGPL v3"
__maintainer__ = "Birkbeck Centre for Technology and Publishing"
import logging

from utils import setting_handler, models

def yield_homepage_element_context(request, homepage_elements):
    plugin = models.Plugin.objects.get(name='HTML')
    try:
        html_block_content = setting_handler.get_plugin_setting(plugin, 'html_block_content', request.journal).value
    except (IndexError, models.PluginSettingValue.DoesNotExist) as e:
        logging.exception(e)
        html_block_content = '
This element has no content.
'

    return {'html_content': html_block_content}
<|endoftext|>