<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from django.contrib.auth.decorators import login_required
from django.core.urlresolvers import reverse
from django.forms.models import inlineformset_factory
from django.views.generic import DetailView
from django.views.generic import CreateView, UpdateView
from data_exports.forms import ExportForm, ColumnForm, ColumnFormSet
from data_exports.models import Export, Column

class ExportView(DetailView):
    model = Export

    def get_context_data(self, **kwargs):
        context = super(ExportView, self).get_context_data(**kwargs)
        model = self.object.model.model_class()
        qs = model.objects.all()
        if hasattr(model, "export_queryset"):
            qs = model.export_queryset(self.request)
        context['data'] = qs
        return context

    def render_to_response(self, context, **kwargs):
        resp = super(ExportView, self).render_to_response(context, **kwargs)
        export_format = self.object.export_format
        if export_format:
            filename = '%s.%s' % (self.object.slug, export_format.file_ext)
            resp['Content-Type'] = export_format.mime
            resp['Content-Disposition'] = 'attachment; filename=%s' % filename
        return resp

    def get_template_names(self, **kwargs):
        template_names = super(ExportView, self).get_template_names(**kwargs)
        export_format = self.object.export_format
        if export_format:
            template_names.insert(0, export_format.template)
        return template_names
export_view = login_required(ExportView.as_view())

class ExportAdd(CreateView):
    model = Export
    fields = "__all__"

    def get_success_url(self):
        return reverse('data_exports:export_cols',
                       kwargs={'slug': self.object.slug})
export_add = login_required(ExportAdd.as_view())

class ExportCols(UpdateView):
    column_inline = inlineformset_factory(Export,
                                          Column,
                                          form=ColumnForm,
                                          formset=ColumnFormSet)
    form_class = ExportForm
    model = Export
    template_name = 'data_exports/export_cols.html'

    def get_context_data(self, **kwargs):
        context = super(ExportCols, self).get_context_data(**kwargs)
        context['formset'] = self.column_inline(instance=self.object)
        if hasattr(self, 'formset'):  # formset was POSTed, but is invalid
            context['formset'] = self.formset
        return context

    def post(self, request, *args, **kwargs):
        self.object = self.get_object()
        form_class = self.get_form_class()
        form = self.get_form(form_class)
        if form.is_valid():
            self.object = form.save()
            formset = self.column_inline(request.POST, instance=self.object)
            if formset.is_valid():
                formset.save()
                return self.form_valid(form)
            else:
                self.formset = formset
                return self.form_invalid(form)
        else:
            return self.form_invalid(form)
export_cols = login_required(ExportCols.as_view())
<commit_msg>Dj2.0 Quick fix urlresolver import 2<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from django.contrib.auth.decorators import login_required
from django.urls import reverse
from django.forms.models import inlineformset_factory
from django.views.generic import DetailView
from django.views.generic import CreateView, UpdateView
from data_exports.forms import ExportForm, ColumnForm, ColumnFormSet
from data_exports.models import Export, Column

class ExportView(DetailView):
    model = Export

    def get_context_data(self, **kwargs):
        context = super(ExportView, self).get_context_data(**kwargs)
        model = self.object.model.model_class()
        qs = model.objects.all()
        if hasattr(model, "export_queryset"):
            qs = model.export_queryset(self.request)
        context['data'] = qs
        return context

    def render_to_response(self, context, **kwargs):
        resp = super(ExportView, self).render_to_response(context, **kwargs)
        export_format = self.object.export_format
        if export_format:
            filename = '%s.%s' % (self.object.slug, export_format.file_ext)
            resp['Content-Type'] = export_format.mime
            resp['Content-Disposition'] = 'attachment; filename=%s' % filename
        return resp

    def get_template_names(self, **kwargs):
        template_names = super(ExportView, self).get_template_names(**kwargs)
        export_format = self.object.export_format
        if export_format:
            template_names.insert(0, export_format.template)
        return template_names
export_view = login_required(ExportView.as_view())

class ExportAdd(CreateView):
    model = Export
    fields = "__all__"

    def get_success_url(self):
        return reverse('data_exports:export_cols',
                       kwargs={'slug': self.object.slug})
export_add = login_required(ExportAdd.as_view())

class ExportCols(UpdateView):
    column_inline = inlineformset_factory(Export,
                                          Column,
                                          form=ColumnForm,
                                          formset=ColumnFormSet)
    form_class = ExportForm
    model = Export
    template_name = 'data_exports/export_cols.html'

    def get_context_data(self, **kwargs):
        context = super(ExportCols, self).get_context_data(**kwargs)
        context['formset'] = self.column_inline(instance=self.object)
        if hasattr(self, 'formset'):  # formset was POSTed, but is invalid
            context['formset'] = self.formset
        return context

    def post(self, request, *args, **kwargs):
        self.object = self.get_object()
        form_class = self.get_form_class()
        form = self.get_form(form_class)
        if form.is_valid():
            self.object = form.save()
            formset = self.column_inline(request.POST, instance=self.object)
            if formset.is_valid():
                formset.save()
                return self.form_valid(form)
            else:
                self.formset = formset
                return self.form_invalid(form)
        else:
            return self.form_invalid(form)
export_cols = login_required(ExportCols.as_view())
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""Command line tool to scaffold a new analysis environment."""

import os
import argparse
from databench.analyses_packaged.scaffold.analysis \
    import __file__ as analysis_src_file

def check_folders(name):
    """Only checks and asks questions. Nothing is written to disk."""

    if os.getcwd().endswith('analyses'):
        correct = raw_input('You are in an analyses folder. This will create '
                            'another analyses folder inside this one. Do '
                            'you want to continue? (y/N)')
        if correct != 'y':
            return False

    if not os.path.exists(os.getcwd()+'/analyses'):
        correct = raw_input('This is the first analysis here. Do '
                            'you want to continue? (y/N)')
        if correct != 'y':
            return False

    if os.path.exists(os.getcwd()+'/analyses/'+name):
        correct = raw_input('An analysis with this name exists already. Do '
                            'you want to continue? (y/N)')
        if correct != 'y':
            return False

    return True

def create_analyses(name, py_native):
    """If it does not exist already, it creates the top level analyses folder
    and it's __init__.py file."""

    if not os.path.exists(os.getcwd()+'/analyses'):
        os.system("mkdir analyses")

    if not os.path.exists(os.getcwd()+'/analyses/__init__.py'):
        with open('analyses/__init__.py', 'w') as f:
            f.write('"""Analyses folder created by `scaffold-databench`. '
                    'Modify me. This text is in `analyses/__init__.py`.'
                    '\n\nSources: '
                    '[github.com/username/project]'
                    '(http://github.com/username/project)"""\n\n')
            f.write('__version__ = "0.0.1"\n')
            f.write('__author__ = "Change Me Please <change@meplease.com>"\n')
            f.write('\n')

    if py_native:
        with open('analyses/__init__.py', 'r') as f:
            existing = f.readlines()
        if 'import '+name+'.analysis\n' in existing:
            print 'WARNING: analysis is already imported in __init__.py.'
        else:
            with open('analyses/__init__.py', 'a') as fa:
                fa.write('import '+name+'.analysis\n')

def copy_scaffold_file(src, dest, name):
    if os.path.exists(dest):
        print 'WARNING: file '+dest+' exists alread. Skipping.'
        return

    with open(src, 'r') as f:
        lines = f.readlines()

    if not lines:
        print 'FATAL: source '+src+' is empty.'
        raise

    # replace
    lines = [l.replace('scaffold', name) for l in lines]
    lines = [l.replace('Scaffold', name.title()) for l in lines]

    with open(dest, 'w') as f:
        for l in lines:
            f.write(l)

def create_analysis(name, py_native):
    """Create analysis files."""

    # analysis folder
    folder = os.getcwd()+'/analyses/'+name
    if not os.path.exists(folder):
        os.system('mkdir '+folder)
    else:
        print 'WARNING: analysis folder '+folder+' already exists.'

    # __init__.py
    if py_native:
        os.system('touch '+folder+'/__init__.py')

    # copy all other files
    for f in ['analysis.py', 'index.html', 'README.md', 'thumbnail.png']:
        src = analysis_src_file.replace('analysis.pyc', f)
        copy_scaffold_file(src, folder+'/'+f, name)

def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('analysis_name',
                        help='Name of the analysis to be created.')
    args = parser.parse_args()

    if not check_folders(args.analysis_name):
        return

    py_native = args.analysis_name.split('_')[-1] not in [
        'py', 'pyspark', 'spark', 'go', 'lua', 'julia', 'r'
    ]

    create_analyses(args.analysis_name, py_native)
    create_analysis(args.analysis_name, py_native)
    print("Done.")

if __name__ == "__main__":
    main()
<commit_msg>new -y option<commit_after>#!/usr/bin/env python

"""Command line tool to scaffold a new analysis environment."""

import os
import argparse
from databench.analyses_packaged.scaffold.analysis \
    import __file__ as analysis_src_file

def check_folders(name):
    """Only checks and asks questions. Nothing is written to disk."""

    if os.getcwd().endswith('analyses'):
        correct = raw_input('You are in an analyses folder. This will create '
                            'another analyses folder inside this one. Do '
                            'you want to continue? (y/N)')
        if correct != 'y':
            return False

    if not os.path.exists(os.getcwd()+'/analyses'):
        correct = raw_input('This is the first analysis here. Do '
                            'you want to continue? (y/N)')
        if correct != 'y':
            return False

    if os.path.exists(os.getcwd()+'/analyses/'+name):
        correct = raw_input('An analysis with this name exists already. Do '
                            'you want to continue? (y/N)')
        if correct != 'y':
            return False

    return True

def create_analyses(name, py_native):
    """If it does not exist already, it creates the top level analyses folder
    and it's __init__.py file."""

    if not os.path.exists(os.getcwd()+'/analyses'):
        os.system("mkdir analyses")

    if not os.path.exists(os.getcwd()+'/analyses/__init__.py'):
        with open('analyses/__init__.py', 'w') as f:
            f.write('"""Analyses folder created by `scaffold-databench`. '
                    'Modify me. This text is in `analyses/__init__.py`.'
                    '\n\nSources: '
                    '[github.com/username/project]'
                    '(http://github.com/username/project)"""\n\n')
            f.write('__version__ = "0.0.1"\n')
            f.write('__author__ = "Change Me Please <change@meplease.com>"\n')
            f.write('\n')

    if py_native:
        with open('analyses/__init__.py', 'r') as f:
            existing = f.readlines()
        if 'import '+name+'.analysis\n' in existing:
            print 'WARNING: analysis is already imported in __init__.py.'
        else:
            with open('analyses/__init__.py', 'a') as fa:
                fa.write('import '+name+'.analysis\n')

def copy_scaffold_file(src, dest, name):
    if os.path.exists(dest):
        print 'WARNING: file '+dest+' exists alread. Skipping.'
        return

    with open(src, 'r') as f:
        lines = f.readlines()

    if not lines:
        print 'FATAL: source '+src+' is empty.'
        raise

    # replace
    lines = [l.replace('scaffold', name) for l in lines]
    lines = [l.replace('Scaffold', name.title()) for l in lines]

    with open(dest, 'w') as f:
        for l in lines:
            f.write(l)

def create_analysis(name, py_native):
    """Create analysis files."""

    # analysis folder
    folder = os.getcwd()+'/analyses/'+name
    if not os.path.exists(folder):
        os.system('mkdir '+folder)
    else:
        print 'WARNING: analysis folder '+folder+' already exists.'

    # __init__.py
    if py_native:
        os.system('touch '+folder+'/__init__.py')

    # copy all other files
    for f in ['analysis.py', 'index.html', 'README.md', 'thumbnail.png']:
        src = analysis_src_file.replace('analysis.pyc', f)
        copy_scaffold_file(src, folder+'/'+f, name)

def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('analysis_name',
                        help='Name of the analysis to be created.')
    parser.add_argument('-y', dest='yes', default=False, action='store_true',
                        help='Answer all questions with yes. Be careful.')
    args = parser.parse_args()

    if not (args.yes or check_folders(args.analysis_name)):
        return

    py_native = args.analysis_name.split('_')[-1] not in [
        'py', 'pyspark', 'spark', 'go', 'lua', 'julia', 'r'
    ]

    create_analyses(args.analysis_name, py_native)
    create_analysis(args.analysis_name, py_native)
    print("Done.")

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>"""Implements type coercion rules for data shapes.

Note that transitive coercions could be supported, but we decide not to since
it may involve calling a whole bunch of functions with a whole bunch of types
to figure out whether this is possible in the face of polymorphic overloads.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from collections import defaultdict
from itertools import chain, product

import datashape
from .error import CoercionError
from .coretypes import CType, TypeVar, Mono
from .typesets import boolean, complexes, floating, integral, signed, unsigned
from . import verify, normalize, Implements, Fixed, Var, Ellipsis, DataShape

class CoercionTable(object):
    """Table to hold coercion rules"""

    def __init__(self):
        self.table = {}
        self.srcs = defaultdict(set)
        self.dsts = defaultdict(set)

    def _reflexivity(self, a):
        if (a, a) not in self.table:
            self.table[a, a] = 0

    def add_coercion(self, src, dst, cost, transitive=True):
        """
        Add a coercion rule
        """
        assert cost >= 0, 'Raw coercion costs must be nonnegative'
        if (src, dst) not in self.table:
            self.srcs[dst].add(src)
            self.dsts[src].add(dst)
            self._reflexivity(src)
            self._reflexivity(dst)
            if src != dst:
                self.table[src, dst] = cost
                if transitive:
                    transitivity(src, dst, self)
        else:
            # Add the cost for the shortest path for the coercion
            self.table[src, dst] = min(self.table[src, dst], cost)

    def coercion_cost(self, src, dst):
        """
        Determine a coercion cost for coercing type `a` to type `b`
        """
        return self.table[src, dst]

_table = CoercionTable()
add_coercion = _table.add_coercion
coercion_cost_table = _table.coercion_cost

#------------------------------------------------------------------------
# Coercion invariants
#------------------------------------------------------------------------

def transitivity(a, b, table=_table):
    """Enforce coercion rule transitivity"""
    # (src, a) in R and (a, b) in R => (src, b) in R
    for src in table.srcs[a]:
        table.add_coercion(src, b, table.coercion_cost(src, a) +
                                   table.coercion_cost(a, b))

    # (a, b) in R and (b, dst) in R => (a, dst) in R
    for dst in table.dsts[b]:
        table.add_coercion(a, dst, table.coercion_cost(a, b) +
                                   table.coercion_cost(b, dst))

#------------------------------------------------------------------------
# Coercion function
#------------------------------------------------------------------------

def coercion_cost(a, b, seen=None):
    """
    Determine a coercion cost from type `a` to type `b`.

    Type `a` and `b'` must be unifiable and normalized.
    """
    # Determine the approximate cost and subtract the term size of the
    # right hand side: the more complicated the RHS, the more specific
    # the match should be
    return _coercion_cost(a, b, seen) - (termsize(b) / 100.0)

def _coercion_cost(a, b, seen=None):
    # TODO: Cost functions for conversion between type constructors in the
    # lattice (implement a "type join")

    if seen is None:
        seen = set()

    if a == b or isinstance(a, TypeVar):
        return 0
    elif isinstance(a, CType) and isinstance(b, CType):
        try:
            return coercion_cost_table(a, b)
        except KeyError:
            raise CoercionError(a, b)
    elif isinstance(b, TypeVar):
        visited = b not in seen
        seen.add(b)
        return 0.1 * visited
    elif isinstance(b, Implements):
        if a in b.typeset:
            return 0.1 - (0.1 / len(b.typeset.types))
        else:
            raise CoercionError(a, b)
    elif isinstance(b, Fixed):
        if isinstance(a, Var):
            return 0.1 # broadcasting penalty

        assert isinstance(a, Fixed)
        if a.val != b.val:
            assert a.val == 1 or b.val == 1
            return 0.1 # broadcasting penalty
        return 0
    elif isinstance(b, Var):
        assert type(a) in [Var, Fixed]
        if isinstance(a, Fixed):
            return 0.1 # broadcasting penalty
        return 0
    elif isinstance(a, DataShape) and isinstance(b, DataShape):
        return coerce_datashape(a, b, seen)
    else:
        verify(a, b)
        return sum([_coercion_cost(x, y, seen) for x, y in zip(a.parameters,
                                                               b.parameters)])

def termsize(term):
    """Determine the size of a type term"""
    if isinstance(term, Mono):
        return sum(termsize(p) for p in term.parameters) + 1
    return 0

def coerce_datashape(a, b, seen):
    # Penalize broadcasting
    broadcast_penalty = abs(len(a.parameters) - len(b.parameters))

    # Penalize ellipsis if one side has it but not the other
    ellipses_a = sum(isinstance(p, Ellipsis) for p in a.parameters)
    ellipses_b = sum(isinstance(p, Ellipsis) for p in b.parameters)
    ellipsis_penalty = ellipses_a ^ ellipses_b

    penalty = broadcast_penalty + ellipsis_penalty

    # Process rest of parameters
    [(a, b)], _ = normalize([(a, b)], [True])
    verify(a, b)
    for x, y in zip(a.parameters, b.parameters):
        penalty += coercion_cost(x, y, seen)

    return penalty

#------------------------------------------------------------------------
# Default coercion rules
#------------------------------------------------------------------------

def add_numeric_rule(types):
    types = list(types)
    for src, dst in zip(types[:-1], types[1:]):
        add_coercion(src, dst, 1)

promotable_unsigned = [datashape.uint8, datashape.uint16, datashape.uint32]
promoted_signed     = [datashape.int16, datashape.int32, datashape.int64]

add_numeric_rule(signed)
add_numeric_rule(unsigned)
add_numeric_rule(floating)
add_numeric_rule(complexes)

add_numeric_rule([datashape.bool_, datashape.int8])
add_numeric_rule([datashape.uint8, datashape.int16])
add_numeric_rule([datashape.uint16, datashape.int32])
add_numeric_rule([datashape.uint32, datashape.int64])

add_numeric_rule([datashape.int16, datashape.float32])
add_numeric_rule([datashape.int32, datashape.float64])
add_numeric_rule([datashape.float32, datashape.complex64])
add_numeric_rule([datashape.float64, datashape.complex128])

# Potentially lossy conversions

# unsigned -> signed
add_numeric_rule([datashape.uint8, datashape.int8])
add_numeric_rule([datashape.uint16, datashape.int16])
add_numeric_rule([datashape.uint32, datashape.int32])
add_numeric_rule([datashape.uint64, datashape.int64])

# signed -> unsigned
add_numeric_rule([datashape.int8, datashape.uint8])
add_numeric_rule([datashape.int16, datashape.uint16])
add_numeric_rule([datashape.int32, datashape.uint32])
add_numeric_rule([datashape.int64, datashape.uint64])

# int -> float
add_numeric_rule([datashape.int32, datashape.float32])
add_numeric_rule([datashape.int64, datashape.float64])

# float -> complex
add_numeric_rule([datashape.float64, datashape.complex64])<commit_msg>Set the cost function of potentially lossy conversions to 1.5 instead of 1.0<commit_after>"""Implements type coercion rules for data shapes.

Note that transitive coercions could be supported, but we decide not to since
it may involve calling a whole bunch of functions with a whole bunch of types
to figure out whether this is possible in the face of polymorphic overloads.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from collections import defaultdict
from itertools import chain, product

import datashape
from .error import CoercionError
from .coretypes import CType, TypeVar, Mono
from .typesets import boolean, complexes, floating, integral, signed, unsigned
from . import verify, normalize, Implements, Fixed, Var, Ellipsis, DataShape

class CoercionTable(object):
    """Table to hold coercion rules"""

    def __init__(self):
        self.table = {}
        self.srcs = defaultdict(set)
        self.dsts = defaultdict(set)

    def _reflexivity(self, a):
        if (a, a) not in self.table:
            self.table[a, a] = 0

    def add_coercion(self, src, dst, cost, transitive=True):
        """
        Add a coercion rule
        """
        assert cost >= 0, 'Raw coercion costs must be nonnegative'
        if (src, dst) not in self.table:
            self.srcs[dst].add(src)
            self.dsts[src].add(dst)
            self._reflexivity(src)
            self._reflexivity(dst)
            if src != dst:
                self.table[src, dst] = cost
                if transitive:
                    transitivity(src, dst, self)
        else:
            # Add the cost for the shortest path for the coercion
            self.table[src, dst] = min(self.table[src, dst], cost)

    def coercion_cost(self, src, dst):
        """
        Determine a coercion cost for coercing type `a` to type `b`
        """
        return self.table[src, dst]

_table = CoercionTable()
add_coercion = _table.add_coercion
coercion_cost_table = _table.coercion_cost

#------------------------------------------------------------------------
# Coercion invariants
#------------------------------------------------------------------------

def transitivity(a, b, table=_table):
    """Enforce coercion rule transitivity"""
    # (src, a) in R and (a, b) in R => (src, b) in R
    for src in table.srcs[a]:
        table.add_coercion(src, b, table.coercion_cost(src, a) +
                                   table.coercion_cost(a, b))

    # (a, b) in R and (b, dst) in R => (a, dst) in R
    for dst in table.dsts[b]:
        table.add_coercion(a, dst, table.coercion_cost(a, b) +
                                   table.coercion_cost(b, dst))

#------------------------------------------------------------------------
# Coercion function
#------------------------------------------------------------------------

def coercion_cost(a, b, seen=None):
    """
    Determine a coercion cost from type `a` to type `b`.

    Type `a` and `b'` must be unifiable and normalized.
    """
    # Determine the approximate cost and subtract the term size of the
    # right hand side: the more complicated the RHS, the more specific
    # the match should be
    return _coercion_cost(a, b, seen) - (termsize(b) / 100.0)

def _coercion_cost(a, b, seen=None):
    # TODO: Cost functions for conversion between type constructors in the
    # lattice (implement a "type join")

    if seen is None:
        seen = set()

    if a == b or isinstance(a, TypeVar):
        return 0
    elif isinstance(a, CType) and isinstance(b, CType):
        try:
            return coercion_cost_table(a, b)
        except KeyError:
            raise CoercionError(a, b)
    elif isinstance(b, TypeVar):
        visited = b not in seen
        seen.add(b)
        return 0.1 * visited
    elif isinstance(b, Implements):
        if a in b.typeset:
            return 0.1 - (0.1 / len(b.typeset.types))
        else:
            raise CoercionError(a, b)
    elif isinstance(b, Fixed):
        if isinstance(a, Var):
            return 0.1 # broadcasting penalty

        assert isinstance(a, Fixed)
        if a.val != b.val:
            assert a.val == 1 or b.val == 1
            return 0.1 # broadcasting penalty
        return 0
    elif isinstance(b, Var):
        assert type(a) in [Var, Fixed]
        if isinstance(a, Fixed):
            return 0.1 # broadcasting penalty
        return 0
    elif isinstance(a, DataShape) and isinstance(b, DataShape):
        return coerce_datashape(a, b, seen)
    else:
        verify(a, b)
        return sum([_coercion_cost(x, y, seen) for x, y in zip(a.parameters,
                                                               b.parameters)])

def termsize(term):
    """Determine the size of a type term"""
    if isinstance(term, Mono):
        return sum(termsize(p) for p in term.parameters) + 1
    return 0

def coerce_datashape(a, b, seen):
    # Penalize broadcasting
    broadcast_penalty = abs(len(a.parameters) - len(b.parameters))

    # Penalize ellipsis if one side has it but not the other
    ellipses_a = sum(isinstance(p, Ellipsis) for p in a.parameters)
    ellipses_b = sum(isinstance(p, Ellipsis) for p in b.parameters)
    ellipsis_penalty = ellipses_a ^ ellipses_b

    penalty = broadcast_penalty + ellipsis_penalty

    # Process rest of parameters
    [(a, b)], _ = normalize([(a, b)], [True])
    verify(a, b)
    for x, y in zip(a.parameters, b.parameters):
        penalty += coercion_cost(x, y, seen)

    return penalty

#------------------------------------------------------------------------
# Default coercion rules
#------------------------------------------------------------------------

def add_numeric_rule(types, distance=1):
    types = list(types)
    for src, dst in zip(types[:-1], types[1:]):
        add_coercion(src, dst, distance)

promotable_unsigned = [datashape.uint8, datashape.uint16, datashape.uint32]
promoted_signed     = [datashape.int16, datashape.int32, datashape.int64]

add_numeric_rule(signed)
add_numeric_rule(unsigned)
add_numeric_rule(floating)
add_numeric_rule(complexes)

add_numeric_rule([datashape.bool_, datashape.int8])
add_numeric_rule([datashape.uint8, datashape.int16])
add_numeric_rule([datashape.uint16, datashape.int32])
add_numeric_rule([datashape.uint32, datashape.int64])

add_numeric_rule([datashape.int16, datashape.float32])
add_numeric_rule([datashape.int32, datashape.float64])
add_numeric_rule([datashape.float32, datashape.complex64])
add_numeric_rule([datashape.float64, datashape.complex128])

# Potentially lossy conversions

# unsigned -> signed
add_numeric_rule([datashape.uint8, datashape.int8], 1.5)
add_numeric_rule([datashape.uint16, datashape.int16], 1.5)
add_numeric_rule([datashape.uint32, datashape.int32], 1.5)
add_numeric_rule([datashape.uint64, datashape.int64], 1.5)

# signed -> unsigned
add_numeric_rule([datashape.int8, datashape.uint8], 1.5)
add_numeric_rule([datashape.int16, datashape.uint16], 1.5)
add_numeric_rule([datashape.int32, datashape.uint32], 1.5)
add_numeric_rule([datashape.int64, datashape.uint64], 1.5)

# int -> float
add_numeric_rule([datashape.int32, datashape.float32], 1.5)
add_numeric_rule([datashape.int64, datashape.float64], 1.5)

# float -> complex
add_numeric_rule([datashape.float64, datashape.complex64], 1.5)
<|endoftext|>


<commit_before>"""
Copyright (c) 2003  Gustavo Niemeyer <niemeyer@conectiva.com>

This module offers extensions to the standard python 2.3+
datetime module.
"""
__author__ = "Gustavo Niemeyer <niemeyer@conectiva.com>"
__license__ = "PSF License"

import datetime
import thread
import signal
import time

class sched:

    def __init__(self, rrule,
                 tolerance=None, last=None,
                 execute=None, args=None, kwargs=None):
        self._rrule = rrule
        if tolerance:
            self._tolerance = datetime.timedelta(seconds=tolerance)
        else:
            self._tolerance = None
        self._last = last
        self._execute = execute
        self._args = args or ()
        self._kwargs = kwargs or {}

    def last(self):
        return self._last

    def next(self, now=None):
        if not now:
            now = datetime.datetime.now()
        return self._rrule.after(now)

    def check(self, now=None, readonly=False):
        if not now:
            now = datetime.datetime.now()
        item = self._rrule.before(now, inc=True)
        if (item is None or item == self._last or
            (self._tolerance and item+self._tolerance < now)):
            return None
        if not readonly:
            self._last = item
            if self._execute:
                self._execute(*self._args, **self._kwargs)
        return item

class schedset:
    def __init__(self):
        self._scheds = []

    def add(self, sched):
        self._scheds.append(sched)

    def next(self, now=None):
        if not now:
            now = datetime.datetime.now()
        res = None
        for sched in self._scheds:
            next = sched.next(now)
            if next and (not res or next < res):
                res = next
        return res

    def check(self, now=None, readonly=False):
        if not now:
            now = datetime.datetime.now()
        res = False
        for sched in self._scheds:
            if sched.check(now, readonly):
                res = True
        return res

class schedthread:

    def __init__(self, sched, lock=None):
        self._sched = sched
        self._lock = lock
        self._running = False

    def running(self):
        return self._running

    def run(self):
        self._running = True
        thread.start_new_thread(self._loop, ())

    def stop(self):
        self._running = False

    def _loop(self):
        while self._running:
            if self._lock:
                self._lock.acquire()
            now = datetime.datetime.now()
            self._sched.check(now)
            if self._lock:
                self._lock.release()
            seconds = _seconds_left(self._sched.next(now))
            if seconds is None:
                self._running = False
                break
            if self._running:
                time.sleep(seconds)

class schedalarm:

    def __init__(self, sched, lock=None):
        self._sched = sched
        self._lock = lock
        self._running = False

    def running(self):
        return self._running

    def run(self):
        self._running = True
        signal.signal(signal.SIGALRM, self._handler)
        self._handler(None, None)

    def stop(self):
        self._running = False

    def _handler(self, sig, frame):
        while self._running:
            if self._lock:
                self._lock.acquire()
            now = datetime.datetime.now()
            self._sched.check(now)
            if self._lock:
                self._lock.release()
            if self._running:
                seconds = _seconds_left(self._sched.next(now))
                if seconds:
                    signal.alarm(seconds)
                    break
                elif seconds is None:
                    self._running = False
                    break

def _seconds_left(next):
    if not next:
        return None
    now = datetime.datetime.now()
    delta = next-now
    seconds = delta.days*86400+delta.seconds
    if seconds < 0:
        seconds = 0
    return seconds

<commit_msg>Removing scheduler.py.<commit_after><|endoftext|>


<commit_before>import copy
import logging
import subprocess

import dcos_test_utils.aws
import dcos_test_utils.onprem
import pkg_resources
from dcos_test_utils.helpers import Url

import dcos_launch.aws
import dcos_launch.util

log = logging.getLogger(__name__)

STATE_FILE = 'LAST_COMPLETED_STAGE'

class OnpremLauncher(dcos_launch.util.AbstractLauncher):
    def __init__(self, config):
        # can only be set during the wait command
        self.bootstrap_host = None
        self.config = config

    def create(self):
        return self.get_bare_cluster_launcher().create()

    def post_state(self, state):
        self.get_ssh_client().command(self.bootstrap_host, ['printf', state, '>', STATE_FILE])

    def get_last_state(self):
        return self.get_ssh_client().command(self.bootstrap_host, ['cat', STATE_FILE]).decode().strip()

    def get_bare_cluster_launcher(self):
        if self.config['platform'] == 'aws':
            return dcos_launch.aws.BareClusterLauncher(self.config)
        else:
            raise dcos_launch.util.LauncherError('Platform currently not supported for onprem: {}'.format(
                self.config['platform']))

    def get_onprem_cluster(self):
        return dcos_test_utils.onprem.OnpremCluster.from_hosts(
            ssh_client=self.get_ssh_client(),
            hosts=self.get_bare_cluster_launcher().get_hosts(),
            num_masters=int(self.config['num_masters']),
            num_private_agents=int(self.config['num_private_agents']),
            num_public_agents=int(self.config['num_public_agents']))

    def get_completed_onprem_config(self, cluster: dcos_test_utils.onprem.OnpremCluster) -> dict:
        onprem_config = self.config['dcos_config']
        # First, try and retrieve the agent list from the cluster
        onprem_config['agent_list'] = [h.private_ip for h in cluster.private_agents]
        onprem_config['public_agent_list'] = [h.private_ip for h in cluster.public_agents]
        onprem_config['master_list'] = [h.private_ip for h in cluster.masters]
        # if the user wanted to use exhibitor as the backend, then start it
        if onprem_config.get('exhibitor_storage_backend') == 'zookeeper':
            onprem_config['exhibitor_zk_hosts'] = cluster.start_bootstrap_zk()
        # if key helper is true then ssh key must be injected, or the key
        # must have been provided as a file and still needs to be injected
        if self.config['key_helper'] or 'ssh_key' not in onprem_config:
            onprem_config['ssh_key'] = self.config['ssh_private_key']
        # check if ssh user was not provided
        if 'ssh_user' not in onprem_config:
            onprem_config['ssh_user'] = self.config['ssh_user']
        # check if the user provided any filenames and convert them into content
        for key_name in ('ip_detect_filename', 'ip_detect_public_filename'):
            if key_name not in onprem_config:
                continue
            new_key_name = key_name.replace('_filename', '_contents')
            if new_key_name in onprem_config:
                raise dcos_launch.util.LauncherError(
                    'InvalidDcosConfig', 'Cannot set *_filename and *_contents simultaneously!')
            onprem_config[new_key_name] = dcos_launch.util.read_file(onprem_config[key_name])
            del onprem_config[key_name]
        # set the simple default IP detect script if not provided
        # currently, only AWS is supported, but when support changes, this will have to update
        if 'ip_detect_contents' not in onprem_config:
            onprem_config['ip_detect_contents'] = pkg_resources.resource_string(
                'dcos_launch', 'ip-detect/aws.sh').decode('utf-8')
        if 'ip_detect_contents' not in onprem_config:
            onprem_config['ip_detect_public_contents'] = pkg_resources.resource_string(
                'dcos_launch', 'ip-detect/aws_public.sh').decode('utf-8')
        # For no good reason the installer uses 'ip_detect_script' instead of 'ip_detect_contents'
        onprem_config['ip_detect_script'] = self.config['dcos_config']['ip_detect_contents']
        del onprem_config['ip_detect_contents']
        log.debug('Generated cluster configuration: {}'.format(onprem_config))
        return onprem_config

    def wait(self):
        log.info('Waiting for bare cluster provisioning status..')
        self.get_bare_cluster_launcher().wait()
        cluster = self.get_onprem_cluster()
        self.bootstrap_host = cluster.bootstrap_host.public_ip
        log.info('Waiting for SSH connectivity to bootstrap host...')
        cluster.ssh_client.wait_for_ssh_connection(cluster.bootstrap_host.public_ip, self.config['ssh_port'])
        try:
            self.get_ssh_client().command(self.bootstrap_host, ['test', '-f', STATE_FILE])
            last_complete = self.get_last_state()
            log.info('Detected previous launch state, continuing '
                     'from last complete stage ({})'.format(last_complete))
        except subprocess.CalledProcessError:
            log.info('No installation state file detected; beginning fresh install...')
            last_complete = None

        if last_complete is None:
            cluster.setup_installer_server(self.config['installer_url'], False)
            last_complete = 'SETUP'
            self.post_state(last_complete)

        installer = dcos_test_utils.onprem.DcosInstallerApiSession(Url(
            'http', self.bootstrap_host, '', '', '', self.config['installer_port']))
        if last_complete == 'SETUP':
            last_complete = 'GENCONF'
            installer.genconf(self.get_completed_onprem_config(cluster))
            self.post_state(last_complete)
        if last_complete == 'GENCONF':
            installer.preflight()
            last_complete = 'PREFLIGHT'
            self.post_state(last_complete)
        if last_complete == 'PREFLIGHT':
            installer.deploy()
            last_complete = 'DEPLOY'
            self.post_state(last_complete)
        if last_complete == 'DEPLOY':
            installer.postflight()
            last_complete = 'POSTFLIGHT'
            self.post_state(last_complete)
        if last_complete != 'POSTFLIGHT':
            raise dcos_launch.util.LauncherError('InconsistentState', 'State on bootstrap host is: ' + last_complete)

    def describe(self):
        """ returns host information stored in the config as
        well as the basic provider info
        """
        cluster = self.get_onprem_cluster()
        extra_info = {
            'bootstrap_host': dcos_launch.util.convert_host_list([cluster.bootstrap_host])[0],
            'masters': dcos_launch.util.convert_host_list(cluster.get_master_ips()),
            'private_agents': dcos_launch.util.convert_host_list(cluster.get_private_agent_ips()),
            'public_agents': dcos_launch.util.convert_host_list(cluster.get_public_agent_ips())}
        desc = copy.copy(self.config)
        desc.update(extra_info)
        # blackout unwanted fields
        del desc['template_body']
        del desc['template_parameters']
        return desc

    def delete(self):
        """ just deletes the hardware
        """
        self.get_bare_cluster_launcher().delete()
<commit_msg>Add more SSH waiting to increase stability<commit_after>import copy
import logging
import subprocess

import dcos_test_utils.aws
import dcos_test_utils.onprem
import pkg_resources
from dcos_test_utils.helpers import Url

import dcos_launch.aws
import dcos_launch.util

log = logging.getLogger(__name__)

STATE_FILE = 'LAST_COMPLETED_STAGE'

class OnpremLauncher(dcos_launch.util.AbstractLauncher):
    def __init__(self, config):
        # can only be set during the wait command
        self.bootstrap_host = None
        self.config = config

    def create(self):
        return self.get_bare_cluster_launcher().create()

    def post_state(self, state):
        self.get_ssh_client().command(self.bootstrap_host, ['printf', state, '>', STATE_FILE])

    def get_last_state(self):
        return self.get_ssh_client().command(self.bootstrap_host, ['cat', STATE_FILE]).decode().strip()

    def get_bare_cluster_launcher(self):
        if self.config['platform'] == 'aws':
            return dcos_launch.aws.BareClusterLauncher(self.config)
        else:
            raise dcos_launch.util.LauncherError('Platform currently not supported for onprem: {}'.format(
                self.config['platform']))

    def get_onprem_cluster(self):
        return dcos_test_utils.onprem.OnpremCluster.from_hosts(
            ssh_client=self.get_ssh_client(),
            hosts=self.get_bare_cluster_launcher().get_hosts(),
            num_masters=int(self.config['num_masters']),
            num_private_agents=int(self.config['num_private_agents']),
            num_public_agents=int(self.config['num_public_agents']))

    def get_completed_onprem_config(self, cluster: dcos_test_utils.onprem.OnpremCluster) -> dict:
        onprem_config = self.config['dcos_config']
        # First, try and retrieve the agent list from the cluster
        onprem_config['agent_list'] = [h.private_ip for h in cluster.private_agents]
        onprem_config['public_agent_list'] = [h.private_ip for h in cluster.public_agents]
        onprem_config['master_list'] = [h.private_ip for h in cluster.masters]
        # if the user wanted to use exhibitor as the backend, then start it
        if onprem_config.get('exhibitor_storage_backend') == 'zookeeper':
            onprem_config['exhibitor_zk_hosts'] = cluster.start_bootstrap_zk()
        # if key helper is true then ssh key must be injected, or the key
        # must have been provided as a file and still needs to be injected
        if self.config['key_helper'] or 'ssh_key' not in onprem_config:
            onprem_config['ssh_key'] = self.config['ssh_private_key']
        # check if ssh user was not provided
        if 'ssh_user' not in onprem_config:
            onprem_config['ssh_user'] = self.config['ssh_user']
        # check if the user provided any filenames and convert them into content
        for key_name in ('ip_detect_filename', 'ip_detect_public_filename'):
            if key_name not in onprem_config:
                continue
            new_key_name = key_name.replace('_filename', '_contents')
            if new_key_name in onprem_config:
                raise dcos_launch.util.LauncherError(
                    'InvalidDcosConfig', 'Cannot set *_filename and *_contents simultaneously!')
            onprem_config[new_key_name] = dcos_launch.util.read_file(onprem_config[key_name])
            del onprem_config[key_name]
        # set the simple default IP detect script if not provided
        # currently, only AWS is supported, but when support changes, this will have to update
        if 'ip_detect_contents' not in onprem_config:
            onprem_config['ip_detect_contents'] = pkg_resources.resource_string(
                'dcos_launch', 'ip-detect/aws.sh').decode('utf-8')
        if 'ip_detect_contents' not in onprem_config:
            onprem_config['ip_detect_public_contents'] = pkg_resources.resource_string(
                'dcos_launch', 'ip-detect/aws_public.sh').decode('utf-8')
        # For no good reason the installer uses 'ip_detect_script' instead of 'ip_detect_contents'
        onprem_config['ip_detect_script'] = self.config['dcos_config']['ip_detect_contents']
        del onprem_config['ip_detect_contents']
        log.debug('Generated cluster configuration: {}'.format(onprem_config))
        return onprem_config

    def wait(self):
        # FIXME: this starts too soon and some machines get weird SSH failures
        log.info('Waiting for bare cluster provisioning status..')
        self.get_bare_cluster_launcher().wait()
        cluster = self.get_onprem_cluster()
        self.bootstrap_host = cluster.bootstrap_host.public_ip
        log.info('Waiting for SSH connectivity to bootstrap host...')
        for host in cluster.masters + cluster.private_agents + cluster.public_agents:
            cluster.ssh_client.wait_for_ssh_connection(host.public_ip, self.config['ssh_port'])
        try:
            self.get_ssh_client().command(self.bootstrap_host, ['test', '-f', STATE_FILE])
            last_complete = self.get_last_state()
            log.info('Detected previous launch state, continuing '
                     'from last complete stage ({})'.format(last_complete))
        except subprocess.CalledProcessError:
            log.info('No installation state file detected; beginning fresh install...')
            last_complete = None

        if last_complete is None:
            cluster.setup_installer_server(self.config['installer_url'], False)
            last_complete = 'SETUP'
            self.post_state(last_complete)

        installer = dcos_test_utils.onprem.DcosInstallerApiSession(Url(
            'http', self.bootstrap_host, '', '', '', self.config['installer_port']))
        if last_complete == 'SETUP':
            last_complete = 'GENCONF'
            installer.genconf(self.get_completed_onprem_config(cluster))
            self.post_state(last_complete)
        if last_complete == 'GENCONF':
            installer.preflight()
            last_complete = 'PREFLIGHT'
            self.post_state(last_complete)
        if last_complete == 'PREFLIGHT':
            installer.deploy()
            last_complete = 'DEPLOY'
            self.post_state(last_complete)
        if last_complete == 'DEPLOY':
            installer.postflight()
            last_complete = 'POSTFLIGHT'
            self.post_state(last_complete)
        if last_complete != 'POSTFLIGHT':
            raise dcos_launch.util.LauncherError('InconsistentState', 'State on bootstrap host is: ' + last_complete)

    def describe(self):
        """ returns host information stored in the config as
        well as the basic provider info
        """
        cluster = self.get_onprem_cluster()
        extra_info = {
            'bootstrap_host': dcos_launch.util.convert_host_list([cluster.bootstrap_host])[0],
            'masters': dcos_launch.util.convert_host_list(cluster.get_master_ips()),
            'private_agents': dcos_launch.util.convert_host_list(cluster.get_private_agent_ips()),
            'public_agents': dcos_launch.util.convert_host_list(cluster.get_public_agent_ips())}
        desc = copy.copy(self.config)
        desc.update(extra_info)
        # blackout unwanted fields
        del desc['template_body']
        del desc['template_parameters']
        return desc

    def delete(self):
        """ just deletes the hardware
        """
        self.get_bare_cluster_launcher().delete()
<|endoftext|>


<commit_before>"""
Parse Astrid xml backup file into simple data structures.
"""

__docformat__ = 'reStructuredText'

from datetime import datetime
import re
from xml.dom import minidom

from dear_astrid.constants import UTC

__all__ = [
  'AstridValueError',
  'parse_xml',
  'parse_task',
  'parse_timestamp',
  'parse_recurrence',
]

# should this subclass ValueError?
class AstridValueError(Exception):
  """Value does not match expected format and cannot be parsed."""

  def __init__(self, key, val):
    Exception.__init__(self,
      'Unknown format for Astrid {0}: {1}'.format(key, val)
    )

# TODO: allow passing alternate parse_* callables?
def parse_xml(xml=None):
  """Parse xml string into list of task dictionaries"""

  xml = minidom.parseString(xml)
  format_ = xml.getElementsByTagName('astrid')[0].getAttribute('format')

  if format_ != '2':
    raise AstridValueError('xml', format_)

  return [parse_task(t) for t in xml.getElementsByTagName('task')]

def parse_task(element):
  """Parse task element into a simple dictionaries."""

  # use method object as shortcut
  eattr = element.getAttribute

  # other fields: timerStart flags is_public hideUntil
  # created modified calendarUri details
  # flags2: remind when due?
  task = {
    'title':         eattr('title'),
    # astrid's "no priority" is 3
    'priority':      int(eattr('importance') or 3),
    'due_date':      parse_timestamp(eattr('dueDate')),
    'recurrence':    parse_recurrence(eattr('recurrence')),
    'repeat_until':  parse_timestamp(eattr('repeatUntil')),
    'completed':     parse_timestamp(eattr('completed')),
    'deleted':       parse_timestamp(eattr('deleted')),
    'estimated':     int(eattr('estimatedSeconds') or 0),
    'elapsed':       int(eattr('elapsedSeconds')   or 0),
    # tag everything with "astrid"
    'tags':          ['astrid'],
    'notes':         eattr('notes') or None,
  }

  for extra in element.getElementsByTagName('metadata'):
    if extra.getAttribute('key') == 'tags-tag':
      task['tags'].append(extra.getAttribute('value'))
    # TODO: alarm

  return task

def parse_timestamp(stamp):
  """Parse astrid date/timestamp (milliseconds) to a datetime instance.

  ::

    >>> parse_timestamp('1361905200321') # doctest: +ELLIPSIS
    datetime.datetime(2013, 2, 26, 19, 0, 0, 321000, tzinfo=...UTC...)

    >>> parse_timestamp('1389812400021').isoformat()
    '2014-01-15T19:00:00.021000+00:00'

  """

  if not stamp or stamp == '0':
    return

  try:
    stamp = int(stamp)
  except ValueError:
    raise AstridValueError('timestamp', stamp)

  # Astrid timestamps are milliseconds so divide by 1000
  # to get a unix timestamp (use '.0' to avoid rounding to whole numbers).
  # NOTE: It is not documented that datetime.fromtimestamp accepts fractions
  # so we might need to change this to call the constructor with ms * 1000.
  return datetime.fromtimestamp(stamp / 1000.0, UTC)

# TODO: consider parsing with https://github.com/collective/icalendar
def parse_recurrence(rule):
  """Convert astrid recurrence rule into dictionary for simplicity.

  ::

    >>> parse_recurrence('RRULE:FREQ=MONTHLY;INTERVAL=12')
    {'FREQ': 'MONTHLY', 'INTERVAL': 12}

  """

  if not rule:
    return

  matched = re.match(r'^RRULE:((?:[A-Z]+=[^;]+;?)+)$', rule)
  if not matched:
    raise AstridValueError('recurrence', rule)

  # TODO: use constants for normalization?
  # TODO: see icalendar.prop (vWeekday, etc) for parsing?

  recur = dict(s.split('=') for s in matched.group(1).split(';'))

  # cast numberic entries for ease of use
  for key in ('INTERVAL',):
    if key in recur:
      recur[key] = int(recur[key])

  return recur
<commit_msg>Separate fraction from timestamp and handle separately<commit_after>"""
Parse Astrid xml backup file into simple data structures.
"""

__docformat__ = 'reStructuredText'

from datetime import datetime
import re
from xml.dom import minidom

from dear_astrid.constants import UTC

__all__ = [
  'AstridValueError',
  'parse_xml',
  'parse_task',
  'parse_timestamp',
  'parse_recurrence',
]

# should this subclass ValueError?
class AstridValueError(Exception):
  """Value does not match expected format and cannot be parsed."""

  def __init__(self, key, val):
    Exception.__init__(self,
      'Unknown format for Astrid {0}: {1}'.format(key, val)
    )

# TODO: allow passing alternate parse_* callables?
def parse_xml(xml=None):
  """Parse xml string into list of task dictionaries"""

  xml = minidom.parseString(xml)
  format_ = xml.getElementsByTagName('astrid')[0].getAttribute('format')

  if format_ != '2':
    raise AstridValueError('xml', format_)

  return [parse_task(t) for t in xml.getElementsByTagName('task')]

def parse_task(element):
  """Parse task element into a simple dictionaries."""

  # use method object as shortcut
  eattr = element.getAttribute

  # other fields: timerStart flags is_public hideUntil
  # created modified calendarUri details
  # flags2: remind when due?
  task = {
    'title':         eattr('title'),
    # astrid's "no priority" is 3
    'priority':      int(eattr('importance') or 3),
    'due_date':      parse_timestamp(eattr('dueDate')),
    'recurrence':    parse_recurrence(eattr('recurrence')),
    'repeat_until':  parse_timestamp(eattr('repeatUntil')),
    'completed':     parse_timestamp(eattr('completed')),
    'deleted':       parse_timestamp(eattr('deleted')),
    'estimated':     int(eattr('estimatedSeconds') or 0),
    'elapsed':       int(eattr('elapsedSeconds')   or 0),
    # tag everything with "astrid"
    'tags':          ['astrid'],
    'notes':         eattr('notes') or None,
  }

  for extra in element.getElementsByTagName('metadata'):
    if extra.getAttribute('key') == 'tags-tag':
      task['tags'].append(extra.getAttribute('value'))
    # TODO: alarm

  return task

def parse_timestamp(stamp):
  """Parse astrid date/timestamp (milliseconds) to a datetime instance.

  ::

    >>> parse_timestamp('1361905200321') # doctest: +ELLIPSIS
    datetime.datetime(2013, 2, 26, 19, 0, 0, 321000, tzinfo=...UTC...)

    >>> parse_timestamp('1389812400021').isoformat()
    '2014-01-15T19:00:00.021000+00:00'

  """

  if not stamp or stamp == '0':
    return

  # Astrid timestamps are milliseconds.  We could divide by 1000.0
  # but that gets us into floating point trouble.  Instead, separate
  # them while it's still a string, then put the fraction back on later.

  # separate seconds from milliseconds (last three digits)
  sec, ms = stamp[0:-3], stamp[-3:]

  try:
    sec = int(sec)
    ms  = int(ms)
  except ValueError:
    raise AstridValueError('timestamp', stamp)

  dt = datetime.fromtimestamp(sec, UTC)
  # put the microseconds on
  return dt.replace(microsecond=(ms * 1000))

# TODO: consider parsing with https://github.com/collective/icalendar
def parse_recurrence(rule):
  """Convert astrid recurrence rule into dictionary for simplicity.

  ::

    >>> parse_recurrence('RRULE:FREQ=MONTHLY;INTERVAL=12')
    {'FREQ': 'MONTHLY', 'INTERVAL': 12}

  """

  if not rule:
    return

  matched = re.match(r'^RRULE:((?:[A-Z]+=[^;]+;?)+)$', rule)
  if not matched:
    raise AstridValueError('recurrence', rule)

  # TODO: use constants for normalization?
  # TODO: see icalendar.prop (vWeekday, etc) for parsing?

  recur = dict(s.split('=') for s in matched.group(1).split(';'))

  # cast numberic entries for ease of use
  for key in ('INTERVAL',):
    if key in recur:
      recur[key] = int(recur[key])

  return recur
<|endoftext|>


<commit_before># -*- coding: iso-8859-1 -*-
# Pretty printers for Eigen::Matrix
# Author: Benjamin Schindler <bschindler@inf.ethz.ch>
# This is still pretty basic as the python extension to gdb is still pretty basic.
# It cannot handle complex eigen types and it doesn't support any of the other eigen types
# Such as quaternion or some other type.
# This code supports fixed size as well as dynamic size matrices
# Licence - ment to be included in Eigen, so dual GPLv3 or LGPL
# NOTE: This code was only tested with the stable branch of eigen!

import gdb
import re
import itertools

class EigenMatrixPrinter:
	"Print Eigen Matrix of some kind"

	def __init__(self, val):
		"Extract all the necessary information"
		# The gdb extension does not support value template arguments - need to extract them by hand
		type = val.type
		if type.code == gdb.TYPE_CODE_REF:
			type = type.target()
		self.type = type.unqualified().strip_typedefs()
		tag = self.type.tag
		regex = re.compile('\<.*\>')
		m = regex.findall(tag)[0][1:-1]
		template_params = m.split(',')
		template_params = map(lambda x:x.replace(" ", ""), template_params)

		self.rows = int(template_params[1])
		self.cols = int(template_params[2])

		if self.rows == 10000:
			self.rows = val['m_storage']['m_rows']

		if self.cols == 10000:
			self.cols = val['m_storage']['m_cols']

		self.innerType = self.type.template_argument(0)

		self.val = val

	class _iterator:
		def __init__ (self, rows, cols, dataPtr):
			self.rows = rows
			self.cols = cols
			self.dataPtr = dataPtr
			self.currentRow = 0
			self.currentCol = 0

		def __iter__ (self):
			return self

		def next(self):
			if self.currentCol >= self.cols:
				raise StopIteration

			row = self.currentRow
			col = self.currentCol
			self.currentRow = self.currentRow + 1
			if self.currentRow >= self.rows:
				self.currentRow = 0
				self.currentCol = self.currentCol + 1

			item = self.dataPtr.dereference()
			self.dataPtr = self.dataPtr + 1

			return ('[%d, %d]' % (row, col), item)

	def children(self):
		data = self.val['m_storage']['m_data']
		# Fixed size matrices have a struct as their storage, so we need to walk through this
		if data.type.code == gdb.TYPE_CODE_STRUCT:
			data = data['array']
			data = data.cast(self.innerType.pointer())
		return self._iterator(self.rows, self.cols, data)

	def to_string(self):
		return "Eigen::Matrix<%s,%d,%d>" % (self.innerType, self.rows, self.cols)

def build_eigen_dictionary ():
	pretty_printers_dict[re.compile('^Eigen::Matrix<.*>$')] = lambda val: EigenMatrixPrinter(val)

def register_eigen_printers(obj):
	"Register eigen pretty-printers with objfile Obj"

	if obj == None:
		obj = gdb
	obj.pretty_printers.append(lookup_function)

def lookup_function(val):
	"Look-up and return a pretty-printer that can print va."

	type = val.type

	if type.code == gdb.TYPE_CODE_REF:
		type = type.target()

	type = type.unqualified().strip_typedefs()

	typename = type.tag
	if typename == None:
		return None

	for function in pretty_printers_dict:
		if function.search(typename):
			return pretty_printers_dict[function](val)

	return None

pretty_printers_dict = {}

build_eigen_dictionary ()
<commit_msg>Correct license header of the gdb pretty printer<commit_after># -*- coding: utf-8 -*-
# This file is part of Eigen, a lightweight C++ template library
# for linear algebra.
#
# Copyright (C) 2009 Benjamin Schindler <bschindler@inf.ethz.ch>
#
# Eigen is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 3 of the License, or (at your option) any later version.
#
# Alternatively, you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of
# the License, or (at your option) any later version.
#
# Eigen is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License or the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License and a copy of the GNU General Public License along with
# Eigen. If not, see <http://www.gnu.org/licenses/>.

# Pretty printers for Eigen::Matrix
# This is still pretty basic as the python extension to gdb is still pretty basic.
# It cannot handle complex eigen types and it doesn't support any of the other eigen types
# Such as quaternion or some other type.
# This code supports fixed size as well as dynamic size matrices

# To use it:
#
# * create a directory and put the file as well as an empty __init__.py in that directory
# * Create a ~/.gdbinit file, that contains the following:

import gdb
import re
import itertools

class EigenMatrixPrinter:
	"Print Eigen Matrix of some kind"

	def __init__(self, val):
		"Extract all the necessary information"
		# The gdb extension does not support value template arguments - need to extract them by hand
		type = val.type
		if type.code == gdb.TYPE_CODE_REF:
			type = type.target()
		self.type = type.unqualified().strip_typedefs()
		tag = self.type.tag
		regex = re.compile('\<.*\>')
		m = regex.findall(tag)[0][1:-1]
		template_params = m.split(',')
		template_params = map(lambda x:x.replace(" ", ""), template_params)

		self.rows = int(template_params[1])
		self.cols = int(template_params[2])

		if self.rows == 10000:
			self.rows = val['m_storage']['m_rows']

		if self.cols == 10000:
			self.cols = val['m_storage']['m_cols']

		self.innerType = self.type.template_argument(0)

		self.val = val

	class _iterator:
		def __init__ (self, rows, cols, dataPtr):
			self.rows = rows
			self.cols = cols
			self.dataPtr = dataPtr
			self.currentRow = 0
			self.currentCol = 0

		def __iter__ (self):
			return self

		def next(self):
			if self.currentCol >= self.cols:
				raise StopIteration

			row = self.currentRow
			col = self.currentCol
			self.currentRow = self.currentRow + 1
			if self.currentRow >= self.rows:
				self.currentRow = 0
				self.currentCol = self.currentCol + 1

			item = self.dataPtr.dereference()
			self.dataPtr = self.dataPtr + 1

			return ('[%d, %d]' % (row, col), item)

	def children(self):
		data = self.val['m_storage']['m_data']
		# Fixed size matrices have a struct as their storage, so we need to walk through this
		if data.type.code == gdb.TYPE_CODE_STRUCT:
			data = data['array']
			data = data.cast(self.innerType.pointer())
		return self._iterator(self.rows, self.cols, data)

	def to_string(self):
		return "Eigen::Matrix<%s,%d,%d>" % (self.innerType, self.rows, self.cols)

def build_eigen_dictionary ():
	pretty_printers_dict[re.compile('^Eigen::Matrix<.*>$')] = lambda val: EigenMatrixPrinter(val)

def register_eigen_printers(obj):
	"Register eigen pretty-printers with objfile Obj"

	if obj == None:
		obj = gdb
	obj.pretty_printers.append(lookup_function)

def lookup_function(val):
	"Look-up and return a pretty-printer that can print va."

	type = val.type

	if type.code == gdb.TYPE_CODE_REF:
		type = type.target()

	type = type.unqualified().strip_typedefs()

	typename = type.tag
	if typename == None:
		return None

	for function in pretty_printers_dict:
		if function.search(typename):
			return pretty_printers_dict[function](val)

	return None

pretty_printers_dict = {}

build_eigen_dictionary ()
<|endoftext|>


<commit_before>from datetime import date
from django.db import models
from django.conf import settings
from django.urls import reverse
from model_utils.fields import StatusField, MonitorField
from model_utils import Choices
from trix.fields import TrixField

class Declaracion(models.Model):
    ESTADOS = Choices('borrador', 'publicado')
    fecha_publicacion = MonitorField(monitor='estado', when=['publicado'], editable=False)
    cargado_por = models.ForeignKey(settings.AUTH_USER_MODEL, editable=False)
    título = models.CharField(max_length=100)
    fecha = models.DateField(default=date.today())
    declaración = TrixField()

    estado = StatusField(choices_name='ESTADOS')
    nota_interna = models.TextField(blank=True, null=True)

    def get_absolute_url(self):
        return reverse('update', kwargs={'pk': self.pk})

    class Meta:
        ordering = ('estado', '-fecha')

<commit_msg>inverse ordering<commit_after>from datetime import date
from django.db import models
from django.conf import settings
from django.urls import reverse
from model_utils.fields import StatusField, MonitorField
from model_utils import Choices
from trix.fields import TrixField

class Declaracion(models.Model):
    ESTADOS = Choices('borrador', 'publicado')
    fecha_publicacion = MonitorField(monitor='estado', when=['publicado'], editable=False)
    cargado_por = models.ForeignKey(settings.AUTH_USER_MODEL, editable=False)
    título = models.CharField(max_length=100)
    fecha = models.DateField(default=date.today())
    declaración = TrixField()

    estado = StatusField(choices_name='ESTADOS')
    nota_interna = models.TextField(blank=True, null=True)

    def get_absolute_url(self):
        return reverse('update', kwargs={'pk': self.pk})

    class Meta:
        ordering = ('estado', '-fecha', '-id')

<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
from __future__ import print_function
from builtins import input

import collections
import itertools
import sys
from dedupe.core import randomPairs, randomPairsMatch
from canonicalize.centroid import getCanonicalRep

def unique(seq) :
    seen = set()
    seen_add = seen.add
    return [x for x in seq if not (x in seen or seen_add(x))]

def consoleLabel(deduper): # pragma: no cover
    '''
    Command line interface for presenting and labeling training pairs
    by the user

    Argument :
    A deduper object
    '''

    finished = False
    use_previous = False
    fields = unique(field.field
                    for field
                    in deduper.data_model.primary_fields)

    buffer_len = 1 # Max number of previous operations
    examples_buffer = []
    uncertain_pairs = []

    while not finished :
        if examples_buffer and use_previous:
            record_pair, _ = examples_buffer.pop(0)
            use_previous = False
        else:
            if not uncertain_pairs:
                uncertain_pairs = deduper.uncertainPairs()
            record_pair = uncertain_pairs.pop()

        n_match_in_buffer = sum(label=='match' for _, label in examples_buffer)
        n_distinct_in_buffer = sum(label=='distinct' for _, label in examples_buffer)

        n_match, n_distinct = (len(deduper.training_pairs['match']) + n_match_in_buffer,
                        len(deduper.training_pairs['distinct']) + n_distinct_in_buffer)

        label = ''

        for pair in record_pair:
            for field in fields:
                line = "%s : %s" % (field, pair[field])
                print(line, file=sys.stderr)
            print(file=sys.stderr)

        print("{0}/10 positive, {1}/10 negative".format(n_match, n_distinct),
                file=sys.stderr)
        print('Do these records refer to the same thing?', file=sys.stderr)

        valid_response = False
        while not valid_response:
            if examples_buffer:
                prompt = '(y)es / (n)o / (u)nsure / (f)inished / (p)revious'
                valid_responses = {'y', 'n', 'u', 'f', 'p'}
            else:
                prompt = '(y)es / (n)o / (u)nsure / (f)inished'
                valid_responses = {'y', 'n', 'u', 'f'}

            print(prompt, file=sys.stderr)
            user_input = input()
            if user_input in valid_responses:
                valid_response = True

            if user_input == 'y' :
                examples_buffer.insert(0, (record_pair, 'match'))
            elif user_input == 'n' :
                examples_buffer.insert(0, (record_pair, 'distinct'))
            elif user_input == 'u':
                examples_buffer.insert(0, (record_pair, 'uncertain'))
            elif user_input == 'f':
                print('Finished labeling', file=sys.stderr)
                finished = True
            elif user_input == 'p':
                use_previous = True
                uncertain_pairs.append(record_pair)

        if len(examples_buffer) > buffer_len:
            record_pair, label = examples_buffer.pop()
            if label in ['distinct', 'match']:
                examples = {'distinct' : [], 'match' : []}
                examples[label].append(record_pair)
                deduper.markPairs(examples)

    for record_pair, label in examples_buffer:
        if label in ['distinct', 'match']:
            examples = {'distinct' : [], 'match' : []}
            examples[label].append(record_pair)
            deduper.markPairs(examples)
#

def trainingDataLink(data_1, data_2, common_key, training_size=50000) : # pragma: nocover
    '''
    Construct training data for consumption by the ActiveLearning
    markPairs method from already linked datasets.

    Arguments :
    data_1        -- Dictionary of records from first dataset, where the keys
                     are record_ids and the values are dictionaries with the
                     keys being field names

    data_2        -- Dictionary of records from second dataset, same form as
                     data_1

    common_key    -- The name of the record field that uniquely identifies
                     a match

    training_size -- the rough limit of the number of training examples,
                     defaults to 50000

    Warning:

    Every match must be identified by the sharing of a common key.
    This function assumes that if two records do not share a common key
    then they are distinct records.
    '''

    identified_records = collections.defaultdict(lambda: [[],[]])
    matched_pairs = set()
    distinct_pairs = set()

    for record_id, record in data_1.items() :
        identified_records[record[common_key]][0].append(record_id)

    for record_id, record in data_2.items() :
        identified_records[record[common_key]][1].append(record_id)

    for keys_1, keys_2 in identified_records.values() :
        if keys_1 and keys_2 :
            matched_pairs.update(itertools.product(keys_1, keys_2))

    keys_1 = list(data_1.keys())
    keys_2 = list(data_2.keys())

    random_pairs = [(keys_1[i], keys_2[j])
                    for i, j
                    in randomPairsMatch(len(data_1), len(data_2),
                                        training_size)]

    distinct_pairs = (pair for pair in random_pairs if pair not in matched_pairs)

    matched_records = [(data_1[key_1], data_2[key_2])
                       for key_1, key_2 in matched_pairs]
    distinct_records = [(data_1[key_1], data_2[key_2])
                        for key_1, key_2 in distinct_pairs]

    training_pairs = {'match' : matched_records,
                      'distinct' : distinct_records}

    return training_pairs

def trainingDataDedupe(data, common_key, training_size=50000) : # pragma: nocover
    '''
    Construct training data for consumption by the ActiveLearning
    markPairs method from an already deduplicated dataset.

    Arguments :
    data          -- Dictionary of records, where the keys are record_ids and
                     the values are dictionaries with the keys being
                     field names

    common_key    -- The name of the record field that uniquely identifies
                     a match

    training_size -- the rough limit of the number of training examples,
                     defaults to 50000

    Warning:

    Every match must be identified by the sharing of a common key.
    This function assumes that if two records do not share a common key
    then they are distinct records.
    '''

    identified_records = collections.defaultdict(list)
    matched_pairs = set()
    distinct_pairs = set()
    unique_record_ids = set()

    # a list of record_ids associated with each common_key
    for record_id, record in data.items() :
        unique_record_ids.add(record_id)
        identified_records[record[common_key]].append(record_id)

    # all combinations of matched_pairs from each common_key group
    for record_ids in identified_records.values() :
        if len(record_ids) > 1 :
            matched_pairs.update(itertools.combinations(sorted(record_ids), 2))

    # calculate indices using dedupe.core.randomPairs to avoid
    # the memory cost of enumerating all possible pairs
    unique_record_ids = list(unique_record_ids)
    pair_indices = randomPairs(len(unique_record_ids), training_size)
    distinct_pairs = set()
    for i, j in pair_indices:
        distinct_pairs.add((unique_record_ids[i],
                            unique_record_ids[j]))

    distinct_pairs -= matched_pairs

    matched_records = [(data[key_1], data[key_2])
                       for key_1, key_2 in matched_pairs]

    distinct_records = [(data[key_1], data[key_2])
                        for key_1, key_2 in distinct_pairs]

    training_pairs = {'match' : matched_records,
                      'distinct' : distinct_records}

    return training_pairs

def canonicalize(record_cluster): # pragma: nocover
    """
    Constructs a canonical representation of a duplicate cluster by
    finding canonical values for each field

    Arguments:
    record_cluster     --A list of records within a duplicate cluster, where
                         the records are dictionaries with field
                         names as keys and field values as values

    """
    return getCanonicalRep(record_cluster)
<commit_msg>a little more tidying of consoleLabel<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
from __future__ import print_function
from builtins import input

import collections
import itertools
import sys
from dedupe.core import randomPairs, randomPairsMatch
from canonicalize.centroid import getCanonicalRep

def unique(seq) :
    seen = set()
    seen_add = seen.add
    return [x for x in seq if not (x in seen or seen_add(x))]

def consoleLabel(deduper): # pragma: no cover
    '''
    Command line interface for presenting and labeling training pairs
    by the user

    Argument :
    A deduper object
    '''

    finished = False
    use_previous = False
    fields = unique(field.field
                    for field
                    in deduper.data_model.primary_fields)

    buffer_len = 1 # Max number of previous operations
    examples_buffer = []
    uncertain_pairs = []

    while not finished :
        if use_previous:
            record_pair, _ = examples_buffer.pop(0)
            use_previous = False
        else:
            if not uncertain_pairs:
                uncertain_pairs = deduper.uncertainPairs()
            record_pair = uncertain_pairs.pop()

        n_match = (len(deduper.training_pairs['match']) +
                   sum(label=='match' for _, label in examples_buffer))
        n_distinct = (len(deduper.training_pairs['distinct']) +
                      sum(label=='match' for _, label in examples_buffer))

        for pair in record_pair:
            for field in fields:
                line = "%s : %s" % (field, pair[field])
                print(line, file=sys.stderr)
            print(file=sys.stderr)

        print("{0}/10 positive, {1}/10 negative".format(n_match, n_distinct),
                file=sys.stderr)
        print('Do these records refer to the same thing?', file=sys.stderr)

        valid_response = False
        user_input = ''
        while not valid_response:
            if examples_buffer:
                prompt = '(y)es / (n)o / (u)nsure / (f)inished / (p)revious'
                valid_responses = {'y', 'n', 'u', 'f', 'p'}
            else:
                prompt = '(y)es / (n)o / (u)nsure / (f)inished'
                valid_responses = {'y', 'n', 'u', 'f'}

            print(prompt, file=sys.stderr)
            user_input = input()
            if user_input in valid_responses:
                valid_response = True

        if user_input == 'y':
            examples_buffer.insert(0, (record_pair, 'match'))
        elif user_input == 'n' :
            examples_buffer.insert(0, (record_pair, 'distinct'))
        elif user_input == 'u':
            examples_buffer.insert(0, (record_pair, 'uncertain'))
        elif user_input == 'f':
            print('Finished labeling', file=sys.stderr)
            finished = True
        elif user_input == 'p':
            use_previous = True
            uncertain_pairs.append(record_pair)

        if len(examples_buffer) > buffer_len:
            record_pair, label = examples_buffer.pop()
            if label in ['distinct', 'match']:
                examples = {'distinct' : [], 'match' : []}
                examples[label].append(record_pair)
                deduper.markPairs(examples)

    for record_pair, label in examples_buffer:
        if label in ['distinct', 'match']:
            examples = {'distinct' : [], 'match' : []}
            examples[label].append(record_pair)
            deduper.markPairs(examples)

def trainingDataLink(data_1, data_2, common_key, training_size=50000) : # pragma: nocover
    '''
    Construct training data for consumption by the ActiveLearning
    markPairs method from already linked datasets.

    Arguments :
    data_1        -- Dictionary of records from first dataset, where the keys
                     are record_ids and the values are dictionaries with the
                     keys being field names

    data_2        -- Dictionary of records from second dataset, same form as
                     data_1

    common_key    -- The name of the record field that uniquely identifies
                     a match

    training_size -- the rough limit of the number of training examples,
                     defaults to 50000

    Warning:

    Every match must be identified by the sharing of a common key.
    This function assumes that if two records do not share a common key
    then they are distinct records.
    '''

    identified_records = collections.defaultdict(lambda: [[],[]])
    matched_pairs = set()
    distinct_pairs = set()

    for record_id, record in data_1.items() :
        identified_records[record[common_key]][0].append(record_id)

    for record_id, record in data_2.items() :
        identified_records[record[common_key]][1].append(record_id)

    for keys_1, keys_2 in identified_records.values() :
        if keys_1 and keys_2 :
            matched_pairs.update(itertools.product(keys_1, keys_2))

    keys_1 = list(data_1.keys())
    keys_2 = list(data_2.keys())

    random_pairs = [(keys_1[i], keys_2[j])
                    for i, j
                    in randomPairsMatch(len(data_1), len(data_2),
                                        training_size)]

    distinct_pairs = (pair for pair in random_pairs if pair not in matched_pairs)

    matched_records = [(data_1[key_1], data_2[key_2])
                       for key_1, key_2 in matched_pairs]
    distinct_records = [(data_1[key_1], data_2[key_2])
                        for key_1, key_2 in distinct_pairs]

    training_pairs = {'match' : matched_records,
                      'distinct' : distinct_records}

    return training_pairs

def trainingDataDedupe(data, common_key, training_size=50000) : # pragma: nocover
    '''
    Construct training data for consumption by the ActiveLearning
    markPairs method from an already deduplicated dataset.

    Arguments :
    data          -- Dictionary of records, where the keys are record_ids and
                     the values are dictionaries with the keys being
                     field names

    common_key    -- The name of the record field that uniquely identifies
                     a match

    training_size -- the rough limit of the number of training examples,
                     defaults to 50000

    Warning:

    Every match must be identified by the sharing of a common key.
    This function assumes that if two records do not share a common key
    then they are distinct records.
    '''

    identified_records = collections.defaultdict(list)
    matched_pairs = set()
    distinct_pairs = set()
    unique_record_ids = set()

    # a list of record_ids associated with each common_key
    for record_id, record in data.items() :
        unique_record_ids.add(record_id)
        identified_records[record[common_key]].append(record_id)

    # all combinations of matched_pairs from each common_key group
    for record_ids in identified_records.values() :
        if len(record_ids) > 1 :
            matched_pairs.update(itertools.combinations(sorted(record_ids), 2))

    # calculate indices using dedupe.core.randomPairs to avoid
    # the memory cost of enumerating all possible pairs
    unique_record_ids = list(unique_record_ids)
    pair_indices = randomPairs(len(unique_record_ids), training_size)
    distinct_pairs = set()
    for i, j in pair_indices:
        distinct_pairs.add((unique_record_ids[i],
                            unique_record_ids[j]))

    distinct_pairs -= matched_pairs

    matched_records = [(data[key_1], data[key_2])
                       for key_1, key_2 in matched_pairs]

    distinct_records = [(data[key_1], data[key_2])
                        for key_1, key_2 in distinct_pairs]

    training_pairs = {'match' : matched_records,
                      'distinct' : distinct_records}

    return training_pairs

def canonicalize(record_cluster): # pragma: nocover
    """
    Constructs a canonical representation of a duplicate cluster by
    finding canonical values for each field

    Arguments:
    record_cluster     --A list of records within a duplicate cluster, where
                         the records are dictionaries with field
                         names as keys and field values as values

    """
    return getCanonicalRep(record_cluster)
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
Convenience functions for in memory deduplication
"""

import collections
import dedupe.core

def dataSample(data, sample_size):
    '''Randomly sample pairs of records from a data dictionary'''

    data_list = data.values()
    random_pairs = dedupe.core.randomPairs(len(data_list), sample_size)

    return tuple((data_list[k1], data_list[k2]) for k1, k2 in random_pairs)

def blockData(data_d, blocker):

    blocks = dedupe.core.OrderedDict({})
    record_blocks = dedupe.core.OrderedDict({})
    key_blocks = dedupe.core.OrderedDict({})

    blocker.tfIdfBlocks(data_d.iteritems())

    for (record_id, record) in data_d.iteritems():
        for key in blocker((record_id, record)):
            blocks.setdefault(key, {}).update({record_id : record})

    blocked_records = tuple(block for block in blocks.values())

    return blocked_records

<commit_msg>pull ordered Dict from backports<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
Convenience functions for in memory deduplication
"""

import collections
import dedupe.core
try:
    from collections import OrderedDict
except ImportError :
    from backport import OrderedDict

def dataSample(data, sample_size):
    '''Randomly sample pairs of records from a data dictionary'''

    data_list = data.values()
    random_pairs = dedupe.core.randomPairs(len(data_list), sample_size)

    return tuple((data_list[k1], data_list[k2]) for k1, k2 in random_pairs)

def blockData(data_d, blocker):

    blocks = OrderedDict({})
    record_blocks =OrderedDict({})
    key_blocks = OrderedDict({})

    blocker.tfIdfBlocks(data_d.iteritems())

    for (record_id, record) in data_d.iteritems():
        for key in blocker((record_id, record)):
            blocks.setdefault(key, {}).update({record_id : record})

    blocked_records = tuple(block for block in blocks.values())

    return blocked_records

<|endoftext|>


<commit_before>import logging
import gobject
from datetime import datetime, timedelta, time
from itertools import izip, imap
from functools import partial

# Victron packages
from ve_utils import exit_on_error
from delegates.base import SystemCalcDelegate
from delegates.batterylife import BatteryLife, BLPATH
from delegates.batterylife import State as BatteryLifeState
from delegates.dvcc import Dvcc

HUB4_SERVICE = 'com.victronenergy.hub4'

# Number of scheduled charge slots
NUM_SCHEDULES = 5

def prev_week_day(adate, w):
	""" finds the previous w-day of the week before adate.
	    Sun=0 or 7, Sat=6, that is what unix uses. """
	w %= 7
	return adate - timedelta(days=(adate.weekday()+7-w) % 7 + 1)

def next_week_day(adate, w):
	""" Finds the next w-day after or equal to adate.
	    Sun=0 or 7, Sat=6, that is what unix uses. """
	w %= 7
	return adate + timedelta(days=(w - adate.weekday() - 1) % 7)

def next_schedule_day(adate, w):
		if w < 7:
			# A specific week day
			return next_week_day(adate, w)
		elif w == 7:
			# 7 days a week
			return adate
		elif w == 8:
			# week days
			if adate.weekday() > 4:
				return next_week_day(adate, 1) # Monday
			return adate

		# w >=9, weekend days
		if adate.weekday() < 5:
			return next_week_day(adate, 6) # Saturday
		return adate

def prev_schedule_day(adate, w):
		if w < 7:
			# A specific week day
			return prev_week_day(adate, w)
		elif w == 7:
			# 7 days a week
			return adate - timedelta(days=1)
		elif w == 8:
			# week days
			if adate.weekday() in (0, 6):
				return prev_week_day(adate, 5) # Mon,Sun preceded by Friday
			return adate - timedelta(days=1)

		# w >= 9, weekend days
		if 0 < adate.weekday() < 5:
			return prev_week_day(adate, 0) # Sunday
		return adate - timedelta(days=1)

class ScheduledWindow(object):
	def __init__(self, starttime, duration):
		self.start = starttime
		self.stop = self.start + timedelta(seconds=duration)

	def __contains__(self, t):
		return self.start <= t < self.stop

	def __eq__(self, other):
		return self.start == other.start and self.stop == other.stop

	def __repr__(self):
		return "Start: {}, Stop: {}".format(self.start, self.stop)

class ScheduledChargeWindow(ScheduledWindow):
	def __init__(self, starttime, duration, soc):
		super(ScheduledChargeWindow, self).__init__(starttime, duration)
		self.soc = soc

	def soc_reached(self, s):
		return not self.soc >= 100 and s >= self.soc

	def __repr__(self):
		return "Start charge: {}, Stop: {}, Soc: {}".format(
			self.start, self.stop, self.soc)

class ScheduledCharging(SystemCalcDelegate):
	""" Let the system do other things based on time schedule. """

	_get_time = datetime.now

	def __init__(self):
		super(ScheduledCharging, self).__init__()
		self.soc = None
		self.pvpower = 0
		self.active = False
		self.hysteresis = True
		self._timer = gobject.timeout_add(5000, exit_on_error, self._on_timer)

	def set_sources(self, dbusmonitor, settings, dbusservice):
		SystemCalcDelegate.set_sources(self, dbusmonitor, settings, dbusservice)
		self._dbusservice.add_path('/Control/ScheduledCharge', value=0)

	def get_input(self):
		return [
			(HUB4_SERVICE, ['/Overrides/ForceCharge', '/Overrides/MaxDischargePower'])
		]

	def settings_changed(self, setting, oldvalue, newvalue):
		if setting.startswith("schedule_soc_"):
			# target SOC was modified. Disable the hysteresis on the next
			# run.
			self.hysteresis = False

	def get_settings(self):
		settings = []

		# Paths for scheduled charging. We'll allow 5 for now.
		for i in range(NUM_SCHEDULES):
			settings.append(("schedule_day_{}".format(i),
				BLPATH + "/Schedule/Charge/{}/Day".format(i), -7, -9, 9))
			settings.append(("schedule_start_{}".format(i),
				BLPATH + "/Schedule/Charge/{}/Start".format(i), 0, 0, 0))
			settings.append(("schedule_duration_{}".format(i),
				BLPATH + "/Schedule/Charge/{}/Duration".format(i), 0, 0, 0))
			settings.append(("schedule_soc_{}".format(i),
				BLPATH + "/Schedule/Charge/{}/Soc".format(i), 100, 0, 100))

		return settings

	@classmethod
	def _charge_windows(klass, today, days, starttimes, durations, stopsocs):
		starttimes = (time(x/3600, x/60 % 60, x % 60) for x in starttimes)

		for d, starttime, duration, soc in izip(days, starttimes, durations, stopsocs):
			if d >= 0:
				d0 = prev_schedule_day(today, d)
				d1 = next_schedule_day(today, d)
				yield ScheduledChargeWindow(
					datetime.combine(d0, starttime), duration, soc)
				yield ScheduledChargeWindow(
					datetime.combine(d1, starttime), duration, soc)

	def charge_windows(self, today):
		days = (self._settings['schedule_day_{}'.format(i)] for i in range(NUM_SCHEDULES))
		starttimes = (self._settings['schedule_start_{}'.format(i)] for i in range(NUM_SCHEDULES))
		durations = (self._settings['schedule_duration_{}'.format(i)] for i in range(NUM_SCHEDULES))
		stopsocs = (self._settings['schedule_soc_{}'.format(i)] for i in range(NUM_SCHEDULES))
		return self._charge_windows(today, days, starttimes, durations, stopsocs)

	@property
	def forcecharge(self):
		return self._dbusmonitor.get_value(HUB4_SERVICE, '/Overrides/ForceCharge')

	@forcecharge.setter
	def forcecharge(self, v):
		return self._dbusmonitor.set_value_async(HUB4_SERVICE,
			'/Overrides/ForceCharge', 1 if v else 0)

	@property
	def maxdischargepower(self):
		return self._dbusmonitor.get_value(HUB4_SERVICE, '/Overrides/MaxDischargePower')

	@maxdischargepower.setter
	def maxdischargepower(self, v):
		return self._dbusmonitor.set_value_async(HUB4_SERVICE, '/Overrides/MaxDischargePower', v)

	def _on_timer(self):
		if self.soc is None:
			return True

		if not Dvcc.instance.has_ess_assistant:
			return True

		if BatteryLife.instance.state == BatteryLifeState.KeepCharged:
			self._dbusservice['/Control/ScheduledCharge'] = 0
			return True

		now = self._get_time()
		today = now.date()

		for w in self.charge_windows(today):
			if now in w:
				if w.soc_reached(self.soc):
					self.forcecharge = False
				elif self.hysteresis and w.soc_reached(self.soc + 5):
					# If we are within 5%, keep it the same, but write it to
					# avoid a timeout.
					self.forcecharge = self.forcecharge
				else:
					# SoC not reached yet
					# Note: soc_reached always returns False for a target of
					# 100%, so this is the only branch that is ever excuted
					# in those cases.
					self.forcecharge = True

				# Signal that scheduled charging is active
				self.active = True

				# The discharge is limited to 1W or whatever is available
				# from PV. 1W essentially disables discharge without
				# disabling feed-in, so Power-Assist and feeding in
				# the excess continues to work. Setting this to the pv-power
				# causes it to directly consume the PV for loads and charge
				# only with the excess. Scale it between 80% and 93%
				# of PV-power depending on the SOC.
				scale = 0.8 + min(max(0, self.soc - w.soc), 1.3)*0.1
				self.maxdischargepower = max(1, round(self.pvpower*scale))
				break
		else:
			self.forcecharge = False
			self.maxdischargepower = -1
			self.active = False

		self._dbusservice['/Control/ScheduledCharge'] = int(self.active)
		self.hysteresis = True
		return True

	def update_values(self, newvalues):
		self.soc = newvalues.get('/Dc/Battery/Soc')
		self.pvpower = max(newvalues.get('/Dc/Pv/Power'), 0)
<commit_msg>Allow schedule charge day to go to -10.<commit_after>import logging
import gobject
from datetime import datetime, timedelta, time
from itertools import izip, imap
from functools import partial

# Victron packages
from ve_utils import exit_on_error
from delegates.base import SystemCalcDelegate
from delegates.batterylife import BatteryLife, BLPATH
from delegates.batterylife import State as BatteryLifeState
from delegates.dvcc import Dvcc

HUB4_SERVICE = 'com.victronenergy.hub4'

# Number of scheduled charge slots
NUM_SCHEDULES = 5

def prev_week_day(adate, w):
	""" finds the previous w-day of the week before adate.
	    Sun=0 or 7, Sat=6, that is what unix uses. """
	w %= 7
	return adate - timedelta(days=(adate.weekday()+7-w) % 7 + 1)

def next_week_day(adate, w):
	""" Finds the next w-day after or equal to adate.
	    Sun=0 or 7, Sat=6, that is what unix uses. """
	w %= 7
	return adate + timedelta(days=(w - adate.weekday() - 1) % 7)

def next_schedule_day(adate, w):
		if w < 7:
			# A specific week day
			return next_week_day(adate, w)
		elif w == 7:
			# 7 days a week
			return adate
		elif w == 8:
			# week days
			if adate.weekday() > 4:
				return next_week_day(adate, 1) # Monday
			return adate

		# w >=9, weekend days
		if adate.weekday() < 5:
			return next_week_day(adate, 6) # Saturday
		return adate

def prev_schedule_day(adate, w):
		if w < 7:
			# A specific week day
			return prev_week_day(adate, w)
		elif w == 7:
			# 7 days a week
			return adate - timedelta(days=1)
		elif w == 8:
			# week days
			if adate.weekday() in (0, 6):
				return prev_week_day(adate, 5) # Mon,Sun preceded by Friday
			return adate - timedelta(days=1)

		# w >= 9, weekend days
		if 0 < adate.weekday() < 5:
			return prev_week_day(adate, 0) # Sunday
		return adate - timedelta(days=1)

class ScheduledWindow(object):
	def __init__(self, starttime, duration):
		self.start = starttime
		self.stop = self.start + timedelta(seconds=duration)

	def __contains__(self, t):
		return self.start <= t < self.stop

	def __eq__(self, other):
		return self.start == other.start and self.stop == other.stop

	def __repr__(self):
		return "Start: {}, Stop: {}".format(self.start, self.stop)

class ScheduledChargeWindow(ScheduledWindow):
	def __init__(self, starttime, duration, soc):
		super(ScheduledChargeWindow, self).__init__(starttime, duration)
		self.soc = soc

	def soc_reached(self, s):
		return not self.soc >= 100 and s >= self.soc

	def __repr__(self):
		return "Start charge: {}, Stop: {}, Soc: {}".format(
			self.start, self.stop, self.soc)

class ScheduledCharging(SystemCalcDelegate):
	""" Let the system do other things based on time schedule. """

	_get_time = datetime.now

	def __init__(self):
		super(ScheduledCharging, self).__init__()
		self.soc = None
		self.pvpower = 0
		self.active = False
		self.hysteresis = True
		self._timer = gobject.timeout_add(5000, exit_on_error, self._on_timer)

	def set_sources(self, dbusmonitor, settings, dbusservice):
		SystemCalcDelegate.set_sources(self, dbusmonitor, settings, dbusservice)
		self._dbusservice.add_path('/Control/ScheduledCharge', value=0)

	def get_input(self):
		return [
			(HUB4_SERVICE, ['/Overrides/ForceCharge', '/Overrides/MaxDischargePower'])
		]

	def settings_changed(self, setting, oldvalue, newvalue):
		if setting.startswith("schedule_soc_"):
			# target SOC was modified. Disable the hysteresis on the next
			# run.
			self.hysteresis = False

	def get_settings(self):
		settings = []

		# Paths for scheduled charging. We'll allow 5 for now.
		for i in range(NUM_SCHEDULES):
			settings.append(("schedule_day_{}".format(i),
				BLPATH + "/Schedule/Charge/{}/Day".format(i), -7, -10, 9))
			settings.append(("schedule_start_{}".format(i),
				BLPATH + "/Schedule/Charge/{}/Start".format(i), 0, 0, 0))
			settings.append(("schedule_duration_{}".format(i),
				BLPATH + "/Schedule/Charge/{}/Duration".format(i), 0, 0, 0))
			settings.append(("schedule_soc_{}".format(i),
				BLPATH + "/Schedule/Charge/{}/Soc".format(i), 100, 0, 100))

		return settings

	@classmethod
	def _charge_windows(klass, today, days, starttimes, durations, stopsocs):
		starttimes = (time(x/3600, x/60 % 60, x % 60) for x in starttimes)

		for d, starttime, duration, soc in izip(days, starttimes, durations, stopsocs):
			if d >= 0:
				d0 = prev_schedule_day(today, d)
				d1 = next_schedule_day(today, d)
				yield ScheduledChargeWindow(
					datetime.combine(d0, starttime), duration, soc)
				yield ScheduledChargeWindow(
					datetime.combine(d1, starttime), duration, soc)

	def charge_windows(self, today):
		days = (self._settings['schedule_day_{}'.format(i)] for i in range(NUM_SCHEDULES))
		starttimes = (self._settings['schedule_start_{}'.format(i)] for i in range(NUM_SCHEDULES))
		durations = (self._settings['schedule_duration_{}'.format(i)] for i in range(NUM_SCHEDULES))
		stopsocs = (self._settings['schedule_soc_{}'.format(i)] for i in range(NUM_SCHEDULES))
		return self._charge_windows(today, days, starttimes, durations, stopsocs)

	@property
	def forcecharge(self):
		return self._dbusmonitor.get_value(HUB4_SERVICE, '/Overrides/ForceCharge')

	@forcecharge.setter
	def forcecharge(self, v):
		return self._dbusmonitor.set_value_async(HUB4_SERVICE,
			'/Overrides/ForceCharge', 1 if v else 0)

	@property
	def maxdischargepower(self):
		return self._dbusmonitor.get_value(HUB4_SERVICE, '/Overrides/MaxDischargePower')

	@maxdischargepower.setter
	def maxdischargepower(self, v):
		return self._dbusmonitor.set_value_async(HUB4_SERVICE, '/Overrides/MaxDischargePower', v)

	def _on_timer(self):
		if self.soc is None:
			return True

		if not Dvcc.instance.has_ess_assistant:
			return True

		if BatteryLife.instance.state == BatteryLifeState.KeepCharged:
			self._dbusservice['/Control/ScheduledCharge'] = 0
			return True

		now = self._get_time()
		today = now.date()

		for w in self.charge_windows(today):
			if now in w:
				if w.soc_reached(self.soc):
					self.forcecharge = False
				elif self.hysteresis and w.soc_reached(self.soc + 5):
					# If we are within 5%, keep it the same, but write it to
					# avoid a timeout.
					self.forcecharge = self.forcecharge
				else:
					# SoC not reached yet
					# Note: soc_reached always returns False for a target of
					# 100%, so this is the only branch that is ever excuted
					# in those cases.
					self.forcecharge = True

				# Signal that scheduled charging is active
				self.active = True

				# The discharge is limited to 1W or whatever is available
				# from PV. 1W essentially disables discharge without
				# disabling feed-in, so Power-Assist and feeding in
				# the excess continues to work. Setting this to the pv-power
				# causes it to directly consume the PV for loads and charge
				# only with the excess. Scale it between 80% and 93%
				# of PV-power depending on the SOC.
				scale = 0.8 + min(max(0, self.soc - w.soc), 1.3)*0.1
				self.maxdischargepower = max(1, round(self.pvpower*scale))
				break
		else:
			self.forcecharge = False
			self.maxdischargepower = -1
			self.active = False

		self._dbusservice['/Control/ScheduledCharge'] = int(self.active)
		self.hysteresis = True
		return True

	def update_values(self, newvalues):
		self.soc = newvalues.get('/Dc/Battery/Soc')
		self.pvpower = max(newvalues.get('/Dc/Pv/Power'), 0)
<|endoftext|>