<commit_before>"""
Unit test for Parameterized.
"""

import unittest
import param
import numbergen

# CEBALERT: not anything like a complete test of Parameterized!

import random

from nose.tools import istest, nottest

@nottest
class _SomeRandomNumbers(object):
    def __call__(self):
        return random.random()

@nottest
class TestPO(param.Parameterized):
    inst = param.Parameter(default=[1,2,3],instantiate=True)
    notinst = param.Parameter(default=[1,2,3],instantiate=False)
    const = param.Parameter(default=1,constant=True)
    ro = param.Parameter(default="Hello",readonly=True)
    ro2 = param.Parameter(default=object(),readonly=True,instantiate=True)

    dyn = param.Dynamic(default=1)

@nottest
class AnotherTestPO(param.Parameterized):
    instPO = param.Parameter(default=TestPO(),instantiate=True)
    notinstPO = param.Parameter(default=TestPO(),instantiate=False)

@nottest
class TestAbstractPO(param.Parameterized):
    __abstract = True

@nottest
class TestParamInstantiation(AnotherTestPO):
    instPO = param.Parameter(default=AnotherTestPO(),instantiate=False)

@istest
class TestParameterized(unittest.TestCase):

    def test_constant_parameter(self):
        """Test that you can't set a constant parameter after construction."""
        testpo = TestPO(const=17)
        self.assertEqual(testpo.const,17)
        self.assertRaises(TypeError,setattr,testpo,'const',10)

        # check you can set on class
        TestPO.const=9
        testpo = TestPO()
        self.assertEqual(testpo.const,9)

    def test_readonly_parameter(self):
        """Test that you can't set a read-only parameter on construction or as an attribute."""
        testpo = TestPO()
        self.assertEqual(testpo.ro,"Hello")

        # CB: couldn't figure out how to use assertRaises
        try:
            t = TestPO(ro=20)
        except TypeError:
            pass
        else:
            raise AssertionError("Read-only parameter was set!")

        t=TestPO()
        self.assertRaises(TypeError,setattr,t,'ro',10)

        # check you cannot set on class
        self.assertRaises(TypeError,setattr,TestPO,'ro',5)

        self.assertEqual(testpo.params()['ro'].constant,True)

        # check that instantiate was ignored for readonly
        self.assertEqual(testpo.params()['ro2'].instantiate,False)

    def test_basic_instantiation(self):
        """Check that instantiated parameters are copied into objects."""

        testpo = TestPO()

        self.assertEqual(testpo.inst,TestPO.inst)
        self.assertEqual(testpo.notinst,TestPO.notinst)

        TestPO.inst[1]=7
        TestPO.notinst[1]=7

        self.assertEqual(testpo.notinst,[1,7,3])
        self.assertEqual(testpo.inst,[1,2,3])

    def test_more_instantiation(self):
        """Show that objects in instantiated Parameters can still share data."""
        anothertestpo = AnotherTestPO()

        ### CB: AnotherTestPO.instPO is instantiated, but
        ### TestPO.notinst is not instantiated - so notinst is still
        ### shared, even by instantiated parameters of AnotherTestPO.
        ### Seems like this behavior of Parameterized could be
        ### confusing, so maybe mention it in documentation somewhere.
        TestPO.notinst[1]=7
        # (if you thought your instPO was completely an independent object, you
        # might be expecting [1,2,3] here)
        self.assertEqual(anothertestpo.instPO.notinst,[1,7,3])

    def test_instantiation_inheritance(self):
        """Check that instantiate=True is always inherited (SF.net #2483932)."""
        t = TestParamInstantiation()
        assert t.params('instPO').instantiate is True
        assert isinstance(t.instPO,AnotherTestPO)

    def test_abstract_class(self):
        """Check that a class declared abstract actually shows up as abstract."""
        self.assertEqual(TestAbstractPO.abstract,True)
        self.assertEqual(TestPO.abstract,False)

    def test_params(self):
        """Basic tests of params() method."""

        # CB: test not so good because it requires changes if params
        # of PO are changed
        assert 'name' in param.Parameterized.params()
        assert len(param.Parameterized.params()) in [1,2]

        ## check for bug where subclass Parameters were not showing up
        ## if params() already called on a super class.
        assert 'inst' in TestPO.params()
        assert 'notinst' in TestPO.params()

        ## check caching
        assert param.Parameterized.params() is param.Parameterized().params(), "Results of params() should be cached." # just for performance reasons

    def test_state_saving(self):
        t = TestPO(dyn=_SomeRandomNumbers())
        g = t.get_value_generator('dyn')
        g._Dynamic_time_fn=None
        assert t.dyn!=t.dyn
        orig = t.dyn
        t.state_push()
        t.dyn
        assert t.inspect_value('dyn')!=orig
        t.state_pop()
        assert t.inspect_value('dyn')==orig

from param import parameterized

@nottest
class some_fn(param.ParameterizedFunction):
   num_phase = param.Number(18)
   frequencies = param.List([99])
   scale = param.Number(0.3)

   def __call__(self,**params_to_override):
       params = parameterized.ParamOverrides(self,params_to_override)
       num_phase = params['num_phase']
       frequencies = params['frequencies']
       scale = params['scale']
       return scale,num_phase,frequencies

instance = some_fn.instance()

@istest
class TestParameterizedFunction(unittest.TestCase):

    def _basic_tests(self,fn):
        self.assertEqual(fn(),(0.3,18,[99]))
        self.assertEqual(fn(frequencies=[1,2,3]),(0.3,18,[1,2,3]))
        self.assertEqual(fn(),(0.3,18,[99]))

        fn.frequencies=[10,20,30]
        self.assertEqual(fn(frequencies=[1,2,3]),(0.3,18,[1,2,3]))
        self.assertEqual(fn(),(0.3,18,[10,20,30]))

    def test_parameterized_function(self):
        self._basic_tests(some_fn)

    def test_parameterized_function_instance(self):
        self._basic_tests(instance)

    def test_pickle_instance(self):
        import pickle
        s = pickle.dumps(instance)
        instance.scale=0.8
        i = pickle.loads(s)
        self.assertEqual(i(),(0.3,18,[10,20,30]))

@nottest
class TestPO1(param.Parameterized):
    x = param.Number(default=numbergen.UniformRandom(lbound=-1,ubound=1,seed=1),bounds=(-1,1))
    y = param.Number(default=1,bounds=(-1,1))

@istest
class TestNumberParameter(unittest.TestCase):

    def test_outside_bounds(self):
        t1 = TestPO1()
        # Test bounds (non-dynamic number)
        try:
            t1.y = 10
        except ValueError:
            pass
        else:
            assert False, "Should raise ValueError."

    def test_outside_bounds_numbergen(self):
        t1 = TestPO1()
        # Test bounds (dynamic number)
        t1.x = numbergen.UniformRandom(lbound=2,ubound=3)  # bounds not checked on set
        try:
            v1 = t1.x
        except ValueError:
            pass
        else:
            assert False, "Should raise ValueError."

if __name__ == "__main__":
    import nose
    nose.runmodule()
<commit_msg>Converted Topographica testParamOverrides doctest to unit tests<commit_after>"""
Unit test for Parameterized.
"""

import unittest
import param
import numbergen

# CEBALERT: not anything like a complete test of Parameterized!

import random
from nose.tools import istest, nottest

from param.parameterized import ParamOverrides

@nottest
class _SomeRandomNumbers(object):
    def __call__(self):
        return random.random()

@nottest
class TestPO(param.Parameterized):
    inst = param.Parameter(default=[1,2,3],instantiate=True)
    notinst = param.Parameter(default=[1,2,3],instantiate=False)
    const = param.Parameter(default=1,constant=True)
    ro = param.Parameter(default="Hello",readonly=True)
    ro2 = param.Parameter(default=object(),readonly=True,instantiate=True)

    dyn = param.Dynamic(default=1)

@nottest
class AnotherTestPO(param.Parameterized):
    instPO = param.Parameter(default=TestPO(),instantiate=True)
    notinstPO = param.Parameter(default=TestPO(),instantiate=False)

@nottest
class TestAbstractPO(param.Parameterized):
    __abstract = True

@nottest
class TestParamInstantiation(AnotherTestPO):
    instPO = param.Parameter(default=AnotherTestPO(),instantiate=False)

@istest
class TestParameterized(unittest.TestCase):

    def test_constant_parameter(self):
        """Test that you can't set a constant parameter after construction."""
        testpo = TestPO(const=17)
        self.assertEqual(testpo.const,17)
        self.assertRaises(TypeError,setattr,testpo,'const',10)

        # check you can set on class
        TestPO.const=9
        testpo = TestPO()
        self.assertEqual(testpo.const,9)

    def test_readonly_parameter(self):
        """Test that you can't set a read-only parameter on construction or as an attribute."""
        testpo = TestPO()
        self.assertEqual(testpo.ro,"Hello")

        # CB: couldn't figure out how to use assertRaises
        try:
            t = TestPO(ro=20)
        except TypeError:
            pass
        else:
            raise AssertionError("Read-only parameter was set!")

        t=TestPO()
        self.assertRaises(TypeError,setattr,t,'ro',10)

        # check you cannot set on class
        self.assertRaises(TypeError,setattr,TestPO,'ro',5)

        self.assertEqual(testpo.params()['ro'].constant,True)

        # check that instantiate was ignored for readonly
        self.assertEqual(testpo.params()['ro2'].instantiate,False)

    def test_basic_instantiation(self):
        """Check that instantiated parameters are copied into objects."""

        testpo = TestPO()

        self.assertEqual(testpo.inst,TestPO.inst)
        self.assertEqual(testpo.notinst,TestPO.notinst)

        TestPO.inst[1]=7
        TestPO.notinst[1]=7

        self.assertEqual(testpo.notinst,[1,7,3])
        self.assertEqual(testpo.inst,[1,2,3])

    def test_more_instantiation(self):
        """Show that objects in instantiated Parameters can still share data."""
        anothertestpo = AnotherTestPO()

        ### CB: AnotherTestPO.instPO is instantiated, but
        ### TestPO.notinst is not instantiated - so notinst is still
        ### shared, even by instantiated parameters of AnotherTestPO.
        ### Seems like this behavior of Parameterized could be
        ### confusing, so maybe mention it in documentation somewhere.
        TestPO.notinst[1]=7
        # (if you thought your instPO was completely an independent object, you
        # might be expecting [1,2,3] here)
        self.assertEqual(anothertestpo.instPO.notinst,[1,7,3])

    def test_instantiation_inheritance(self):
        """Check that instantiate=True is always inherited (SF.net #2483932)."""
        t = TestParamInstantiation()
        assert t.params('instPO').instantiate is True
        assert isinstance(t.instPO,AnotherTestPO)

    def test_abstract_class(self):
        """Check that a class declared abstract actually shows up as abstract."""
        self.assertEqual(TestAbstractPO.abstract,True)
        self.assertEqual(TestPO.abstract,False)

    def test_params(self):
        """Basic tests of params() method."""

        # CB: test not so good because it requires changes if params
        # of PO are changed
        assert 'name' in param.Parameterized.params()
        assert len(param.Parameterized.params()) in [1,2]

        ## check for bug where subclass Parameters were not showing up
        ## if params() already called on a super class.
        assert 'inst' in TestPO.params()
        assert 'notinst' in TestPO.params()

        ## check caching
        assert param.Parameterized.params() is param.Parameterized().params(), "Results of params() should be cached." # just for performance reasons

    def test_state_saving(self):
        t = TestPO(dyn=_SomeRandomNumbers())
        g = t.get_value_generator('dyn')
        g._Dynamic_time_fn=None
        assert t.dyn!=t.dyn
        orig = t.dyn
        t.state_push()
        t.dyn
        assert t.inspect_value('dyn')!=orig
        t.state_pop()
        assert t.inspect_value('dyn')==orig

from param import parameterized

@nottest
class some_fn(param.ParameterizedFunction):
   num_phase = param.Number(18)
   frequencies = param.List([99])
   scale = param.Number(0.3)

   def __call__(self,**params_to_override):
       params = parameterized.ParamOverrides(self,params_to_override)
       num_phase = params['num_phase']
       frequencies = params['frequencies']
       scale = params['scale']
       return scale,num_phase,frequencies

instance = some_fn.instance()

@istest
class TestParameterizedFunction(unittest.TestCase):

    def _basic_tests(self,fn):
        self.assertEqual(fn(),(0.3,18,[99]))
        self.assertEqual(fn(frequencies=[1,2,3]),(0.3,18,[1,2,3]))
        self.assertEqual(fn(),(0.3,18,[99]))

        fn.frequencies=[10,20,30]
        self.assertEqual(fn(frequencies=[1,2,3]),(0.3,18,[1,2,3]))
        self.assertEqual(fn(),(0.3,18,[10,20,30]))

    def test_parameterized_function(self):
        self._basic_tests(some_fn)

    def test_parameterized_function_instance(self):
        self._basic_tests(instance)

    def test_pickle_instance(self):
        import pickle
        s = pickle.dumps(instance)
        instance.scale=0.8
        i = pickle.loads(s)
        self.assertEqual(i(),(0.3,18,[10,20,30]))

@nottest
class TestPO1(param.Parameterized):
    x = param.Number(default=numbergen.UniformRandom(lbound=-1,ubound=1,seed=1),bounds=(-1,1))
    y = param.Number(default=1,bounds=(-1,1))

@istest
class TestNumberParameter(unittest.TestCase):

    def test_outside_bounds(self):
        t1 = TestPO1()
        # Test bounds (non-dynamic number)
        try:
            t1.y = 10
        except ValueError:
            pass
        else:
            assert False, "Should raise ValueError."

    def test_outside_bounds_numbergen(self):
        t1 = TestPO1()
        # Test bounds (dynamic number)
        t1.x = numbergen.UniformRandom(lbound=2,ubound=3)  # bounds not checked on set
        try:
            v1 = t1.x
        except ValueError:
            pass
        else:
            assert False, "Should raise ValueError."

@istest
class TestParamOverrides(unittest.TestCase):

    def setUp(self):
        super(TestParamOverrides, self).setUp()
        self.po = param.Parameterized(name='A',print_level=0)

    def test_init_name(self):
        self.assertEqual(self.po.name, 'A')

    def test_simple_override(self):
        overrides = ParamOverrides(self.po,{'name':'B'})
        self.assertEqual(overrides['name'], 'B')
        self.assertEqual(overrides['print_level'], 0)

    def test_missing_key(self):
        overrides = ParamOverrides(self.po,{'name':'B'})
        try:
            overrides['doesnotexist']
        except AttributeError:
            pass
        except:
            raise AssertionError("ParamOverrides should give AttributeError when key can't be found.")
        else:
            raise AssertionError("Test supposed to lookup non-existent attribute and raise error.")

if __name__ == "__main__":
    import nose
    nose.runmodule()
<|endoftext|>


<commit_before>from pyramid.settings import asbool

from pyesgf.search import SearchConnection
from pyesgf.search.consts import TYPE_DATASET, TYPE_AGGREGATION, TYPE_FILE

import logging
LOGGER = logging.getLogger(__name__)

def date_from_filename(filename):
    """Example cordex:
    tas_EUR-44i_ECMWF-ERAINT_evaluation_r1i1p1_HMS-ALADIN52_v1_mon_200101-200812.nc
    """
    LOGGER.debug('filename=%s', filename)
    result = None
    value = filename.split('.')
    value.pop()  # remove .nc
    value = value[-1]  # part with date
    value = value.split('_')[-1]  # only date part
    LOGGER.debug('date part = %s', value)
    if value != 'fx':
        value = value.split('-')  # split start-end
        start_year = int(value[0][:4])  # keep only the year
        end_year = int(value[1][:4])
        result = (start_year, end_year)
    return result

def variable_filter(constraints, variables):
    """return True if variable fulfills contraints"""
    var_types = ['variable', 'cf_standard_name', 'variable_long_name']

    success = True
    cs = constraints.mixed()
    # check different types of variables
    for var_type in var_types:
        # is there a constrain for this variable type?
        if var_type in cs:
            # at least one variable constraint must be fulfilled
            success = False
            # do we have this variable type?
            if var_type in variables:
                # do we have an allowed value?
                allowed_values = cs.get(var_type)
                if variables[var_type] in allowed_values:
                    # if one variables matches then we are ok
                    return True
    return success

def temporal_filter(filename, start=None, end=None):
    """return True if file is in timerange start/end"""
    # TODO: keep fixed fields fx ... see esgsearch.js
    """
    // fixed fields are always in time range
    if ($.inArray("fx", doc.time_frequency) >= 0) {
    return true;
    }
    """

    LOGGER.debug('filename=%s, start_date=%s, end_date=%s', filename, start, end)

    if start is None or end is None:
        return True
    start_end = date_from_filename(filename)
    if start_end is None:  # fixed field
        return True
    start_year = start_end[0]
    end_year = start_end[1]
    if end and start_year > end:
        LOGGER.debug('skip: %s > %s', start_year, end)
        return False
    if start and end_year < start:
        LOGGER.debug('skip: %s < %s', end_year, start)
        return False
    LOGGER.debug("pass: %s", filename)
    return True

class ESGFSearch(object):
    def __init__(self, request):
        self.request = request
        self.parse_params()
        settings = self.request.registry.settings
        self.conn = SearchConnection(settings.get('esgfsearch.url'), distrib=self.distrib)

    def parse_params(self):
        self.query = self.request.params.get('query', '')
        self.selected = self.request.params.get('selected', 'project')
        self.limit = int(self.request.params.get('limit', '0'))
        self.distrib = asbool(self.request.params.get('distrib', 'false'))
        self.latest = asbool(self.request.params.get('latest', 'true'))
        if self.latest is False:
            self.latest = None  # all versions
        self.replica = asbool(self.request.params.get('replica', 'false'))
        if self.replica is True:
            self.replica = None  # master + replica
        if 'start' in self.request.params and 'end' in self.request.params:
            self.temporal = True
            self.start = int(self.request.params['start'])
            self.end = int(self.request.params['end'])
        else:
            self.temporal = False
            self.start = self.end = None
        self.constraints = self.request.params.get('constraints')

    def query_params(self):
        return dict(
            query=self.query or '',
            selected=self.selected,
            distrib=str(self.distrib).lower(),
            replica=str(self.replica).lower(),
            latest=str(self.latest).lower(),
            temporal=str(self.temporal).lower(),
            start=self.start or 2001,
            end=self.end or 2005,
            constraints=self.constraints,
        )

    def search_files(self):
        dataset_id = self.request.params.get(
            'dataset_id',
            'cmip5.output1.MPI-M.MPI-ESM-LR.1pctCO2.day.atmos.cfDay.r1i1p1.v20120314|esgf1.dkrz.de')
        ctx = self.conn.new_context(search_type=TYPE_FILE, latest=self.latest, replica=self.replica)
        ctx = ctx.constrain(dataset_id=dataset_id)
        paged_results = []
        for result in ctx.search():
            LOGGER.debug("check: %s", result.filename)
            if temporal_filter(result.filename, self.start, self.end):
                paged_results.append(dict(
                    filename=result.filename,
                    download_url=result.download_url,
                    opendap_url=result.opendap_url,
                ))
        return dict(files=paged_results)

    def search_datasets(self):
        constraints = dict()
        if self.constraints:
            for constrain in self.constraints.split(','):
                if constrain.strip():
                    key, value = constrain.split(':', 1)
                    constraints[key] = value
        ctx = self.conn.new_context(search_type=TYPE_DATASET, latest=self.latest, replica=self.replica)
        ctx = ctx.constrain(**constraints)
        if self.temporal:
            ctx = ctx.constrain(
                from_timestamp="{}-01-01T12:00:00Z".format(self.start),
                to_timestamp="{}-12-31T12:00:00Z".format(self.end))
        results = ctx.search(batch_size=10, ignore_facet_check=False)
        categories = sorted([tag for tag in ctx.facet_counts if len(ctx.facet_counts[tag]) > 1])
        keywords = sorted(ctx.facet_counts[self.selected].keys())
        pinned_facets = []
        for facet in ctx.facet_counts:
            if len(ctx.facet_counts[facet]) == 1:
                pinned_facets.append("{}:{}".format(facet, ctx.facet_counts[facet].keys()[0]))
        pinned_facets = sorted(pinned_facets)
        paged_results = []
        for i in range(0, min(10, ctx.hit_count)):
            paged_results.append(dict(
                id=results[i].json['master_id'],
                title=results[i].json['title'],
                dataset_id=results[i].dataset_id,
                number_of_files=results[i].number_of_files,
                catalog_url=results[i].urls['THREDDS'][0][0]))
        return dict(
            hit_count=ctx.hit_count,
            categories=','.join(categories),
            keywords=','.join(keywords),
            pinned_facets=','.join(pinned_facets),
            results=paged_results)
<commit_msg>remove constraints from pinned_facets<commit_after>from pyramid.settings import asbool

from pyesgf.search import SearchConnection
from pyesgf.search.consts import TYPE_DATASET, TYPE_AGGREGATION, TYPE_FILE

import logging
LOGGER = logging.getLogger(__name__)

def date_from_filename(filename):
    """Example cordex:
    tas_EUR-44i_ECMWF-ERAINT_evaluation_r1i1p1_HMS-ALADIN52_v1_mon_200101-200812.nc
    """
    LOGGER.debug('filename=%s', filename)
    result = None
    value = filename.split('.')
    value.pop()  # remove .nc
    value = value[-1]  # part with date
    value = value.split('_')[-1]  # only date part
    LOGGER.debug('date part = %s', value)
    if value != 'fx':
        value = value.split('-')  # split start-end
        start_year = int(value[0][:4])  # keep only the year
        end_year = int(value[1][:4])
        result = (start_year, end_year)
    return result

def variable_filter(constraints, variables):
    """return True if variable fulfills contraints"""
    var_types = ['variable', 'cf_standard_name', 'variable_long_name']

    success = True
    cs = constraints.mixed()
    # check different types of variables
    for var_type in var_types:
        # is there a constrain for this variable type?
        if var_type in cs:
            # at least one variable constraint must be fulfilled
            success = False
            # do we have this variable type?
            if var_type in variables:
                # do we have an allowed value?
                allowed_values = cs.get(var_type)
                if variables[var_type] in allowed_values:
                    # if one variables matches then we are ok
                    return True
    return success

def temporal_filter(filename, start=None, end=None):
    """return True if file is in timerange start/end"""
    # TODO: keep fixed fields fx ... see esgsearch.js
    """
    // fixed fields are always in time range
    if ($.inArray("fx", doc.time_frequency) >= 0) {
    return true;
    }
    """

    LOGGER.debug('filename=%s, start_date=%s, end_date=%s', filename, start, end)

    if start is None or end is None:
        return True
    start_end = date_from_filename(filename)
    if start_end is None:  # fixed field
        return True
    start_year = start_end[0]
    end_year = start_end[1]
    if end and start_year > end:
        LOGGER.debug('skip: %s > %s', start_year, end)
        return False
    if start and end_year < start:
        LOGGER.debug('skip: %s < %s', end_year, start)
        return False
    LOGGER.debug("pass: %s", filename)
    return True

class ESGFSearch(object):
    def __init__(self, request):
        self.request = request
        self.parse_params()
        settings = self.request.registry.settings
        self.conn = SearchConnection(settings.get('esgfsearch.url'), distrib=self.distrib)

    def parse_params(self):
        self.query = self.request.params.get('query', '')
        self.selected = self.request.params.get('selected', 'project')
        self.limit = int(self.request.params.get('limit', '0'))
        self.distrib = asbool(self.request.params.get('distrib', 'false'))
        self.latest = asbool(self.request.params.get('latest', 'true'))
        if self.latest is False:
            self.latest = None  # all versions
        self.replica = asbool(self.request.params.get('replica', 'false'))
        if self.replica is True:
            self.replica = None  # master + replica
        if 'start' in self.request.params and 'end' in self.request.params:
            self.temporal = True
            self.start = int(self.request.params['start'])
            self.end = int(self.request.params['end'])
        else:
            self.temporal = False
            self.start = self.end = None
        self.constraints = self.request.params.get('constraints')

    def query_params(self):
        return dict(
            query=self.query or '',
            selected=self.selected,
            distrib=str(self.distrib).lower(),
            replica=str(self.replica).lower(),
            latest=str(self.latest).lower(),
            temporal=str(self.temporal).lower(),
            start=self.start or 2001,
            end=self.end or 2005,
            constraints=self.constraints,
        )

    def search_files(self):
        dataset_id = self.request.params.get(
            'dataset_id',
            'cmip5.output1.MPI-M.MPI-ESM-LR.1pctCO2.day.atmos.cfDay.r1i1p1.v20120314|esgf1.dkrz.de')
        ctx = self.conn.new_context(search_type=TYPE_FILE, latest=self.latest, replica=self.replica)
        ctx = ctx.constrain(dataset_id=dataset_id)
        paged_results = []
        for result in ctx.search():
            LOGGER.debug("check: %s", result.filename)
            if temporal_filter(result.filename, self.start, self.end):
                paged_results.append(dict(
                    filename=result.filename,
                    download_url=result.download_url,
                    opendap_url=result.opendap_url,
                ))
        return dict(files=paged_results)

    def search_datasets(self):
        constraints = dict()
        if self.constraints:
            for constrain in self.constraints.split(','):
                if constrain.strip():
                    key, value = constrain.split(':', 1)
                    constraints[key] = value
        ctx = self.conn.new_context(search_type=TYPE_DATASET, latest=self.latest, replica=self.replica)
        ctx = ctx.constrain(**constraints)
        if self.temporal:
            ctx = ctx.constrain(
                from_timestamp="{}-01-01T12:00:00Z".format(self.start),
                to_timestamp="{}-12-31T12:00:00Z".format(self.end))
        results = ctx.search(batch_size=10, ignore_facet_check=False)
        categories = sorted([tag for tag in ctx.facet_counts if len(ctx.facet_counts[tag]) > 1])
        keywords = sorted(ctx.facet_counts[self.selected].keys())
        pinned_facets = []
        for facet in ctx.facet_counts:
            if facet not in constraints and len(ctx.facet_counts[facet]) == 1:
                pinned_facets.append("{}:{}".format(facet, ctx.facet_counts[facet].keys()[0]))
        pinned_facets = sorted(pinned_facets)
        paged_results = []
        for i in range(0, min(10, ctx.hit_count)):
            paged_results.append(dict(
                id=results[i].json['master_id'],
                title=results[i].json['title'],
                dataset_id=results[i].dataset_id,
                number_of_files=results[i].number_of_files,
                catalog_url=results[i].urls['THREDDS'][0][0]))
        return dict(
            hit_count=ctx.hit_count,
            categories=','.join(categories),
            keywords=','.join(keywords),
            pinned_facets=','.join(pinned_facets),
            results=paged_results)
<|endoftext|>


<commit_before>from ceramic_forms import Form

class Route:
    def __init__(
        self,
        handler=None,
        permissions=(),
        handles_subtree=False,
        httpexception_handlers=dict(),
        routemap=dict(),
    ):
        self.routemap = routemap
        self.permissions = permissions
        self.handles_subtree = handles_subtree
        self.handler = handler
        self.depends_on = [handler] if handler is not None else []
        self.exc_handlers = getattr(self, 'httpexception_handlers', {})
        self.exc_handlers.update(httpexception_handlers)
        self.depends_on += list(self.exc_handlers.values())

    def __call__(self, component=None):
        self.component = component
        return self

    def get_view(self, request):
        return getattr(self.component, request.method, None)

    def values(self):
        return self.routemap.values()

    def _defining_attributes(self):
        return (
            self.handler,
            self.permissions,
            self.handles_subtree,
            self.exc_handlers
        )

    def __add__(self, routemap):
        return Route(*self._defining_attributes() + (routemap,))

    def __eq__(self, other):
        return self._defining_attributes() == other._defining_attributes()

    def __hash__(self):
        return hash((tuple(self.depends_on), self.handles_subtree, self.permissions))

    def __setitem__(self, *args, **kwargs):
        return self.routemap.__setitem__(*args, **kwargs)

    def __getitem__(self, *args, **kwargs):
        return self.routemap.__getitem__(*args, **kwargs)

    def __contains__(self, key):
        return self.routemap.__contains__(key)

    def __iter__(self):
        for key in self.routemap:
            yield key

    def __repr__(self):
        return "<Route {}>".format(id(self))

def _match_pathpart(routemap, part):
    #wrap schema and pathpart in a list because Form cannot handle naked values.
    subject = [part]
    for key in routemap:
        validator = Form([key])
        if validator.validate(subject):
            return validator.cleaned[0], routemap[key]

def _match_routes(path, routemap):
    matched = [('/', routemap)]
    for part in path:
        match = _match_pathpart(routemap, part)
        if match is None:
            if routemap.handles_subtree == False:
                return 404
        else:
            part, routemap = match
        matched.append((part, routemap))
    return matched

class RouteApi:
    def __init__(self, request, routemap):
        self.routemap = routemap
        path = [p for p in request.path.split('/') if len(p)]
        self._matched_routes = _match_routes(path, routemap)

    @property
    def path(self):
        return [p for p, _ in self._matched_routes[1:]]

    @property
    def vars(self):
        routemap = self.routemap
        vars = []
        for part, route in self._matched_routes:
            if part in routemap:
                routemap = route
            elif route != routemap:
                vars.append(part)
                routemap = route
        return tuple(vars)

    @property
    def relative(self):
        routemap = self.routemap
        relative_paths = []
        current_relative = []
        for part, route in self._matched_routes:
            if route == routemap and route.handles_subtree:
                current_relative.append(part)
            else:
                routemap = route
                if current_relative:
                    relative_paths.append(tuple(current_relative))
                    current_relative = []
        if current_relative:
            relative_paths.append(tuple(current_relative))
        return tuple(relative_paths)

    @property
    def route(self):
        return self._matched_routes[-1][1]

    @property
    def routes(self):
        visited = set()
        return [r for _, r in self._matched_routes\
                    if r not in visited and not visited.add(r)]

def _routeset(routemap, routeset):
    for value in routemap.values():
        routeset.add(value)
        _routeset(value, routeset)
    return routeset

def routeset(routemap):
    routeset = set()
    routeset.add(routemap)
    return _routeset(routemap, routeset)
<commit_msg>removed adding exception handlers to Route dependencies<commit_after>from ceramic_forms import Form

class Route:
    def __init__(
        self,
        handler=None,
        permissions=(),
        handles_subtree=False,
        httpexception_handlers=dict(),
        routemap=dict(),
    ):
        self.routemap = routemap
        self.permissions = permissions
        self.handles_subtree = handles_subtree
        self.handler = handler
        self.depends_on = [handler] if handler is not None else []
        self.exc_handlers = getattr(self, 'httpexception_handlers', {})
        self.exc_handlers.update(httpexception_handlers)

    def __call__(self, component=None):
        self.component = component
        return self

    def get_view(self, request):
        return getattr(self.component, request.method, None)

    def values(self):
        return self.routemap.values()

    def _defining_attributes(self):
        return (
            self.handler,
            self.permissions,
            self.handles_subtree,
            self.exc_handlers
        )

    def __add__(self, routemap):
        return Route(*self._defining_attributes() + (routemap,))

    def __eq__(self, other):
        return self._defining_attributes() == other._defining_attributes()

    def __hash__(self):
        return hash((tuple(self.depends_on), self.handles_subtree, self.permissions))

    def __setitem__(self, *args, **kwargs):
        return self.routemap.__setitem__(*args, **kwargs)

    def __getitem__(self, *args, **kwargs):
        return self.routemap.__getitem__(*args, **kwargs)

    def __contains__(self, key):
        return self.routemap.__contains__(key)

    def __iter__(self):
        for key in self.routemap:
            yield key

    def __repr__(self):
        return "<Route {}>".format(id(self))

def _match_pathpart(routemap, part):
    #wrap schema and pathpart in a list because Form cannot handle naked values.
    subject = [part]
    for key in routemap:
        validator = Form([key])
        if validator.validate(subject):
            return validator.cleaned[0], routemap[key]

def _match_routes(path, routemap):
    matched = [('/', routemap)]
    for part in path:
        match = _match_pathpart(routemap, part)
        if match is None:
            if routemap.handles_subtree == False:
                return 404
        else:
            part, routemap = match
        matched.append((part, routemap))
    return matched

class RouteApi:
    def __init__(self, request, routemap):
        self.routemap = routemap
        path = [p for p in request.path.split('/') if len(p)]
        self._matched_routes = _match_routes(path, routemap)

    @property
    def path(self):
        return [p for p, _ in self._matched_routes[1:]]

    @property
    def vars(self):
        routemap = self.routemap
        vars = []
        for part, route in self._matched_routes:
            if part in routemap:
                routemap = route
            elif route != routemap:
                vars.append(part)
                routemap = route
        return tuple(vars)

    @property
    def relative(self):
        routemap = self.routemap
        relative_paths = []
        current_relative = []
        for part, route in self._matched_routes:
            if route == routemap and route.handles_subtree:
                current_relative.append(part)
            else:
                routemap = route
                if current_relative:
                    relative_paths.append(tuple(current_relative))
                    current_relative = []
        if current_relative:
            relative_paths.append(tuple(current_relative))
        return tuple(relative_paths)

    @property
    def route(self):
        return self._matched_routes[-1][1]

    @property
    def routes(self):
        visited = set()
        return [r for _, r in self._matched_routes\
                    if r not in visited and not visited.add(r)]

def _routeset(routemap, routeset):
    for value in routemap.values():
        routeset.add(value)
        _routeset(value, routeset)
    return routeset

def routeset(routemap):
    routeset = set()
    routeset.add(routemap)
    return _routeset(routemap, routeset)
<|endoftext|>


<commit_before>"""Python currencies"""
from .core.currency import (
    currency, currencydb, ccypair, currency_pair,
    dump_currency_table
)
from .core.country import (
    country, countryccy, set_new_country,
    countries, set_country_map, country_map,
    CountryError, eurozone, print_eurozone
)
from .core.daycounter import *     # noqa
from .dates.converters import (
    todate, date2timestamp, timestamp2date, yyyymmdd2date,
    date2yyyymmdd, juldate2date, date2juldate, date_from_string,
    jstimestamp
)
from .dates.futures import future_date_to_code, future_month_dict
from .dates.period import period, Period

__all__ = [
    'currency', 'currencydb', 'ccypair', 'currency_pair',
    'dump_currency_table',
    #
    'country', 'countryccy', 'set_new_country',
    'countries', 'set_country_map', 'country_map',
    'CountryError', 'eurozone', 'print_eurozone',
    'future_date_to_code', 'future_month_dict',
    'period', 'Period',
    'todate', 'date2timestamp', 'timestamp2date',
    'yyyymmdd2date', 'date2yyyymmdd', 'juldate2date',
    'date2juldate', 'date_from_string', 'jstimestamp'
]

# Shortcuts
def cross(code):
    return currency(code).as_cross()

def crossover(code):
    return currency(code).as_cross('/')

def all():
    return currencydb().keys()

def g7():
    return ['EUR', 'GBP', 'USD', 'CAD']

def g10():
    return g7() + ['CHF', 'SEK', 'JPY']

def g10m():
    """modified g10 = G10 + AUD, NZD, NOK
    """
    return g10() + ['AUD', 'NZD', 'NOK']
<commit_msg>removed wild imports<commit_after>"""Python currencies"""
from .core.currency import (
    currency, currencydb, ccypair, currency_pair,
    dump_currency_table
)
from .core.country import (
    country, countryccy, set_new_country,
    countries, set_country_map, country_map,
    CountryError, eurozone, print_eurozone
)
from .core.daycounter import getdc, ActActYears, alldc
from .dates.converters import (
    todate, date2timestamp, timestamp2date, yyyymmdd2date,
    date2yyyymmdd, juldate2date, date2juldate, date_from_string,
    jstimestamp
)
from .dates.futures import future_date_to_code, future_month_dict
from .dates.period import period, Period

__all__ = [
    'currency', 'currencydb', 'ccypair', 'currency_pair',
    'dump_currency_table',
    #
    'getdc', 'ActActYears', 'alldc',
    #
    'country', 'countryccy', 'set_new_country',
    'countries', 'set_country_map', 'country_map',
    'CountryError', 'eurozone', 'print_eurozone',
    'future_date_to_code', 'future_month_dict',
    'period', 'Period',
    'todate', 'date2timestamp', 'timestamp2date',
    'yyyymmdd2date', 'date2yyyymmdd', 'juldate2date',
    'date2juldate', 'date_from_string', 'jstimestamp'
]

# Shortcuts
def cross(code):
    return currency(code).as_cross()

def crossover(code):
    return currency(code).as_cross('/')

def all():
    return currencydb().keys()

def g7():
    return ['EUR', 'GBP', 'USD', 'CAD']

def g10():
    return g7() + ['CHF', 'SEK', 'JPY']

def g10m():
    """modified g10 = G10 + AUD, NZD, NOK
    """
    return g10() + ['AUD', 'NZD', 'NOK']
<|endoftext|>


<commit_before>#
# Copyright (c) 2005-2007 rPath, Inc.
#
# All Rights Reserved
#

import sys

validBuildTypes = {
    'BOOTABLE_IMAGE'    : 0,
    'INSTALLABLE_ISO'   : 1,
    'STUB_IMAGE'        : 2,
    'RAW_FS_IMAGE'      : 3,
    'NETBOOT_IMAGE'     : 4,
    'TARBALL'           : 5,
    'LIVE_ISO'          : 6,
    'RAW_HD_IMAGE'      : 7,
    'VMWARE_IMAGE'      : 8,
    'VMWARE_ESX_IMAGE'  : 9,
    'VIRTUAL_PC_IMAGE'  : 10,
    'XEN_OVA'           : 11,
    'VIRTUAL_IRON'      : 12,
    'PARALLELS'         : 13,
    'AMI'               : 14,
}

TYPES = validBuildTypes.values()

# add all the defined image types directly to the module so that the standard
# approach of "buildtypes.IMAGE_TYPE" will result in the expected enum
sys.modules[__name__].__dict__.update(validBuildTypes)

deprecatedBuildTypes = {
    'QEMU_IMAGE' : RAW_HD_IMAGE
    }

#
# These are identifying pieces of information that we can extract from the
# flavor of a build, but not necessarily tied to any particular build type.
#
# These can sometimes be used as a buildType, indexes starting at 100.
#
flavorFlags = {
    'XEN_DOMU':     100,
    'APPLIANCE':    101,
}

FLAG_TYPES = flavorFlags.values()

flavorFlagsFromId = dict((x[1], x[0]) for x in flavorFlags.items())

sys.modules[__name__].__dict__.update(flavorFlags)

flavorFlagFlavors = {
    XEN_DOMU:   "use: xen, domU",
    APPLIANCE:  "use: appliance",
}

flavorFlagNames = {
    XEN_DOMU:   "Xen Virtual Appliance",
    APPLIANCE:  "Appliance",
}

#BOOTABLE_IMAGE Should never get stored in the DB and therefore doesn't need a name

# NOTA BENE: Using Latin-1 here is harmful to XML-RPC which expects UTF-8
# Until we figure out the root cause, use "(R)" for registered trademark here.

typeNames = {
    NETBOOT_IMAGE:      "Netboot Image",
    INSTALLABLE_ISO:    "Installable CD/DVD",
    RAW_FS_IMAGE:       "Raw Filesystem Image",
    STUB_IMAGE:         "Stub Image",
    RAW_HD_IMAGE:       "Raw Hard Disk Image",
    VMWARE_IMAGE:       "VMware (R) Virtual Appliance",
    VMWARE_ESX_IMAGE:   "VMware (R) ESX Server Virtual Appliance",
    LIVE_ISO:           "Demo CD/DVD (Live CD/DVD)",
    TARBALL:            "Compressed Tar File",
    VIRTUAL_PC_IMAGE:   "Microsoft (R) VHD Virtual Appliance",
    XEN_OVA:            "Xen Enterprise Virtual Appliance",
    VIRTUAL_IRON:       "Virtual Iron Virtual Appliance",
    PARALLELS:          "Parallels Virtual Appliance",
    AMI:                "Amazon Machine Image",
}

typeNamesShort = {
    NETBOOT_IMAGE:      "Netboot",
    INSTALLABLE_ISO:    "Inst CD/DVD",
    RAW_FS_IMAGE:       "Raw FS",
    STUB_IMAGE:         "Stub",
    RAW_HD_IMAGE:       "HDD",
    VMWARE_IMAGE:       "VMware (R)",
    VMWARE_ESX_IMAGE:   "VMware (R) ESX",
    LIVE_ISO:           "Demo CD/DVD",
    TARBALL:            "Tar",
    VIRTUAL_PC_IMAGE:   "Virtual Server",
    XEN_OVA:            "XVA",
    VIRTUAL_IRON:       "Virtual Iron",
    PARALLELS:          "Parallels",
    AMI:                "AMI",
}

typeNamesMarketing = {
    NETBOOT_IMAGE:      "Netboot Image",
    INSTALLABLE_ISO:    "Installable CD/DVD",
    RAW_FS_IMAGE:       "Mountable Filesystem",
    STUB_IMAGE:         "Stub Image",
    RAW_HD_IMAGE:       "Parallels, QEMU (Raw Hard Disk)",
    VMWARE_IMAGE:       "VMware (R) Virtual Appliance",
    VMWARE_ESX_IMAGE:   "VMware (R) ESX Server Virtual Appliance",
    LIVE_ISO:           "Demo CD/DVD (Live CD/DVD)",
    TARBALL:            "TAR File",
    VIRTUAL_PC_IMAGE:   "Microsoft (R) VHD Virtual Server",
    XEN_OVA:            "Xen Enterprise Virtual Appliance",
    VIRTUAL_IRON:       "Virtual Iron Virtual Appliance",
    PARALLELS:          "Parallels Virtual Appliance",
    AMI:                "Amazon Machine Image",

    # flavor flags here
    XEN_DOMU:           "Xen DomU",
    APPLIANCE:          "Appliance",
}

# sizes are listed in bytes...
discSizes = {
    'CD: 650 MB'  : '681574400',
    'CD: 700 MB'  : '734003200',
    'DVD: 4.7 GB' : '4700000000',
    'DVD: 8.5 GB' : '8500000000',
    }

# code generator run by make to generate javascript constants
# should only be run by the makefile in mint/web/content/javascript
def codegen():
    s = "// this Javascript was generated by mint/buildtypes.py\n"
    s += "// do not edit or check into source control\n"

    s += "var maxBuildType = %d;" % max(validBuildTypes.values())

    s += "var buildTypeNames = {"
    i = []
    for k, v in typeNames.items():
        i.append("    '%d':  '%s'" % (k, v,))
    s += ", ".join(i)
    s += "};"

    s += "var buildTypeNamesShort = {"
    i = []
    for k, v in typeNamesShort.items():
        i.append("    '%d':  '%s'" % (k, v,))
    s += ", ".join(i)
    s += "};"

    s += "var buildTypeNamesMarketing = {"
    i = []
    for k, v in typeNamesMarketing.items():
        i.append("    '%d':  '%s'" % (k, v,))
    s += ", ".join(i)
    s += "};"

    for k, v in validBuildTypes.items():
        s += "%s = %d;\n" % (k, v)
    return s

if __name__ == "__main__": #pragma: no cover
    if len(sys.argv) > 1 and sys.argv[1] == "--genjs":
        print codegen()
        sys.exit(0)
    else:
        sys.exit(1)

<commit_msg>define Update ISO<commit_after>#
# Copyright (c) 2005-2007 rPath, Inc.
#
# All Rights Reserved
#

import sys

validBuildTypes = {
    'BOOTABLE_IMAGE'    : 0,
    'INSTALLABLE_ISO'   : 1,
    'STUB_IMAGE'        : 2,
    'RAW_FS_IMAGE'      : 3,
    'NETBOOT_IMAGE'     : 4,
    'TARBALL'           : 5,
    'LIVE_ISO'          : 6,
    'RAW_HD_IMAGE'      : 7,
    'VMWARE_IMAGE'      : 8,
    'VMWARE_ESX_IMAGE'  : 9,
    'VIRTUAL_PC_IMAGE'  : 10,
    'XEN_OVA'           : 11,
    'VIRTUAL_IRON'      : 12,
    'PARALLELS'         : 13,
    'AMI'               : 14,
    'UPDATE_ISO'        : 15,
}

TYPES = validBuildTypes.values()

# add all the defined image types directly to the module so that the standard
# approach of "buildtypes.IMAGE_TYPE" will result in the expected enum
sys.modules[__name__].__dict__.update(validBuildTypes)

deprecatedBuildTypes = {
    'QEMU_IMAGE' : RAW_HD_IMAGE
    }

#
# These are identifying pieces of information that we can extract from the
# flavor of a build, but not necessarily tied to any particular build type.
#
# These can sometimes be used as a buildType, indexes starting at 100.
#
flavorFlags = {
    'XEN_DOMU':     100,
    'APPLIANCE':    101,
}

FLAG_TYPES = flavorFlags.values()

flavorFlagsFromId = dict((x[1], x[0]) for x in flavorFlags.items())

sys.modules[__name__].__dict__.update(flavorFlags)

flavorFlagFlavors = {
    XEN_DOMU:   "use: xen, domU",
    APPLIANCE:  "use: appliance",
}

flavorFlagNames = {
    XEN_DOMU:   "Xen Virtual Appliance",
    APPLIANCE:  "Appliance",
}

#BOOTABLE_IMAGE Should never get stored in the DB and therefore doesn't need a name

# NOTA BENE: Using Latin-1 here is harmful to XML-RPC which expects UTF-8
# Until we figure out the root cause, use "(R)" for registered trademark here.

typeNames = {
    NETBOOT_IMAGE:      "Netboot Image",
    INSTALLABLE_ISO:    "Installable CD/DVD",
    RAW_FS_IMAGE:       "Raw Filesystem Image",
    STUB_IMAGE:         "Stub Image",
    RAW_HD_IMAGE:       "Raw Hard Disk Image",
    VMWARE_IMAGE:       "VMware (R) Virtual Appliance",
    VMWARE_ESX_IMAGE:   "VMware (R) ESX Server Virtual Appliance",
    LIVE_ISO:           "Demo CD/DVD (Live CD/DVD)",
    TARBALL:            "Compressed Tar File",
    VIRTUAL_PC_IMAGE:   "Microsoft (R) VHD Virtual Appliance",
    XEN_OVA:            "Xen Enterprise Virtual Appliance",
    VIRTUAL_IRON:       "Virtual Iron Virtual Appliance",
    PARALLELS:          "Parallels Virtual Appliance",
    AMI:                "Amazon Machine Image",
    UPDATE_ISO:         "Update CD/DVD",
}

typeNamesShort = {
    NETBOOT_IMAGE:      "Netboot",
    INSTALLABLE_ISO:    "Inst CD/DVD",
    RAW_FS_IMAGE:       "Raw FS",
    STUB_IMAGE:         "Stub",
    RAW_HD_IMAGE:       "HDD",
    VMWARE_IMAGE:       "VMware (R)",
    VMWARE_ESX_IMAGE:   "VMware (R) ESX",
    LIVE_ISO:           "Demo CD/DVD",
    TARBALL:            "Tar",
    VIRTUAL_PC_IMAGE:   "Virtual Server",
    XEN_OVA:            "XVA",
    VIRTUAL_IRON:       "Virtual Iron",
    PARALLELS:          "Parallels",
    AMI:                "AMI",
    UPDATE_ISO:         "Update CD/DVD",
}

typeNamesMarketing = {
    NETBOOT_IMAGE:      "Netboot Image",
    INSTALLABLE_ISO:    "Installable CD/DVD",
    RAW_FS_IMAGE:       "Mountable Filesystem",
    STUB_IMAGE:         "Stub Image",
    RAW_HD_IMAGE:       "Parallels, QEMU (Raw Hard Disk)",
    VMWARE_IMAGE:       "VMware (R) Virtual Appliance",
    VMWARE_ESX_IMAGE:   "VMware (R) ESX Server Virtual Appliance",
    LIVE_ISO:           "Demo CD/DVD (Live CD/DVD)",
    TARBALL:            "TAR File",
    VIRTUAL_PC_IMAGE:   "Microsoft (R) VHD Virtual Server",
    XEN_OVA:            "Xen Enterprise Virtual Appliance",
    VIRTUAL_IRON:       "Virtual Iron Virtual Appliance",
    PARALLELS:          "Parallels Virtual Appliance",
    AMI:                "Amazon Machine Image",
    UPDATE_ISO:         "Update CD/DVD",

    # flavor flags here
    XEN_DOMU:           "Xen DomU",
    APPLIANCE:          "Appliance",
}

# sizes are listed in bytes...
discSizes = {
    'CD: 650 MB'  : '681574400',
    'CD: 700 MB'  : '734003200',
    'DVD: 4.7 GB' : '4700000000',
    'DVD: 8.5 GB' : '8500000000',
    }

# code generator run by make to generate javascript constants
# should only be run by the makefile in mint/web/content/javascript
def codegen():
    s = "// this Javascript was generated by mint/buildtypes.py\n"
    s += "// do not edit or check into source control\n"

    s += "var maxBuildType = %d;" % max(validBuildTypes.values())

    s += "var buildTypeNames = {"
    i = []
    for k, v in typeNames.items():
        i.append("    '%d':  '%s'" % (k, v,))
    s += ", ".join(i)
    s += "};"

    s += "var buildTypeNamesShort = {"
    i = []
    for k, v in typeNamesShort.items():
        i.append("    '%d':  '%s'" % (k, v,))
    s += ", ".join(i)
    s += "};"

    s += "var buildTypeNamesMarketing = {"
    i = []
    for k, v in typeNamesMarketing.items():
        i.append("    '%d':  '%s'" % (k, v,))
    s += ", ".join(i)
    s += "};"

    for k, v in validBuildTypes.items():
        s += "%s = %d;\n" % (k, v)
    return s

if __name__ == "__main__": #pragma: no cover
    if len(sys.argv) > 1 and sys.argv[1] == "--genjs":
        print codegen()
        sys.exit(0)
    else:
        sys.exit(1)

<|endoftext|>


<commit_before>"""cell.result"""

from __future__ import absolute_import
from __future__ import with_statement

from kombu.pools import producers

from .exceptions import CellError, NoReplyError

__all__ = ['AsyncResult']

class AsyncResult(object):
    Error = CellError
    NoReplyError = NoReplyError

    def __init__(self, ticket, actor):
        self.ticket = ticket
        self.actor = actor

    def _first(self, replies):
        if replies is not None:
            replies = list(replies)
            if replies:
                return replies[0]
        raise self.NoReplyError('No reply received within time constraint')

    def get(self, **kwargs):
        return self._first(self.gather(**dict(kwargs, limit=1)))

    def gather(self, propagate=True, **kwargs):
        connection = self.actor.connection
        gather = self._gather
        with producers[connection].acquire(block=True) as producer:
            for r in gather(producer.connection, producer.channel, self.ticket,
                            propagate=propagate, **kwargs):
                yield r

    def _gather(self, *args, **kwargs):
        propagate = kwargs.pop('propagate', True)
        return (self.to_python(reply, propagate=propagate)
                    for reply in self.actor._collect_replies(*args, **kwargs))

    def to_python(self, reply, propagate=True):
        try:
            return reply['ok']
        except KeyError:
            error = self.Error(*reply.get('nok') or ())
            if propagate:
                raise error
            return error
<commit_msg>Add result property to AsyncResult (it blocks if the result has not been previously retrieved, or return the result otherwise)<commit_after>"""cell.result"""

from __future__ import absolute_import
from __future__ import with_statement

from kombu.pools import producers

from .exceptions import CellError, NoReplyError

__all__ = ['AsyncResult']

class AsyncResult(object):
    Error = CellError
    NoReplyError = NoReplyError

    def __init__(self, ticket, actor):
        self.ticket = ticket
        self.actor = actor
        self._result = None

    def _first(self, replies):
        if replies is not None:
            replies = list(replies)
            if replies:
                return replies[0]
        raise self.NoReplyError('No reply received within time constraint')

    @property
    def result(self):
        if not self._result:
            self._result = self.get()
        return self.result

    def get(self, **kwargs):
        return self._first(self.gather(**dict(kwargs, limit=1)))

    def gather(self, propagate=True, **kwargs):
        connection = self.actor.connection
        gather = self._gather
        with producers[connection].acquire(block=True) as producer:
            for r in gather(producer.connection, producer.channel, self.ticket,
                            propagate=propagate, **kwargs):
                yield r

    def _gather(self, *args, **kwargs):
        propagate = kwargs.pop('propagate', True)
        return (self.to_python(reply, propagate=propagate)
                    for reply in self.actor._collect_replies(*args, **kwargs))

    def to_python(self, reply, propagate=True):
        try:
            return reply['ok']
        except KeyError:
            error = self.Error(*reply.get('nok') or ())
            if propagate:
                raise error
            return error
<|endoftext|>


<commit_before>
"""
test_measures
-------------
Collections of measures.

"""

import numpy as np
from ..artificial_data.create_artificial_timeseries import create_random_ts,\
    create_random_raster, create_brownian_noise_regular_ts
from ..Measures.hurst_measures import hurst
from ..Measures.hurst_measures import create_RS_scales_sequence,\
    create_aggvar_scales_sequence, create_peng_scales_sequence,\
    create_scales_higuchi_sequence, create_scales_periodogram_sequence
from ..Measures.fractal_dimension_measures import pfd, hfd
from ..Measures.information_theory_measures import entropy, svd_entropy,\
    spectral_entropy, fisher_info, dfa
from ..Measures.measures import hjorth

def test():
    levels_rand, magnitudes_rand = [[(200, .1)]], [1.]
    levels = [[(200, .01)], [(200, 20.), (5, 1.)], [(200, 50.0)],
              [(200, 15.), (10, .2)]]
    magnitudes = [.1, -10., -5., 15.]
    ts_rand, vals_rand = create_random_ts(levels_rand, magnitudes_rand)
    ts, vals = create_random_ts(levels, magnitudes)
    n_times, n_elements, n_regs = 200, 1, 10
    regimes_info = [(n_regs, np.random.random(n_regs))]
    raster = create_random_raster(len(ts), n_elements, regimes_info).squeeze()

    times_rd, rand_ts = create_brownian_noise_regular_ts(n_times)
    normal_rand = np.random.randn(n_times)
    uniform_rand = np.random.random(n_times)

    #### Test measures
    #################################################################

    ## Entropy measure
    mea = entropy(raster)
    assert(type(mea) == float)

    ## Hurst measure
#    T = create_RS_scales_sequence(vals, sequence='complete')
#    R_S, T = hurst_alternative_rs_values(vals, T)
#    mea = general_rs_fit(R_S, T)
#    print '0', mea
#    R_S, T = hurst_rs_values(vals, T)
#    mea = general_rs_fit(R_S, T)
#    print '1', mea
#    T = create_RS_scales_sequence(vals, sequence='power')
#    R_S, T = hurst_rs_values(vals, T)
#    mea = general_rs_fit(R_S, T)
#    print '2', mea
#    T = create_RS_scales_sequence(vals, sequence=T)
#    R_S, T = hurst_rs_values(vals, T)
#    mea = general_rs_fit(R_S, T)
#    print '4', mea

    #### Hurst measure
    ##################
    ## Some failure. Not correct values obtained
    ##################
#    T = create_RS_scales_sequence(rand_ts, sequence='power')
#    T = create_RS_scales_sequence(rand_ts, sequence='complete')
#    T = create_RS_scales_sequence(rand_ts, sequence=T)
#    H = hurst(rand_ts, T, method='RS')
##    print '0', H
#    H = hurst(normal_rand, T, method='RS')
##    print '0', H
#    H = hurst(uniform_rand, T, method='RS')
##    print '0', H
#
#    H = hurst(rand_ts, scales='power', method='RS_alternative')
##    print '1', H
#    H = hurst(normal_rand, scales='power', method='RS_alternative')
##    print '1', H
#    H = hurst(uniform_rand, scales='power', method='RS_alternative')
##    print '1', H
#
#    T = create_aggvar_scales_sequence(rand_ts)
#    T = create_aggvar_scales_sequence(rand_ts, T)
#    H = hurst(rand_ts, T, method='aggvar')
##    print '2', H
#    H = hurst(normal_rand, T, method='aggvar')
##    print '2', H
#    H = hurst(uniform_rand, T, method='aggvar')
##    print '2', H
#
#    T = create_peng_scales_sequence(rand_ts)
#    T = create_peng_scales_sequence(rand_ts, T)
#    H = hurst(rand_ts, T, method='peng')
##    print '3', H
#    H = hurst(normal_rand, T, method='peng')
##    print '3', H
#    H = hurst(uniform_rand, T, method='peng')
##    print '3', H
#
#    T = create_scales_higuchi_sequence(rand_ts)
#    T = create_scales_higuchi_sequence(rand_ts, T)
#    H = hurst(rand_ts, T, method='higuchi')
##    print '4', H
#    H = hurst(normal_rand, T, method='higuchi')
##    print '4', H
#    H = hurst(uniform_rand, T, method='higuchi')
##    print '4', H
#
#    T = create_scales_periodogram_sequence(rand_ts)
#    T = create_scales_periodogram_sequence(rand_ts, T)
#    H = hurst(rand_ts, T, method='per')
##    print '5', H
#    H = hurst(normal_rand, T, method='per')
##    print '5', H
#    H = hurst(uniform_rand, T, method='per')
##    print '5', H

    #### Fractal dimenstion measure
    ###############################
    ## Use fit utilities and generalize fit
    ##################
    pfd(rand_ts)
    pfd(normal_rand)
    pfd(uniform_rand)
    hfd(rand_ts)
    hfd(normal_rand)
    hfd(uniform_rand)

    #### Information theory measure
    ###############################
    ##
    ##################
    entropy(rand_ts)
    entropy(normal_rand)
    entropy(uniform_rand)
    svd_entropy(rand_ts)
    svd_entropy(normal_rand)
    svd_entropy(uniform_rand)
    spectral_entropy(rand_ts)
    spectral_entropy(normal_rand)
    spectral_entropy(uniform_rand)
    fisher_info(rand_ts)
    fisher_info(normal_rand)
    fisher_info(uniform_rand)
    dfa(rand_ts)
    dfa(normal_rand)
    dfa(uniform_rand)

    #### Fractal dimenstion measure
    ###############################
    ## Use fit utilities and generalize fit
    ##################
    hjorth(rand_ts)
    hjorth(normal_rand)
    hjorth(uniform_rand)
<commit_msg>Exploring tests: test measures.<commit_after>
"""
test_measures
-------------
Collections of measures.

"""

import numpy as np
from ..artificial_data.create_artificial_timeseries import create_random_ts,\
    create_random_raster, create_brownian_noise_regular_ts
from ..Measures.hurst_measures import hurst
from ..Measures.hurst_measures import create_RS_scales_sequence,\
    create_aggvar_scales_sequence, create_peng_scales_sequence,\
    create_scales_higuchi_sequence, create_scales_periodogram_sequence
from ..Measures.fractal_dimension_measures import pfd, hfd
from ..Measures.information_theory_measures import entropy, svd_entropy,\
    spectral_entropy, fisher_info, dfa
from ..Measures.measures import hjorth

def test():
    levels_rand, magnitudes_rand = [[(200, .1)]], [1.]
    levels = [[(200, .01)], [(200, 20.), (5, 1.)], [(200, 50.0)],
              [(200, 15.), (10, .2)]]
    magnitudes = [.1, -10., -5., 15.]
    ts_rand, vals_rand = create_random_ts(levels_rand, magnitudes_rand)
    ts, vals = create_random_ts(levels, magnitudes)
    n_times, n_elements, n_regs = 200, 1, 10
    regimes_info = [(n_regs, np.random.random(n_regs))]
    raster = create_random_raster(len(ts), n_elements, regimes_info).squeeze()

    times_rd, rand_ts = create_brownian_noise_regular_ts(n_times)
    normal_rand = np.random.randn(n_times)
    uniform_rand = np.random.random(n_times)

    #### Test measures
    #################################################################

    ## Entropy measure
    mea = entropy(raster)
    assert(type(mea) == float)

    ## Hurst measure
#    T = create_RS_scales_sequence(vals, sequence='complete')
#    R_S, T = hurst_alternative_rs_values(vals, T)
#    mea = general_rs_fit(R_S, T)
#    print '0', mea
#    R_S, T = hurst_rs_values(vals, T)
#    mea = general_rs_fit(R_S, T)
#    print '1', mea
#    T = create_RS_scales_sequence(vals, sequence='power')
#    R_S, T = hurst_rs_values(vals, T)
#    mea = general_rs_fit(R_S, T)
#    print '2', mea
#    T = create_RS_scales_sequence(vals, sequence=T)
#    R_S, T = hurst_rs_values(vals, T)
#    mea = general_rs_fit(R_S, T)
#    print '4', mea

    #### Hurst measure
    ##################
    ## Some failure. Not correct values obtained
    ##################
#    T = create_RS_scales_sequence(rand_ts, sequence='power')
#    T = create_RS_scales_sequence(rand_ts, sequence='complete')
#    T = create_RS_scales_sequence(rand_ts, sequence=T)
#    H = hurst(rand_ts, T, method='RS')
##    print '0', H
#    H = hurst(normal_rand, T, method='RS')
##    print '0', H
#    H = hurst(uniform_rand, T, method='RS')
##    print '0', H
#
#    H = hurst(rand_ts, scales='power', method='RS_alternative')
##    print '1', H
#    H = hurst(normal_rand, scales='power', method='RS_alternative')
##    print '1', H
#    H = hurst(uniform_rand, scales='power', method='RS_alternative')
##    print '1', H
#
#    T = create_aggvar_scales_sequence(rand_ts)
#    T = create_aggvar_scales_sequence(rand_ts, T)
#    H = hurst(rand_ts, T, method='aggvar')
##    print '2', H
#    H = hurst(normal_rand, T, method='aggvar')
##    print '2', H
#    H = hurst(uniform_rand, T, method='aggvar')
##    print '2', H
#
#    T = create_peng_scales_sequence(rand_ts)
#    T = create_peng_scales_sequence(rand_ts, T)
#    H = hurst(rand_ts, T, method='peng')
##    print '3', H
#    H = hurst(normal_rand, T, method='peng')
##    print '3', H
#    H = hurst(uniform_rand, T, method='peng')
##    print '3', H
#
#    T = create_scales_higuchi_sequence(rand_ts)
#    T = create_scales_higuchi_sequence(rand_ts, T)
#    H = hurst(rand_ts, T, method='higuchi')
##    print '4', H
#    H = hurst(normal_rand, T, method='higuchi')
##    print '4', H
#    H = hurst(uniform_rand, T, method='higuchi')
##    print '4', H
#
#    T = create_scales_periodogram_sequence(rand_ts)
#    T = create_scales_periodogram_sequence(rand_ts, T)
#    H = hurst(rand_ts, T, method='per')
##    print '5', H
#    H = hurst(normal_rand, T, method='per')
##    print '5', H
#    H = hurst(uniform_rand, T, method='per')
##    print '5', H
#
#    #### Fractal dimenstion measure
#    ###############################
#    ## Use fit utilities and generalize fit
#    ##################
#    pfd(rand_ts)
#    pfd(normal_rand)
#    pfd(uniform_rand)
#    hfd(rand_ts)
#    hfd(normal_rand)
#    hfd(uniform_rand)
#
#    #### Information theory measure
#    ###############################
#    ##
#    ##################
#    entropy(rand_ts)
#    entropy(normal_rand)
#    entropy(uniform_rand)
#    svd_entropy(rand_ts)
#    svd_entropy(normal_rand)
#    svd_entropy(uniform_rand)
#    spectral_entropy(rand_ts)
#    spectral_entropy(normal_rand)
#    spectral_entropy(uniform_rand)
#    fisher_info(rand_ts)
#    fisher_info(normal_rand)
#    fisher_info(uniform_rand)
#    dfa(rand_ts)
#    dfa(normal_rand)
#    dfa(uniform_rand)
#
#    #### Fractal dimenstion measure
#    ###############################
#    ## Use fit utilities and generalize fit
#    ##################
#    hjorth(rand_ts)
#    hjorth(normal_rand)
#    hjorth(uniform_rand)
<|endoftext|>


<commit_before>import argparse
import os.path
import sys
import dbt.project as project
import dbt.task.run as run_task
import dbt.task.compile as compile_task
import dbt.task.debug as debug_task
import dbt.task.clean as clean_task
import dbt.task.deps as deps_task
import dbt.task.init as init_task

def main(args=None):
    if args is None:
        args = sys.argv[1:]

    p = argparse.ArgumentParser(prog='dbt: data build tool')
    subs = p.add_subparsers()

    base_subparser = argparse.ArgumentParser(add_help=False)
    base_subparser.add_argument('--profile', default=["user"], nargs='+', type=str, help='Which profile to load')

    sub = subs.add_parser('init', parents=[base_subparser])
    sub.add_argument('project_name', type=str, help='Name of the new project')
    sub.set_defaults(cls=init_task.InitTask, which='init')

    sub = subs.add_parser('clean', parents=[base_subparser])
    sub.set_defaults(cls=clean_task.CleanTask, which='clean')

    sub = subs.add_parser('compile', parents=[base_subparser])
    sub.set_defaults(cls=compile_task.CompileTask, which='compile')

    sub = subs.add_parser('debug', parents=[base_subparser])
    sub.set_defaults(cls=debug_task.DebugTask, which='debug')

    sub = subs.add_parser('deps', parents=[base_subparser])
    sub.set_defaults(cls=deps_task.DepsTask, which='deps')

    sub = subs.add_parser('run', parents=[base_subparser])
    sub.set_defaults(cls=run_task.RunTask, which='run')

    parsed = p.parse_args(args)

    if parsed.which == 'init':
        # bypass making a project file if we're running `dbt init`
        parsed.cls(args=parsed).run()

    elif os.path.isfile('dbt_project.yml'):
        proj = project.read_project('dbt_project.yml').with_profiles(parsed.profile)
        parsed.cls(args=parsed, project=proj).run()

    else:
        raise RuntimeError("dbt must be run from a project root directory with a dbt_project.yml file")

<commit_msg>comment<commit_after>import argparse
import os.path
import sys
import dbt.project as project
import dbt.task.run as run_task
import dbt.task.compile as compile_task
import dbt.task.debug as debug_task
import dbt.task.clean as clean_task
import dbt.task.deps as deps_task
import dbt.task.init as init_task

def main(args=None):
    if args is None:
        args = sys.argv[1:]

    p = argparse.ArgumentParser(prog='dbt: data build tool')
    subs = p.add_subparsers()

    base_subparser = argparse.ArgumentParser(add_help=False)
    base_subparser.add_argument('--profile', default=["user"], nargs='+', type=str, help='Which profile to load')

    sub = subs.add_parser('init', parents=[base_subparser])
    sub.add_argument('project_name', type=str, help='Name of the new project')
    sub.set_defaults(cls=init_task.InitTask, which='init')

    sub = subs.add_parser('clean', parents=[base_subparser])
    sub.set_defaults(cls=clean_task.CleanTask, which='clean')

    sub = subs.add_parser('compile', parents=[base_subparser])
    sub.set_defaults(cls=compile_task.CompileTask, which='compile')

    sub = subs.add_parser('debug', parents=[base_subparser])
    sub.set_defaults(cls=debug_task.DebugTask, which='debug')

    sub = subs.add_parser('deps', parents=[base_subparser])
    sub.set_defaults(cls=deps_task.DepsTask, which='deps')

    sub = subs.add_parser('run', parents=[base_subparser])
    sub.set_defaults(cls=run_task.RunTask, which='run')

    parsed = p.parse_args(args)

    if parsed.which == 'init':
        # bypass looking for a project file if we're running `dbt init`
        parsed.cls(args=parsed).run()

    elif os.path.isfile('dbt_project.yml'):
        proj = project.read_project('dbt_project.yml').with_profiles(parsed.profile)
        parsed.cls(args=parsed, project=proj).run()

    else:
        raise RuntimeError("dbt must be run from a project root directory with a dbt_project.yml file")

<|endoftext|>


<commit_before>import asyncio

class Cursor:
    def __init__(self, impl, connection):
        self._conn = connection
        self._impl = impl
        self._loop = connection.loop

    def close(self):
        fut = self._conn._execute(self._impl.close)
        return fut

    @property
    def connection(self):
        return self._conn

    @property
    def rowcount(self):
        return self._conn

    @property
    def description(self):
        return self._conn.description

    @property
    def closed(self):
        return self._impl.closed

    @property
    def arraysize(self):
        return self._conn.arraysize

    @arraysize.setter
    def arraysize(self, size):
        self._conn.arraysize = size

    def execute(self, sql, *params):
        fut = self._conn._execute(self._impl.execute, sql, *params)
        return fut

    def executemany(self, sql, *params):
        fut = self._conn._execute(self._impl.executemany, sql, *params)
        return fut

    @asyncio.coroutine
    def setinputsizes(self, *args, **kwargs):
        return None

    @asyncio.coroutine
    def setoutputsize(self, *args, **kwargs):
        return None

    def fetchone(self):
        fut = self._conn._execute(self._impl.fetchone)
        return fut

    def fetchall(self):
        fut = self._conn._execute(self._impl.fetchall)
        return fut

    def fetchmany(self, size):
        fut = self._conn._execute(self._impl.fetchmany, size)
        return fut

    def nextset(self):
        fut = self._conn._execute(self._impl.nextset)
        return fut

    def tables(self, table=None, catalog=None, schema=None, tableType=None):
        fut = self._conn._execute(self._impl.tables, table=table,
                                  catalog=catalog, schema=schema,
                                  tableType=tableType)
        return fut

    def columns(self, table=None, catalog=None, schema=None, column=None):
        fut = self._conn._execute(self._impl.columns, table=table,
                                  catalog=catalog, schema=schema,
                                  column=column)
        return fut

    @asyncio.coroutine
    def statistics(self, catalog=None, schema=None, unique=False, quick=True):
        fut = self._conn._execute(self._impl.statistics, catalog=catalog,
                                  schema=schema, unique=unique, quick=quick)
        return fut

    def rowIdColumns(self, table, catalog=None, schema=None, nullable=True):
        fut = self._conn._execute(self._impl.rowIdColumns, table,
                                  catalog=catalog, schema=schema,
                                  nullable=nullable)
        return fut

    def rowVerColumns(self, table, catalog=None, schema=None, nullable=True):
        fut = self._conn._execute(self._impl.rowVerColumns, table,
                                  catalog=catalog, schema=schema,
                                  nullable=nullable)
        return fut

    def primaryKeys(self, table, catalog=None, schema=None):
        fut = self._conn._execute(self._impl.primaryKeys, table,
                                  catalog=catalog, schema=schema)
        return fut

    # TODO: check source code of pyodbc regadring table argument
    def foreignKeys(self, table=None, catalog=None, schema=None,
                    foreignTable=None, foreignCatalog=None,
                    foreignSchema=None):
        fut = self._conn._execute(self._impl.foreignKeys, table=table,
                                  catalog=catalog, schema=schema,
                                  foreignTable=foreignTable,
                                  foreignCatalog=foreignCatalog,
                                  foreignSchema=foreignSchema)
        return fut

    def getTypeInfo(self, sqlType=None):
        fut = self._conn._execute(self._impl.getTypeInfo, sqlType)
        return fut

    def procedures(self, procedure=None, catalog=None, schema=None):
        fut = self._conn._execute(self._impl.procedures, procedure=procedure,
                                  catalog=catalog, schema=schema)
        return fut

    def procedureColumns(self, procedure=None, catalog=None, schema=None):
        fut = self._conn._execute(self._impl.procedureColumns,
                                  procedure=procedure, catalog=catalog,
                                  schema=schema)
        return fut

    def skip(self, count):
        fut = self._conn._execute(self._impl.skip, count)
        return fut

    def commit(self):
        fut = self._conn._execute(self._impl.commit)
        return fut

    def rollback(self):
        fut = self._conn._execute(self._impl.rollback)
        return fut
<commit_msg>fix bunch of cursors methods<commit_after>import asyncio

class Cursor:
    def __init__(self, impl, connection):
        self._conn = connection
        self._impl = impl
        self._loop = connection.loop

    @asyncio.coroutine
    def close(self):
        resp = yield from self._conn._execute(self._impl.close)
        self._conn = None
        return resp

    @property
    def connection(self):
        return self._conn

    @property
    def rowcount(self):
        return self._impl.rowcount

    @property
    def description(self):
        return self._impl.description

    @property
    def closed(self):
        return self._conn is None

    @property
    def arraysize(self):
        return self._impl.arraysize

    @arraysize.setter
    def arraysize(self, size):
        self._conn.arraysize = size

    def execute(self, sql, *params):
        fut = self._conn._execute(self._impl.execute, sql, *params)
        return fut

    def executemany(self, sql, *params):
        fut = self._conn._execute(self._impl.executemany, sql, *params)
        return fut

    @asyncio.coroutine
    def setinputsizes(self, *args, **kwargs):
        return None

    @asyncio.coroutine
    def setoutputsize(self, *args, **kwargs):
        return None

    def fetchone(self):
        fut = self._conn._execute(self._impl.fetchone)
        return fut

    def fetchall(self):
        fut = self._conn._execute(self._impl.fetchall)
        return fut

    def fetchmany(self, size):
        fut = self._conn._execute(self._impl.fetchmany, size)
        return fut

    def nextset(self):
        fut = self._conn._execute(self._impl.nextset)
        return fut

    def tables(self, table=None, catalog=None, schema=None, tableType=None):
        fut = self._conn._execute(self._impl.tables, table=table,
                                  catalog=catalog, schema=schema,
                                  tableType=tableType)
        return fut

    def columns(self, table=None, catalog=None, schema=None, column=None):
        fut = self._conn._execute(self._impl.columns, table=table,
                                  catalog=catalog, schema=schema,
                                  column=column)
        return fut

    @asyncio.coroutine
    def statistics(self, catalog=None, schema=None, unique=False, quick=True):
        fut = self._conn._execute(self._impl.statistics, catalog=catalog,
                                  schema=schema, unique=unique, quick=quick)
        return fut

    def rowIdColumns(self, table, catalog=None, schema=None, nullable=True):
        fut = self._conn._execute(self._impl.rowIdColumns, table,
                                  catalog=catalog, schema=schema,
                                  nullable=nullable)
        return fut

    def rowVerColumns(self, table, catalog=None, schema=None, nullable=True):
        fut = self._conn._execute(self._impl.rowVerColumns, table,
                                  catalog=catalog, schema=schema,
                                  nullable=nullable)
        return fut

    def primaryKeys(self, table, catalog=None, schema=None):
        fut = self._conn._execute(self._impl.primaryKeys, table,
                                  catalog=catalog, schema=schema)
        return fut

    # TODO: check source code of pyodbc regadring table argument
    def foreignKeys(self, table=None, catalog=None, schema=None,
                    foreignTable=None, foreignCatalog=None,
                    foreignSchema=None):
        fut = self._conn._execute(self._impl.foreignKeys, table=table,
                                  catalog=catalog, schema=schema,
                                  foreignTable=foreignTable,
                                  foreignCatalog=foreignCatalog,
                                  foreignSchema=foreignSchema)
        return fut

    def getTypeInfo(self, sqlType=None):
        fut = self._conn._execute(self._impl.getTypeInfo, sqlType)
        return fut

    def procedures(self, procedure=None, catalog=None, schema=None):
        fut = self._conn._execute(self._impl.procedures, procedure=procedure,
                                  catalog=catalog, schema=schema)
        return fut

    def procedureColumns(self, procedure=None, catalog=None, schema=None):
        fut = self._conn._execute(self._impl.procedureColumns,
                                  procedure=procedure, catalog=catalog,
                                  schema=schema)
        return fut

    def skip(self, count):
        fut = self._conn._execute(self._impl.skip, count)
        return fut

    def commit(self):
        fut = self._conn._execute(self._impl.commit)
        return fut

    def rollback(self):
        fut = self._conn._execute(self._impl.rollback)
        return fut
<|endoftext|>


<commit_before>"""
Prototype selection

Given a set of n elements (S) and their pairwise distances in terms of a
skbio distance matrix (DM) and a given number (k << n), find the sub-set (s)
consisting of exactly k elements, such that s best represents the full set S.

Here, we define "best represents" as maximizing the sum of distances between
all points, i.e. sum {over a,b in S} DM[a,b]. This is our objective function.

This problem is known to be NP-hard [1], thus we need to resort to heuristics.
In the future, this module will implement several heuristics, whose quality can
be measured by the objective function for each problem instance, since there is
no global winner.
For completeness, the exact but exponential algorithm is implemented, too.
  "prototype_selection_exhaustive"

[1] Gamez, J. Esteban, François Modave, and Olga Kosheleva.
    "Selecting the most representative sample is NP-hard:
     Need for expert (fuzzy) knowledge."
    Fuzzy Systems, 2008. FUZZ-IEEE 2008.
"""

# needed for signature type annotations, but only works for python >= 3.5
# from typing import Sequence, Tuple
from itertools import combinations

import numpy as np
import scipy as sp
from skbio.stats.distance import DistanceMatrix

def distance_sum(elements, dm):
    '''Compute the sum of pairwise distances for the given elements according to
    the given distance matrix.

    Parameters
    ----------
    elements: sequence of str
        list or elements for which the sum of distances is computed
    dm: skbio.stats.distance.DistanceMatrix
        pairwise distance matrix.

    Returns
    -------
    float:
        the sum of all pairwise distances of dm for IDs in elements

    Notes
    -----
    function signature with type annotation for future use with python >= 3.5
    def distance_sum(elements: Sequence[str], dm: DistanceMatrix) -> float:
    '''

    return np.tril(dm.filter(elements).data).sum()

def prototype_selection_exhaustive(dm, num_prototypes,
                                   max_combinations_to_test=200000):
    '''Select k prototypes for given distance matrix

    Parameters
    ----------
    dm: skbio.stats.distance.DistanceMatrix
        pairwise distances for all elements in the full set S.
        Must be symmetric and non-hollow.
    num_prototypes: int
        Number of prototypes to select for distance matrix.
        Must be >= 2, since a single prototype is useless.
        Must be smaller than the number of elements in the distance matrix,
        otherwise no reduction is necessary.
    max_combinations_to_test: int
        The maximal number of combinations to test. If exceeding, the function
        declines execution.

    Returns
    -------
    sequence of str
        A sequence holding selected prototypes, i.e. a sub-set of the
        elements in the distance matrix.

    Raises
    ------
    RuntimeError
        Combinatorics explode even for small instances. To save the user from
        waiting (almost) forever, this function declines execution if the
        number of combinations to test are too high,
        i.e. > max_combinations_to_test
    ValueError
        The number of prototypes to be found should be at least 2 and at most
        one element smaller than elements in the distance matrix. Otherwise, a
        ValueError is raised.

    Notes
    -----
    This is the reference implementation for an exact algorithm for the
    prototype selection problem. It has an exponential runtime and will only
    operate on small instances (< max_combinations_to_test).
    Idea: test all (n over k) combinations of selecting k elements from n with-
          out replacement. Compute the objective for each such combination and
          report the combination with maximal value.

    function signature with type annotation for future use with python >= 3.5:
    def prototype_selection_exhaustive(dm: DistanceMatrix, num_prototypes: int,
    max_combinations_to_test: int=200000) -> Sequence[str]:
    '''
    if num_prototypes < 2:
        raise ValueError(("'num_prototypes' must be >= 2, since a single "
                          "prototype is useless."))
    if num_prototypes >= len(dm.ids):
        raise ValueError(("'num_prototypes' must be smaller than the number of"
                          " elements in the distance matrix, otherwise no "
                          "reduction is necessary."))

    num_combinations = sp.special.binom(len(dm.ids), num_prototypes)
    if num_combinations >= max_combinations_to_test:
        raise RuntimeError(("Cowardly refuse to test %i combinations. Use a "
                            "heuristic implementation for instances with more "
                            "than %i combinations instead!")
                           % (num_combinations, max_combinations_to_test))

    max_dist, max_set = -1 * np.infty, None
    for s in set(combinations(dm.ids, num_prototypes)):
        d = distance_sum(s, dm)
        if d > max_dist:
            max_dist, max_set = d, s
    return max_set

def prototype_selection_constructive_maxdist(dm, num_prototypes, steps=10):
    '''Heuristically select k prototypes for given distance matrix.

       Prototype selection is NP-hard. This is an implementation of a greedy
       correctness heuristic: Greedily grow the set of prototypes by adding the
       element with the largest sum of distances to the non-prototype elements.
       Start with the two elements that are globally most distant from each
       other. The set of prototypes is then constructively grown by adding the
       element showing largest sum of distances to all non-prototype elements
       in the distance matrix in each iteration.

    Parameters
    ----------
    dm: skbio.stats.distance.DistanceMatrix
        Pairwise distances for all elements in the full set S.
    num_prototypes: int
        Number of prototypes to select for distance matrix.
        Must be >= 2, since a single prototype is useless.
        Must be smaller than the number of elements in the distance matrix,
        otherwise no reduction is necessary.

    Returns
    -------
    sequence of str
        A sequence holding selected prototypes, i.e. a sub-set of the
        elements in the distance matrix.

    Raises
    ------
    RuntimeError
        Combinatorics explode even for small instances. To save the user from
        waiting (almost) forever, this function declines execution if the
        number of combinations to test are too high,
        i.e. > max_combinations_to_test
    ValueError
        The number of prototypes to be found should be at least 2 and at most
        one element smaller than elements in the distance matrix. Otherwise, a
        ValueError is raised.

    Notes
    -----
    Timing: %timeit -n 100 prototype_selection_constructive_maxdist(dm, 100)
            100 loops, best of 3: 1.43 s per loop
            where the dm holds 27,398 elements
    function signature with type annotation for future use with python >= 3.5:
    def prototype_selection_constructive_maxdist(dm: DistanceMatrix,
    num_prototypes: int) -> Sequence[str]:
    '''
    if num_prototypes < 2:
        raise ValueError(("'num_prototypes' must be >= 2, since a single "
                          "prototype is useless."))
    if num_prototypes >= len(dm.ids):
        raise ValueError(("'num_prototypes' must be smaller than the number of"
                          " elements in the distance matrix, otherwise no "
                          "reduction is necessary."))

    # initially mark all elements as uncovered, i.e. as not being a prototype
    uncovered = np.asarray([np.True_] * dm.shape[0])

    # the first two prototypes are those elements that have the globally
    # maximal distance in the distance matrix. Mark those two elements as
    # being covered, i.e. prototypes
    distance, resSet = dm.data.max(), list(np.unravel_index(dm.data.argmax(),
                                           dm.data.shape))
    uncovered[resSet] = np.False_

    # repeat until enough prototypes have been selected:
    #  the new prototype is the element that has maximal distance sum to all
    #  non-prototype elements in the distance matrix.
    while dm.shape[0] - np.sum(uncovered) < num_prototypes:
        maxElmIdx = (dm.data[list(resSet), :].sum(axis=0) * uncovered).argmax()
        uncovered[maxElmIdx] = np.False_
        resSet.append(maxElmIdx)

    # return the ids of the selected prototype elements
    return tuple([dm.ids[idx] for idx, x in enumerate(uncovered)
                 if x == np.False_])
<commit_msg>parameter steps was not used<commit_after>"""
Prototype selection

Given a set of n elements (S) and their pairwise distances in terms of a
skbio distance matrix (DM) and a given number (k << n), find the sub-set (s)
consisting of exactly k elements, such that s best represents the full set S.

Here, we define "best represents" as maximizing the sum of distances between
all points, i.e. sum {over a,b in S} DM[a,b]. This is our objective function.

This problem is known to be NP-hard [1], thus we need to resort to heuristics.
In the future, this module will implement several heuristics, whose quality can
be measured by the objective function for each problem instance, since there is
no global winner.
For completeness, the exact but exponential algorithm is implemented, too.
  "prototype_selection_exhaustive"

[1] Gamez, J. Esteban, François Modave, and Olga Kosheleva.
    "Selecting the most representative sample is NP-hard:
     Need for expert (fuzzy) knowledge."
    Fuzzy Systems, 2008. FUZZ-IEEE 2008.
"""

# needed for signature type annotations, but only works for python >= 3.5
# from typing import Sequence, Tuple
from itertools import combinations

import numpy as np
import scipy as sp
from skbio.stats.distance import DistanceMatrix

def distance_sum(elements, dm):
    '''Compute the sum of pairwise distances for the given elements according to
    the given distance matrix.

    Parameters
    ----------
    elements: sequence of str
        list or elements for which the sum of distances is computed
    dm: skbio.stats.distance.DistanceMatrix
        pairwise distance matrix.

    Returns
    -------
    float:
        the sum of all pairwise distances of dm for IDs in elements

    Notes
    -----
    function signature with type annotation for future use with python >= 3.5
    def distance_sum(elements: Sequence[str], dm: DistanceMatrix) -> float:
    '''

    return np.tril(dm.filter(elements).data).sum()

def prototype_selection_exhaustive(dm, num_prototypes,
                                   max_combinations_to_test=200000):
    '''Select k prototypes for given distance matrix

    Parameters
    ----------
    dm: skbio.stats.distance.DistanceMatrix
        pairwise distances for all elements in the full set S.
        Must be symmetric and non-hollow.
    num_prototypes: int
        Number of prototypes to select for distance matrix.
        Must be >= 2, since a single prototype is useless.
        Must be smaller than the number of elements in the distance matrix,
        otherwise no reduction is necessary.
    max_combinations_to_test: int
        The maximal number of combinations to test. If exceeding, the function
        declines execution.

    Returns
    -------
    sequence of str
        A sequence holding selected prototypes, i.e. a sub-set of the
        elements in the distance matrix.

    Raises
    ------
    RuntimeError
        Combinatorics explode even for small instances. To save the user from
        waiting (almost) forever, this function declines execution if the
        number of combinations to test are too high,
        i.e. > max_combinations_to_test
    ValueError
        The number of prototypes to be found should be at least 2 and at most
        one element smaller than elements in the distance matrix. Otherwise, a
        ValueError is raised.

    Notes
    -----
    This is the reference implementation for an exact algorithm for the
    prototype selection problem. It has an exponential runtime and will only
    operate on small instances (< max_combinations_to_test).
    Idea: test all (n over k) combinations of selecting k elements from n with-
          out replacement. Compute the objective for each such combination and
          report the combination with maximal value.

    function signature with type annotation for future use with python >= 3.5:
    def prototype_selection_exhaustive(dm: DistanceMatrix, num_prototypes: int,
    max_combinations_to_test: int=200000) -> Sequence[str]:
    '''
    if num_prototypes < 2:
        raise ValueError(("'num_prototypes' must be >= 2, since a single "
                          "prototype is useless."))
    if num_prototypes >= len(dm.ids):
        raise ValueError(("'num_prototypes' must be smaller than the number of"
                          " elements in the distance matrix, otherwise no "
                          "reduction is necessary."))

    num_combinations = sp.special.binom(len(dm.ids), num_prototypes)
    if num_combinations >= max_combinations_to_test:
        raise RuntimeError(("Cowardly refuse to test %i combinations. Use a "
                            "heuristic implementation for instances with more "
                            "than %i combinations instead!")
                           % (num_combinations, max_combinations_to_test))

    max_dist, max_set = -1 * np.infty, None
    for s in set(combinations(dm.ids, num_prototypes)):
        d = distance_sum(s, dm)
        if d > max_dist:
            max_dist, max_set = d, s
    return max_set

def prototype_selection_constructive_maxdist(dm, num_prototypes):
    '''Heuristically select k prototypes for given distance matrix.

       Prototype selection is NP-hard. This is an implementation of a greedy
       correctness heuristic: Greedily grow the set of prototypes by adding the
       element with the largest sum of distances to the non-prototype elements.
       Start with the two elements that are globally most distant from each
       other. The set of prototypes is then constructively grown by adding the
       element showing largest sum of distances to all non-prototype elements
       in the distance matrix in each iteration.

    Parameters
    ----------
    dm: skbio.stats.distance.DistanceMatrix
        Pairwise distances for all elements in the full set S.
    num_prototypes: int
        Number of prototypes to select for distance matrix.
        Must be >= 2, since a single prototype is useless.
        Must be smaller than the number of elements in the distance matrix,
        otherwise no reduction is necessary.

    Returns
    -------
    sequence of str
        A sequence holding selected prototypes, i.e. a sub-set of the
        elements in the distance matrix.

    Raises
    ------
    RuntimeError
        Combinatorics explode even for small instances. To save the user from
        waiting (almost) forever, this function declines execution if the
        number of combinations to test are too high,
        i.e. > max_combinations_to_test
    ValueError
        The number of prototypes to be found should be at least 2 and at most
        one element smaller than elements in the distance matrix. Otherwise, a
        ValueError is raised.

    Notes
    -----
    Timing: %timeit -n 100 prototype_selection_constructive_maxdist(dm, 100)
            100 loops, best of 3: 1.43 s per loop
            where the dm holds 27,398 elements
    function signature with type annotation for future use with python >= 3.5:
    def prototype_selection_constructive_maxdist(dm: DistanceMatrix,
    num_prototypes: int) -> Sequence[str]:
    '''
    if num_prototypes < 2:
        raise ValueError(("'num_prototypes' must be >= 2, since a single "
                          "prototype is useless."))
    if num_prototypes >= len(dm.ids):
        raise ValueError(("'num_prototypes' must be smaller than the number of"
                          " elements in the distance matrix, otherwise no "
                          "reduction is necessary."))

    # initially mark all elements as uncovered, i.e. as not being a prototype
    uncovered = np.asarray([np.True_] * dm.shape[0])

    # the first two prototypes are those elements that have the globally
    # maximal distance in the distance matrix. Mark those two elements as
    # being covered, i.e. prototypes
    distance, resSet = dm.data.max(), list(np.unravel_index(dm.data.argmax(),
                                           dm.data.shape))
    uncovered[resSet] = np.False_

    # repeat until enough prototypes have been selected:
    #  the new prototype is the element that has maximal distance sum to all
    #  non-prototype elements in the distance matrix.
    while dm.shape[0] - np.sum(uncovered) < num_prototypes:
        maxElmIdx = (dm.data[list(resSet), :].sum(axis=0) * uncovered).argmax()
        uncovered[maxElmIdx] = np.False_
        resSet.append(maxElmIdx)

    # return the ids of the selected prototype elements
    return tuple([dm.ids[idx] for idx, x in enumerate(uncovered)
                 if x == np.False_])
<|endoftext|>