<commit_before>'''
Bigmetadata tasks

tasks to download and create metadata
'''

import os

from collections import OrderedDict
from datetime import datetime
from luigi import (Task, IntParameter, LocalTarget, Parameter, WrapperTask)
from tasks.base_tasks import ColumnsTask, TableTask, TagsTask, CSV2TempTableTask, MetaWrapper
from tasks.util import shell, classpath, underscore_slugify
from tasks.tags import SectionTags, SubsectionTags, UnitTags
from tasks.meta import OBSColumn, current_session, OBSTag, UNIVERSE
from tasks.us.census.tiger import GeoidColumns, SumLevel

# cherry-picked datasets
HOMETYPES = {
    'AllHomes': 'All homes',
    'SingleFamilyResidence': 'Single Family Homes',
    'AllHomesPlusMultifamily': 'All homes plus multifamily',
    'SingleFamilyResidenceRental': 'Single Family residence rental',
    'Sfr': 'Single Family residence rental',
}

MEASURES_HUMAN = {
    'Zhvi': 'Zillow Home Value Index',
    'Zri': 'Zillow Rental Index',
    'MedianValuePerSqft': 'Median value per square foot',
    'MedianRentalPricePerSqft': 'Median rental price per square foot'
}

MEASURES_UNITS = {
    'Zhvi': 'index',
    'Zri': 'index',
    'MedianValuePerSqft': 'money',
    'MedianRentalPricePerSqft': 'money'
}

HOMETYPES_DESCRIPTION = {
    'AllHomes': 'Zillow defines all homes as single-family, condominium and '
                'co-operative homes with a county record. Unless specified, '
                'all series cover this segment of the housing stock.',
    'SingleFamilyResidence': 'Single family residences are detached, '
                             'free-standing residential buildings. ',
    'AllHomesPlusMultifamily': 'In addition to "All homes", which Zillow defines '
                               'as single-family, condominium and co-operative '
                               'homes with a county record, this group includes '
                               'units in buildings with 5 or more housing '
                               'units that are not a condominiums or co-ops.',
    'SingleFamilyResidenceRental': 'Single Family residence rental is defined '
                                   'as detached, free-standing residential '
                                   'buildings which are rented out.',
}

HOMETYPES_DESCRIPTION['Sfr'] = HOMETYPES_DESCRIPTION['SingleFamilyResidence']

MEASURES_DESCRIPTION = {
    'Zhvi': 'The Zillow Home Value Index (ZHVI) is a time series tracking the '
            'monthly median home value (in US Dollars) in a particular '
            'geographical region. In general, each ZHVI time series begins in '
            'April 1996. See `Zillow\'s methodology '
            '<http://www.zillow.com/research/zhvi-methodology-6032/>`_ for '
            'more information.',
    'Zri': 'Similar to Zillow\'s ZHVI, the Zillow Rent Index (ZRI) tracks the '
           'monthly median rent (in US Dollars) in different geographical '
           'regions. In general, each ZRI time series beginds in November, '
           '2010.  See `Zillow\'s methodology '
           '<http://www.zillow.com/research/zillow-rent-index-methodology-2393/>`_ '
           'for more information.',
    'MedianValuePerSqft': 'Median of the estimated monthly rent price (US '
                          'Dollars) of all homes, per square foot. This is '
                          'calculated by taking the estimated rent price for '
                          'a home and dividing it by the homes square footage.',
    'MedianRentalPricePerSqft': 'Median of the value (US Dollars) of all homes '
                                'per square foot. This number is calculated by '
                                'taking the estimated home value for each home '
                                'in a given region and dividing it by the '
                                'home\'s square footage.'
}

HOMES = ('AllHomes', 'SingleFamilyResidence', )
RENTALS = ('AllHomesPlusMultifamily', 'SingleFamilyResidenceRental',
           'AllHomes', 'Sfr', )
MEDIAN_VALUE_HOMETYPES = ('AllHomes',)
MEDIAN_RENTAL_HOMETYPES = ('AllHomes', 'Sfr', 'CondoCoop',)

def measure_name(hometype):
    if hometype in HOMES:
        measure_name = 'Zhvi'
    elif hometype == 'Sfr':
        measure_name = None
    elif hometype in RENTALS:
        measure_name = 'Zri'
    else:
        raise Exception('Unknown hometype "{hometype}"'.format(hometype=hometype))

    return measure_name

def measure_aggregation(hometype):
    aggregation = []

    if hometype in MEDIAN_VALUE_HOMETYPES:
        aggregation.append('MedianValuePerSqft')

    if hometype in MEDIAN_RENTAL_HOMETYPES:
        aggregation.append('MedianRentalPricePerSqft')

    return aggregation

class ZillowSourceTags(TagsTask):

    def tags(self):
        return [OBSTag(id='zillow-source',
                       name='Zillow Data',
                       type='source',
                       description='Zillow makes available data free for reuse `here <http://www.zillow.com/research/data/>`_.', )]

class ZillowLicenseTags(TagsTask):

    def version(self):
        return 2

    def tags(self):
        return [OBSTag(id='zillow-license',
                       name='Zillow Terms of Use for "Aggregate Data"',
                       type='license',
                       description='May be used for non-personal uses, e.g., real estate market analysis. More information `here <http://www.zillow.com/corp/Terms.htm>`_', )]

class ZillowTags(TagsTask):

    def version(self):
        return 2

    def tags(self):
        return [
            OBSTag(id='indexes',
                   name='Zillow Home Value and Rental Indexes',
                   type='subsection',
                   description='Zillow home value and rental indexes.'),
        ]

class DownloadZillow(Task):

    geography = Parameter()
    hometype = Parameter()
    measure = Parameter()
    last_year = IntParameter()
    last_month = IntParameter()

    URL = 'http://files.zillowstatic.com/research/public/{geography}/{geography}_{measure}_{hometype}.csv'

    def url(self):
        return self.URL.format(geography=self.geography, hometype=self.hometype,
                               measure=self.measure)

    @property
    def last_time(self):
        if not hasattr(self, '_last_time'):
            last_time = shell('curl -s {url} | head -n 1'.format(url=self.url()))
            self._last_time = last_time.strip().split(',')[-1].strip('"')
        return self._last_time

    def run(self):
        self.output().makedirs()
        shell('wget \'{url}\' -O {output}'.format(url=self.url(),
                                                  output=self.output().path))

    def output(self):
        return LocalTarget(os.path.join('tmp', classpath(self), self.task_id) +
                           '_' + underscore_slugify(self.last_time) + '.csv')

class ZillowValueColumns(ColumnsTask):

    def requires(self):
        return {
            'tags': ZillowTags(),
            'subsections': SubsectionTags(),
            'sections': SectionTags(),
            'units': UnitTags(),
            'source': ZillowSourceTags(),
            'license': ZillowLicenseTags(),
        }

    def version(self):
        return 7

    def build_column(self, input_, hometype, measure, hometype_human, aggregate, targets={}):
        tag = input_['tags']['indexes']
        united_states = input_['sections']['united_states']
        housing = input_['subsections']['housing']
        units = input_['units']
        source = input_['source']['zillow-source']
        license = input_['license']['zillow-license']

        measure_human = MEASURES_HUMAN[measure]
        measure_unit = MEASURES_UNITS[measure]

        col = OBSColumn(type='Numeric',
                        name='{measure} for {hometype}'.format(
                            measure=measure_human,
                            hometype=hometype_human),
                        aggregate=aggregate,
                        weight=1,
                        description='{measure_description} {hometype_description}'.format(
                            measure_description=MEASURES_DESCRIPTION[measure],
                            hometype_description=HOMETYPES_DESCRIPTION[hometype],
                            ),
                        targets=targets,
                        tags=[tag, united_states, housing, units[measure_unit], license, source])

        return col

    def columns(self):
        input_ = self.input()

        columns = OrderedDict()

        for hometype, hometype_human in HOMETYPES.items():
            measure = measure_name(hometype)

            if measure:
                col_id = '{hometype}_{measure}'.format(hometype=hometype,
                                                       measure=measure)
                columns[col_id] = self.build_column(input_, hometype, measure, hometype_human, 'index')

                aggregations = measure_aggregation(hometype)
                for aggregation in aggregations:
                    targets = {columns[col_id]: UNIVERSE}

                    col_id = '{hometype}_{aggregation}'.format(hometype=hometype,
                                                               aggregation=aggregation)
                    columns[col_id] = self.build_column(input_, hometype, aggregation, hometype_human,
                                                        'median', targets)

        return columns

class ZillowGeoColumns(ColumnsTask):

    def columns(self):
        # TODO manually generate columns before value columns

        columns = OrderedDict([
            ('RegionID', OBSColumn(type='Text',
                                   name='Zillow geography identifier',
                                   description="",
                                   weight=0)),
            ('RegionName', OBSColumn(type='Text',
                                     name='ZCTA5',
                                     description="Zip Code Tabulation Area id",
                                     weight=0)),
            ('City', OBSColumn(type='Text',
                               name='City Name',
                               description="",
                               weight=0)),
            ('State', OBSColumn(type='Text',
                                name='State Name',
                                description="",
                                weight=0)),
            ('Metro', OBSColumn(type='Text',
                                name='Metro Area Name',
                                description="",
                                weight=0)),
            ('CountyName', OBSColumn(type='Text',
                                     name='County Name',
                                     description="",
                                     weight=0)),
            ('SizeRank', OBSColumn(type='Text',
                                   name='Size Rank',
                                   description="",
                                   weight=0)),
            ('State', OBSColumn(type='Text',
                                name='State Name',
                                description="",
                                weight=0))
        ])

        return columns

class WideZillow(CSV2TempTableTask):

    geography = Parameter()  # example: Zip
    hometype = Parameter()  # example: SingleFamilyResidence
    measure = Parameter()
    last_year = IntParameter()
    last_month = IntParameter()

    def requires(self):
        return DownloadZillow(geography=self.geography, hometype=self.hometype,
                              measure=self.measure, last_year=self.last_year,
                              last_month=self.last_month)

    def input_csv(self):
        return self.input().path

class Zillow(TableTask):

    year = IntParameter()
    month = IntParameter()
    geography = Parameter()  # example: Zip

    def version(self):
        return 5

    def requires(self):
        requirements = {
            'metadata': ZillowValueColumns(),
            'geoids': GeoidColumns()
        }
        for hometype, hometype_human in HOMETYPES.items():
            measure = measure_name(hometype)

            if measure:
                table_id = '{hometype}_{measure}'.format(hometype=hometype,
                                                         measure=measure)
                requirements[table_id] = WideZillow(
                    geography=self.geography, hometype=hometype, measure=measure,
                    last_month=datetime.now().month, last_year=datetime.now().year)

                aggregations = measure_aggregation(hometype)
                for aggregation in aggregations:
                    table_id = '{hometype}_{measure}'.format(hometype=hometype,
                                                             measure=aggregation)
                    requirements[table_id] = WideZillow(
                        geography=self.geography, hometype=hometype, measure=aggregation,
                        last_month=datetime.now().month, last_year=datetime.now().year)

        return requirements

    def timespan(self):
        return '{year}-{month}'.format(year=str(self.year).zfill(2),
                                       month=str(self.month).zfill(2))

    def columns(self):
        input_ = self.input()
        if self.geography == 'Zip':
            tiger_geo = 'zcta5'
        else:
            # will happen for metro areas, cities, neighborhoods, state, county
            raise Exception('unrecognized geography {}'.format(self.geography))

        columns = OrderedDict([
            ('region_name', input_['geoids'][tiger_geo + '_geoid']),
        ])
        columns.update(input_['metadata'])

        return columns

    def populate(self):
        session = current_session()

        insert = True
        input_ = self.input()
        output = self.output()
        session.execute('ALTER TABLE {output} ADD PRIMARY KEY (region_name)'.format(
            output=output.table))
        session.flush()
        for key, value in input_['metadata'].items():
            input_table = input_[key].table
            stmt = '''INSERT INTO {output} (region_name, {col_id})
                      SELECT "RegionName", "{year}-{month}"::NUMERIC
                      FROM {input_table}
                      ON CONFLICT (region_name)
                         DO UPDATE SET {col_id} = EXCLUDED.{col_id}'''
            session.execute(stmt.format(
                output=output.table,
                year=str(self.year).zfill(2),
                month=str(self.month).zfill(2),
                col_id=key,
                input_table=input_table))
            if insert:
                insert = False

class AllZillow(WrapperTask):

    def requires(self):
        now = datetime.now()
        for geography in ('Zip', ):
            for year in range(2010, now.year + 1):
                for month in range(1, 13):
                    if now.year == year and now.month <= month:
                        continue
                    if year == 2010 and month <= 10:
                        continue
                    yield Zillow(geography=geography, year=year, month=month)

class ZillowMetaWrapper(MetaWrapper):
    geography = Parameter()
    year = IntParameter()
    month = IntParameter()

    now = datetime.now()

    params = {
        'geography': ['Zip'],
        'year': list(range(2010, now.year + 1)),
        'month': list(range(1, 13))
    }

    def tables(self):
        now = datetime.now()
        if now.year == self.year and now.month <= self.month:
            return
        if self.year == 2010 and self.month <= 10:
            return
        yield Zillow(geography=self.geography, year=self.year, month=self.month)
        yield SumLevel(year=str(2015), geography='zcta5')
<commit_msg>Fixed circular dependencies in Zillow<commit_after>'''
Bigmetadata tasks

tasks to download and create metadata
'''

import os

from collections import OrderedDict
from datetime import datetime
from luigi import (Task, IntParameter, LocalTarget, Parameter, WrapperTask)
from tasks.base_tasks import ColumnsTask, TableTask, TagsTask, CSV2TempTableTask, MetaWrapper
from tasks.util import shell, classpath, underscore_slugify
from tasks.tags import SectionTags, SubsectionTags, UnitTags
from tasks.meta import OBSColumn, current_session, OBSTag, UNIVERSE
from tasks.us.census.tiger import GeoidColumns, SumLevel, GEOID_SUMLEVEL_COLUMN, GEOID_SHORELINECLIPPED_COLUMN

# cherry-picked datasets
HOMETYPES = {
    'AllHomes': 'All homes',
    'SingleFamilyResidence': 'Single Family Homes',
    'AllHomesPlusMultifamily': 'All homes plus multifamily',
    'SingleFamilyResidenceRental': 'Single Family residence rental',
    'Sfr': 'Single Family residence rental',
}

MEASURES_HUMAN = {
    'Zhvi': 'Zillow Home Value Index',
    'Zri': 'Zillow Rental Index',
    'MedianValuePerSqft': 'Median value per square foot',
    'MedianRentalPricePerSqft': 'Median rental price per square foot'
}

MEASURES_UNITS = {
    'Zhvi': 'index',
    'Zri': 'index',
    'MedianValuePerSqft': 'money',
    'MedianRentalPricePerSqft': 'money'
}

HOMETYPES_DESCRIPTION = {
    'AllHomes': 'Zillow defines all homes as single-family, condominium and '
                'co-operative homes with a county record. Unless specified, '
                'all series cover this segment of the housing stock.',
    'SingleFamilyResidence': 'Single family residences are detached, '
                             'free-standing residential buildings. ',
    'AllHomesPlusMultifamily': 'In addition to "All homes", which Zillow defines '
                               'as single-family, condominium and co-operative '
                               'homes with a county record, this group includes '
                               'units in buildings with 5 or more housing '
                               'units that are not a condominiums or co-ops.',
    'SingleFamilyResidenceRental': 'Single Family residence rental is defined '
                                   'as detached, free-standing residential '
                                   'buildings which are rented out.',
}

HOMETYPES_DESCRIPTION['Sfr'] = HOMETYPES_DESCRIPTION['SingleFamilyResidence']

MEASURES_DESCRIPTION = {
    'Zhvi': 'The Zillow Home Value Index (ZHVI) is a time series tracking the '
            'monthly median home value (in US Dollars) in a particular '
            'geographical region. In general, each ZHVI time series begins in '
            'April 1996. See `Zillow\'s methodology '
            '<http://www.zillow.com/research/zhvi-methodology-6032/>`_ for '
            'more information.',
    'Zri': 'Similar to Zillow\'s ZHVI, the Zillow Rent Index (ZRI) tracks the '
           'monthly median rent (in US Dollars) in different geographical '
           'regions. In general, each ZRI time series beginds in November, '
           '2010.  See `Zillow\'s methodology '
           '<http://www.zillow.com/research/zillow-rent-index-methodology-2393/>`_ '
           'for more information.',
    'MedianValuePerSqft': 'Median of the estimated monthly rent price (US '
                          'Dollars) of all homes, per square foot. This is '
                          'calculated by taking the estimated rent price for '
                          'a home and dividing it by the homes square footage.',
    'MedianRentalPricePerSqft': 'Median of the value (US Dollars) of all homes '
                                'per square foot. This number is calculated by '
                                'taking the estimated home value for each home '
                                'in a given region and dividing it by the '
                                'home\'s square footage.'
}

HOMES = ('AllHomes', 'SingleFamilyResidence', )
RENTALS = ('AllHomesPlusMultifamily', 'SingleFamilyResidenceRental',
           'AllHomes', 'Sfr', )
MEDIAN_VALUE_HOMETYPES = ('AllHomes',)
MEDIAN_RENTAL_HOMETYPES = ('AllHomes', 'Sfr', 'CondoCoop',)

def measure_name(hometype):
    if hometype in HOMES:
        measure_name = 'Zhvi'
    elif hometype == 'Sfr':
        measure_name = None
    elif hometype in RENTALS:
        measure_name = 'Zri'
    else:
        raise Exception('Unknown hometype "{hometype}"'.format(hometype=hometype))

    return measure_name

def measure_aggregation(hometype):
    aggregation = []

    if hometype in MEDIAN_VALUE_HOMETYPES:
        aggregation.append('MedianValuePerSqft')

    if hometype in MEDIAN_RENTAL_HOMETYPES:
        aggregation.append('MedianRentalPricePerSqft')

    return aggregation

class ZillowSourceTags(TagsTask):

    def tags(self):
        return [OBSTag(id='zillow-source',
                       name='Zillow Data',
                       type='source',
                       description='Zillow makes available data free for reuse `here <http://www.zillow.com/research/data/>`_.', )]

class ZillowLicenseTags(TagsTask):

    def version(self):
        return 2

    def tags(self):
        return [OBSTag(id='zillow-license',
                       name='Zillow Terms of Use for "Aggregate Data"',
                       type='license',
                       description='May be used for non-personal uses, e.g., real estate market analysis. More information `here <http://www.zillow.com/corp/Terms.htm>`_', )]

class ZillowTags(TagsTask):

    def version(self):
        return 2

    def tags(self):
        return [
            OBSTag(id='indexes',
                   name='Zillow Home Value and Rental Indexes',
                   type='subsection',
                   description='Zillow home value and rental indexes.'),
        ]

class DownloadZillow(Task):

    geography = Parameter()
    hometype = Parameter()
    measure = Parameter()
    last_year = IntParameter()
    last_month = IntParameter()

    URL = 'http://files.zillowstatic.com/research/public/{geography}/{geography}_{measure}_{hometype}.csv'

    def url(self):
        return self.URL.format(geography=self.geography, hometype=self.hometype,
                               measure=self.measure)

    @property
    def last_time(self):
        if not hasattr(self, '_last_time'):
            last_time = shell('curl -s {url} | head -n 1'.format(url=self.url()))
            self._last_time = last_time.strip().split(',')[-1].strip('"')
        return self._last_time

    def run(self):
        self.output().makedirs()
        shell('wget \'{url}\' -O {output}'.format(url=self.url(),
                                                  output=self.output().path))

    def output(self):
        return LocalTarget(os.path.join('tmp', classpath(self), self.task_id) +
                           '_' + underscore_slugify(self.last_time) + '.csv')

class ZillowValueColumns(ColumnsTask):

    def requires(self):
        return {
            'tags': ZillowTags(),
            'subsections': SubsectionTags(),
            'sections': SectionTags(),
            'units': UnitTags(),
            'source': ZillowSourceTags(),
            'license': ZillowLicenseTags(),
        }

    def version(self):
        return 7

    def build_column(self, input_, hometype, measure, hometype_human, aggregate, targets={}):
        tag = input_['tags']['indexes']
        united_states = input_['sections']['united_states']
        housing = input_['subsections']['housing']
        units = input_['units']
        source = input_['source']['zillow-source']
        license = input_['license']['zillow-license']

        measure_human = MEASURES_HUMAN[measure]
        measure_unit = MEASURES_UNITS[measure]

        col = OBSColumn(type='Numeric',
                        name='{measure} for {hometype}'.format(
                            measure=measure_human,
                            hometype=hometype_human),
                        aggregate=aggregate,
                        weight=1,
                        description='{measure_description} {hometype_description}'.format(
                            measure_description=MEASURES_DESCRIPTION[measure],
                            hometype_description=HOMETYPES_DESCRIPTION[hometype],
                            ),
                        targets=targets,
                        tags=[tag, united_states, housing, units[measure_unit], license, source])

        return col

    def columns(self):
        input_ = self.input()

        columns = OrderedDict()

        for hometype, hometype_human in HOMETYPES.items():
            measure = measure_name(hometype)

            if measure:
                col_id = '{hometype}_{measure}'.format(hometype=hometype,
                                                       measure=measure)
                columns[col_id] = self.build_column(input_, hometype, measure, hometype_human, 'index')

                aggregations = measure_aggregation(hometype)
                for aggregation in aggregations:
                    targets = {columns[col_id]: UNIVERSE}

                    col_id = '{hometype}_{aggregation}'.format(hometype=hometype,
                                                               aggregation=aggregation)
                    columns[col_id] = self.build_column(input_, hometype, aggregation, hometype_human,
                                                        'median', targets)

        return columns

class ZillowGeoColumns(ColumnsTask):

    def columns(self):
        # TODO manually generate columns before value columns

        columns = OrderedDict([
            ('RegionID', OBSColumn(type='Text',
                                   name='Zillow geography identifier',
                                   description="",
                                   weight=0)),
            ('RegionName', OBSColumn(type='Text',
                                     name='ZCTA5',
                                     description="Zip Code Tabulation Area id",
                                     weight=0)),
            ('City', OBSColumn(type='Text',
                               name='City Name',
                               description="",
                               weight=0)),
            ('State', OBSColumn(type='Text',
                                name='State Name',
                                description="",
                                weight=0)),
            ('Metro', OBSColumn(type='Text',
                                name='Metro Area Name',
                                description="",
                                weight=0)),
            ('CountyName', OBSColumn(type='Text',
                                     name='County Name',
                                     description="",
                                     weight=0)),
            ('SizeRank', OBSColumn(type='Text',
                                   name='Size Rank',
                                   description="",
                                   weight=0)),
            ('State', OBSColumn(type='Text',
                                name='State Name',
                                description="",
                                weight=0))
        ])

        return columns

class WideZillow(CSV2TempTableTask):

    geography = Parameter()  # example: Zip
    hometype = Parameter()  # example: SingleFamilyResidence
    measure = Parameter()
    last_year = IntParameter()
    last_month = IntParameter()

    def requires(self):
        return DownloadZillow(geography=self.geography, hometype=self.hometype,
                              measure=self.measure, last_year=self.last_year,
                              last_month=self.last_month)

    def input_csv(self):
        return self.input().path

class Zillow(TableTask):

    year = IntParameter()
    month = IntParameter()
    geography = Parameter()  # example: Zip

    def version(self):
        return 6

    def requires(self):
        requirements = {
            'metadata': ZillowValueColumns(),
            'geoids': GeoidColumns()
        }
        for hometype, hometype_human in HOMETYPES.items():
            measure = measure_name(hometype)

            if measure:
                table_id = '{hometype}_{measure}'.format(hometype=hometype,
                                                         measure=measure)
                requirements[table_id] = WideZillow(
                    geography=self.geography, hometype=hometype, measure=measure,
                    last_month=datetime.now().month, last_year=datetime.now().year)

                aggregations = measure_aggregation(hometype)
                for aggregation in aggregations:
                    table_id = '{hometype}_{measure}'.format(hometype=hometype,
                                                             measure=aggregation)
                    requirements[table_id] = WideZillow(
                        geography=self.geography, hometype=hometype, measure=aggregation,
                        last_month=datetime.now().month, last_year=datetime.now().year)

        return requirements

    def timespan(self):
        return '{year}-{month}'.format(year=str(self.year).zfill(2),
                                       month=str(self.month).zfill(2))

    def columns(self):
        input_ = self.input()
        if self.geography == 'Zip':
            tiger_geo = 'zcta5'
        else:
            # will happen for metro areas, cities, neighborhoods, state, county
            raise Exception('unrecognized geography {}'.format(self.geography))

        columns = OrderedDict([
            ('region_name_sl', input_['geoids'][tiger_geo + GEOID_SUMLEVEL_COLUMN]),
            ('region_name_sc', input_['geoids'][tiger_geo + GEOID_SHORELINECLIPPED_COLUMN]),
        ])
        columns.update(input_['metadata'])

        return columns

    def populate(self):
        session = current_session()

        insert = True
        input_ = self.input()
        output = self.output()
        session.execute('ALTER TABLE {output} ADD PRIMARY KEY (region_name)'.format(
            output=output.table))
        session.flush()
        for key, value in input_['metadata'].items():
            input_table = input_[key].table
            stmt = '''INSERT INTO {output} (region_name_sl, region_name_sc, {col_id})
                      SELECT "RegionName", "RegionName", "{year}-{month}"::NUMERIC
                      FROM {input_table}
                      ON CONFLICT (region_name)
                         DO UPDATE SET {col_id} = EXCLUDED.{col_id}'''
            session.execute(stmt.format(
                output=output.table,
                year=str(self.year).zfill(2),
                month=str(self.month).zfill(2),
                col_id=key,
                input_table=input_table))
            if insert:
                insert = False

class AllZillow(WrapperTask):

    def requires(self):
        now = datetime.now()
        for geography in ('Zip', ):
            for year in range(2010, now.year + 1):
                for month in range(1, 13):
                    if now.year == year and now.month <= month:
                        continue
                    if year == 2010 and month <= 10:
                        continue
                    yield Zillow(geography=geography, year=year, month=month)

class ZillowMetaWrapper(MetaWrapper):
    geography = Parameter()
    year = IntParameter()
    month = IntParameter()

    now = datetime.now()

    params = {
        'geography': ['Zip'],
        'year': list(range(2010, now.year + 1)),
        'month': list(range(1, 13))
    }

    def tables(self):
        now = datetime.now()
        if now.year == self.year and now.month <= self.month:
            return
        if self.year == 2010 and self.month <= 10:
            return
        yield Zillow(geography=self.geography, year=self.year, month=self.month)
        yield SumLevel(year=str(2015), geography='zcta5')
<|endoftext|>


<commit_before>from twilio.twiml import Response
from django_twilio.decorators import twilio_view
from django.views.decorators.http import require_http_methods
from django.utils import timezone
from dateutil.parser import parse as dateparse

from sked.models import Event, Session

@twilio_view
@require_http_methods(['POST', ])
def coming_up(request):
    sessions = Session.objects.filter(is_public=True, event=Event.objects.current())
    r = Response()
    inmsg = request.POST.get('Body').strip() or 'next'
    if inmsg.lower() == 'next':
        messages = _as_sms(Session.objects.next())
    elif inmsg.lower() == 'now':
        messages = _as_sms(Session.objects.current())
    elif inmsg.lower() == 'lunch':
        try:
            messages = _as_sms(Session.objects.filter(start_time__date=timezone.now().date(),
                                                      title__icontains='lunch')[0])
        except IndexError:
            messages = ["No lunch on the schedule for today, sorry."]
    elif inmsg == '?' or inmsg.lower() == 'help':
        messages = ["Welcome to TCamp!\n\nOptions:\nnow: Current sessions\nnext: Next timeslot\nlunch: When's lunch?\n<time>, eg. 4:30pm: What's happening at 4:30?"]
    else:
        try:
            ts = dateparse(inmsg).replace(tzinfo=timezone.get_current_timezone())
            messages = _as_sms(sessions.filter(start_time__lte=ts, end_time__gte=ts))
        except:
            messages = ['Unable to parse that time. Try something like "4:30", or "next"']

    l = len(messages)
    for i, message in enumerate(messages):
        r.sms(message + '\n(%d/%d)' % (i+1, l))
    return r

def _as_sms(qset):
    msgs = ['No events.']
    now = timezone.now()
    if not qset.count():
        return msgs

    tm = qset[0].start_time.astimezone(timezone.get_current_timezone())
    if tm.date() == now.date():
        msgs[0] = u'At %s:\n' % tm.strftime('%-I:%M')
    else:
        msgs[0] = u'%s at %s:\n' % (tm.strftime('%A'), tm.strftime('%-I:%M'))

    for s in qset:
        line = u'\n%s (%s)\n' % (s.title, s.location.name)
        if len(msgs[-1] + line) <= 150:
            msgs[-1] += line
        else:
            msgs.append(line)

    return msgs
<commit_msg>fix date lookup for lunch<commit_after>from twilio.twiml import Response
from django_twilio.decorators import twilio_view
from django.views.decorators.http import require_http_methods
from django.utils import timezone
from dateutil.parser import parse as dateparse

from sked.models import Event, Session

@twilio_view
@require_http_methods(['POST', ])
def coming_up(request):
    sessions = Session.objects.filter(is_public=True, event=Event.objects.current())
    r = Response()
    inmsg = request.POST.get('Body').strip() or 'next'
    if inmsg.lower() == 'next':
        messages = _as_sms(Session.objects.next())
    elif inmsg.lower() == 'now':
        messages = _as_sms(Session.objects.current())
    elif inmsg.lower() == 'lunch':
        try:
            now = timezone.now()
            messages = _as_sms(Session.objects.filter(start_time__day=now.day,
                                                      start_time__month=now.month,
                                                      start_time__year=now.year,
                                                      title__icontains='lunch')[0])
        except IndexError:
            messages = ["No lunch on the schedule for today, sorry."]
    elif inmsg == '?' or inmsg.lower() == 'help':
        messages = ["Welcome to TCamp!\n\nOptions:\nnow: Current sessions\nnext: Next timeslot\nlunch: When's lunch?\n<time>, eg. 4:30pm: What's happening at 4:30?\n"]
    else:
        try:
            ts = dateparse(inmsg).replace(tzinfo=timezone.get_current_timezone())
            messages = _as_sms(sessions.filter(start_time__lte=ts, end_time__gte=ts))
        except:
            messages = ['Unable to parse that time. Try something like "4:30", or "next"']

    l = len(messages)
    for i, message in enumerate(messages):
        r.sms(message + '\n(%d/%d)' % (i+1, l))
    return r

def _as_sms(qset):
    msgs = ['No events.']
    now = timezone.now()
    if not qset.count():
        return msgs

    tm = qset[0].start_time.astimezone(timezone.get_current_timezone())
    if tm.date() == now.date():
        msgs[0] = u'At %s:\n' % tm.strftime('%-I:%M')
    else:
        msgs[0] = u'%s at %s:\n' % (tm.strftime('%A'), tm.strftime('%-I:%M'))

    for s in qset:
        line = u'\n%s (%s)\n' % (s.title, s.location.name)
        if len(msgs[-1] + line) <= 150:
            msgs[-1] += line
        else:
            msgs.append(line)

    return msgs
<|endoftext|>


<commit_before>import os
import sys
import unittest

class TestSystemConfig(unittest.TestCase):
    def test_01_tdda_path(self):
        print('\ntype tdda')
        with os.popen('type tdda') as f:
            path = f.read()
        print(path)
        print('which tdda')
        with os.popen('which tdda') as f:
            path = f.read()
        print(path)

    def test_02_path(self):
        path = os.environ.get('PATH')
        print('PATH=%s' % path)
        print('COMPONENTS:')
        for p in path.split(':'):
            print(p)
        print()

    def test_03_pythonpath(self):
        print('PYTHON PATH:')
        for p in sys.path:
            print(p)

if __name__ == '__main__':
    unittest.main()

<commit_msg>More config test reports.<commit_after>import os
import sys
import unittest

class TestSystemConfig(unittest.TestCase):
    def test_01_tdda_path(self):
        print('\ntype tdda')
        with os.popen('type tdda') as f:
            path = f.read()
        print(path)
        print('which tdda')
        with os.popen('which tdda') as f:
            path = f.read()
        print(path)

    def test_02_path(self):
        path = os.environ.get('PATH')
        print('\nPATH=%s' % path)
        print('COMPONENTS:')
        for p in path.split(':'):
            print(p)
        print()

    def test_03_pythonpath(self):
        print('\nPYTHON PATH:')
        for p in sys.path:
            print(p)

    def test_04_numpy_pandas_versions(self):
        try:
            import numpy
            print('\nnumpy version:', numpy.__version__)
        except ImportError:
            print('numpy not found.')

        try:
            import pandas
            print('\npandas version:', pandas.__version__)
        except ImportError:
            print('pandas not found.')

if __name__ == '__main__':
    unittest.main()

<|endoftext|>


<commit_before>import logging
import re
from phovea_server.security import current_user

__author__ = 'Samuel Gratzl'
_log = logging.getLogger(__name__)
REGEX_TYPE = type(re.compile(''))

def _clean_query(query):
  if callable(query):
    return 'custom function'
  query = query.strip()
  query = query.replace('\n', '')
  # remove two spaces till there are no more
  while '  ' in query:
    query = query.replace('  ', ' ')
  return query

class DBFilterData(object):
  def __init__(self, group, sub_query, join):
    self.group = group
    self.sub_query = sub_query
    self.join = join

class DBView(object):
  def __init__(self, idtype=None, query=None):
    self.description = ''
    self.idtype = idtype
    self.query = query
    self.queries = {}
    self.columns = {}
    self.columns_filled_up = None
    self.replacements = []
    self.valid_replacements = {}
    self.arguments = []
    self.filters = {}
    self.table = None
    self.security = None
    self.assign_ids = False

  def needs_to_fill_up_columns(self):
    return self.columns_filled_up is False and self.table is not None

  def dump(self, name):
    from collections import OrderedDict
    r = OrderedDict(name=name, description=self.description)
    if self.idtype:
      r['idType'] = self.idtype
    r['query'] = _clean_query(self.query)
    args = [a for a in self.arguments]
    args.extend(self.replacements)
    r['arguments'] = args
    if self.columns:
      r['columns'] = self.columns.values()
    if self.filters:
      r['filters'] = self.filters.keys()
    if self.queries:
      r['queries'] = {k: _clean_query(v) for k, v in self.queries.items()}
    return r

  def is_valid_filter(self, key):
    if key in self.filters:
      return True
    if key in self.columns:
      return True
    # if not specified and not the columns completed
    return not self.filters and not self.columns_filled_up

  def get_filter_subquery(self, key):
    if key in self.filters and self.filters[key].sub_query is not None:
      return self.filters[key].sub_query
    if ('filter_' + key) in self.queries:  # compatibility
      return self.queries['filter_' + key]
    return key + ' {operator} {value}'

  def get_filter_group(self, key):
    if key in self.filters:
      return self.filters[key].group
    return None

  def get_filter_subjoin(self, key):
    if key in self.filters:
      return self.filters[key].join
    return None

  def filter_groups(self):
    r = set([v.group for v in self.filters.values()])
    if None in r:
      r.remove(None)
    return r

  def is_valid_replacement(self, key, value):
    if key not in self.replacements:
      return False
    if key not in self.valid_replacements:
      return True
    v = self.valid_replacements[key]
    if isinstance(v, list):
      return value in v
    if v == int or v == float:
      return type(value) == v
    if isinstance(v, REGEX_TYPE):
      return v.match(value)
    _log.info(u'unknown %s %s %s', key, value, v)
    return True

  def is_valid_argument(self, key):
    return key in self.arguments

  def can_access(self):
    if self.security is None:
      return True
    if callable(self.security):
      return self.security(current_user())
    role = unicode(self.security)
    return current_user().has_role(role)

class DBViewBuilder(object):
  """
  db view builder pattern implementation
  """

  def __init__(self):
    self.v = DBView()

  def clone(self, view):
    """
    initializes based on a given view
    :param view: the view to copy from
    :return: self
    """
    self.v.idtype = view.idtype
    self.v.description = view.description
    self.v.query = view.query
    self.v.queries = view.queries.copy()
    self.v.columns = view.columns.copy()
    self.v.replacements = list(view.replacements)
    self.v.arguments = list(view.arguments)
    self.v.filters = view.filters.copy()
    self.v.valid_replacements = view.valid_replacements.copy()
    self.v.security = view.security
    return self

  def description(self, desc):
    """
    optional description of this query
    :param desc: the description text
    :return: self
    """
    self.v.description = desc
    return self

  def assign_ids(self):
    """
    defines that before results of this view are returned the unique ids should be assigned based on the contained 'id' field
    :return: self
    """
    self.v.assign_ids = True
    return self

  def idtype(self, idtype):
    """
    specify the IDType of which the resulting rows are
    :param idtype: the idtype
    :return: self
    """
    self.v.idtype = idtype
    return self

  def table(self, table):
    """
    sets the base table name of this query e.g. for generating queries
    :param table: the table name
    :return: self
    """
    self.v.table = table
    return self

  def query(self, key, query=None):
    """
    sets or adds another query to this builder
    :param key: optional the key of this query if not given the default query will be set
    :param query: the sql query
    :return: self
    """
    if query is None:
      query = key
      self.v.query = query
    else:
      self.v.queries[key] = query
    return self

  def callback(self, key, callback=None):
    """
    instead of setting an SQL query setting a callback function that returns the result similar to executing the SQL query
    :param key: optional key
    :param callback: (engine: SQLAlchemyEngine, arguments: dict of query arguments, filters: dict of filters) => array of dicts
    :return:
    """
    if callback is None:
      callback = key
      self.v.query = callback
    else:
      self.v.queries[key] = callback
    return self

  def filters(self, keys, alias=None, table=None, group=None, join=None):
    """
    specify possible filter keys
    :param keys: the list of possible filters
    :param alias: shortcut for just specifying the left hand alias: {alias} {operator} {value}
    :param table: shortcut for just specifying the table alias: {table}.{key} {operator} {value}
    :param group: to inject the filter in another group and_group_where
    :param join: additional join to inject
    :return:
    """
    for key in keys:
      self.filter(key, alias=alias, table=table, group=group, join=join)
    return self

  def filter(self, key, replacement=None, alias=None, table=None, group=None, join=None):
    """
    add a possible filter with replacement strategy of type {key} {operator} {value}
    :param key: filter key
    :param replacement: optional the full replacement string has to inclue {operator} and {value}
    :param alias: shortcut for just specifying the left hand alias: {alias} {operator} {value}
    :param table: shortcut for just specifying the table alias: {table}.{key} {operator} {value}
    :param group: to inject the filter in another group and_group_where
    :param join: additional join to inject
    :return: self
    """
    if table is not None:
      alias = u'{}.{}'.format(table, key)
    if alias is not None:
      replacement = alias + ' {operator} {value}'
    self.v.filters[key] = DBFilterData(group, replacement, join)
    return self

  def append(self, key, query=None):
    """
    append something to the query
    :param key: optional key
    :param query: the subpart to append
    :return: self
    """
    if query is None:
      query = key
      self.v.query += query
    else:
      self.v.queries[key] += query
    return self

  def query_stats(self, query):
    """
    shortcut for query('stats', query)
    :param query: the query
    :return: self
    """
    self.v.queries['stats'] = query
    return self

  def query_categories(self, query):
    """
    shortcut for query('categories', query)
    :param query: the query
    :return: self
    """
    self.v.queries['categories'] = query
    return self

  def derive_columns(self):
    """
    specify that the columns should be automatically derived, requires that 'table' is given
    :return: self
    """
    self.v.columns_filled_up = False
    return self

  def column(self, name, **kwargs):
    """
    specify a column along with a type for the result
    :param name: name of column
    :param kwargs: additional attributes such as label and type
    :return: self
    """
    if 'label' not in kwargs:
      kwargs['label'] = name
    kwargs['column'] = name
    self.v.columns[name] = kwargs
    return self

  def replace(self, replace, valid_replacements=None):
    """
    specify to replace a certain piece of the query (using {replace}) within the query
    :param replace: the key to replace
    :param valid_replacements: optional validation to avoid SQL injection, possible values: a list of strings, a regex, int or float
    :return: self
    """
    self.v.replacements.append(replace)
    if valid_replacements is not None:
      self.v.valid_replacements[replace] = valid_replacements
    return self

  def arg(self, arg):
    """
    adds another argument of this query (using :arg) which will be replaced within SQL
    :param arg: the argument key
    :return: self
    """
    self.v.arguments.append(arg)
    return self

  def call(self, f=None):
    """
    shortcut for f(self)
    :param f: the function to call
    :return: self
    """
    if f is not None:
      f(self)
    return self

  def security(self, security_check):
    """
    adds a security check for this view
    :param security_check: either a string = role_name or a function that will get the user as first argument
    :return: self
    """
    self.v.security = security_check
    return self

  def build(self):
    """
    builds the query and end this builder
    :return: the built query
    """
    return self.v

def limit_offset(builder):
  """
  helper function to append the limit and offset suffix
  :param builder: the current query builder
  :return:
  """
  return builder.append(' LIMIT {limit} OFFSET {offset}').replace('limit', int).replace('offset', int).arg('query')

def inject_where_clause(builder, clause):
  """
    helper function to inject an additional where clause
    :param builder: the current builder
    :param clause: the clause to inject
    :return:
    """
  query = builder.v.query
  if callable(query):
    return builder
  lower = query.lower()
  index = lower.find(' where ')
  if index >= 0:
    index += len(' where ')  # get the end
    builder.query(u'{} ({}) AND {}'.format(query[:index], clause, query[index:]))
  else:
    before = -1
    for before_q in [' order by', ' group by', ' limit', ' offset']:
      before = lower.find(before_q)
      if before >= 0:
        break
    if before < 0:
      # append
      builder.append(' WHERE ').append(clause)
    else:
      builder.query(u'{} WHERE {} {}'.format(query[:index], clause, query[index:]))
  return builder

def inject_where(builder):
  """
  helper function to append to the query the generated where clause
  :param builder: the current builder
  :return:
  """
  query = builder.v.query
  if callable(query):
    return builder
  lower = query.lower()
  where = lower.find(' where ')
  before = -1
  for before_q in [' group by', ' order by', ' limit', ' offset']:
    before = lower.find(before_q)
    if before >= 0:
      break

  if where >= 0:
    if before < 0:
      builder.append(u' {and_where}')
    else:
      builder.query(u'{} {{and_where}} {}'.format(query[:before], query[before:]))
    builder.replace('and_where')
    query = builder.v.query
    builder.query(u'{} {{joins}} {}'.format(query[:where], query[where:]))
    builder.replace('joins')
  else:
    if before < 0:
      builder.append('{joins} {where}')
    else:
      builder.query(u'{} {{joins}} {{where}} {}'.format(query[:before], query[before:]))
    builder.replace('where')
    builder.replace('joins')
  return builder

def add_common_queries(queries, table, idtype, id_query, columns=None, call_function=None, prefix=None,
                       name_column='name'):
  """
  create a set of common queries
  :param queries: dict where the queries should be stored
  :param table: base table name
  :param idtype: idtype of the table
  :param id_query: the snippet to create the required 'id' column
  :param columns: a list of columns for validation
  :param call_function: another call function
  :param prefix: optional prefix instead of the table name
  :param name_column: name of the name column used to verify items
  :return: None
  """
  if prefix is None:
    prefix = table

  queries[prefix + '_items'] = DBViewBuilder().idtype(idtype).table(table).query(u"""
        SELECT {id}, {{column}} AS text
        FROM {table} WHERE LOWER({{column}}) LIKE :query
        ORDER BY {{column}} ASC""".format(id=id_query, table=table)).replace('column', columns).assign_ids().call(call_function).call(limit_offset).arg('query').build()

  queries[prefix + '_items_verify'] = DBViewBuilder().idtype(idtype).table(table).query(u"""
        SELECT {id}, {name} AS text
        FROM {table}""".format(id=id_query, table=table, name=name_column)).assign_ids().call(call_function).call(inject_where).filter(name_column, 'lower({name}) {{operator}} {{value}}'.format(name=name_column)).build()

  queries[prefix + '_unique'] = DBViewBuilder().query(u"""
        SELECT d as id, d as text
        FROM (
          SELECT distinct {{column}} AS d
          FROM {table} WHERE LOWER({{column}}) LIKE :query
          ) as t
        ORDER BY d ASC""".format(table=table)).replace('column', columns).call(limit_offset).arg('query').build()

  queries[prefix + '_unique_all'] = DBViewBuilder().query(u"""
        SELECT distinct {{column}} AS text
        FROM {table} ORDER BY {{column}} ASC """.format(table=table)).replace('column', columns).build()

"""
 default aggregation
"""
default_agg_score = DBViewBuilder().query('{agg}({data_subtype})').replace('agg', ['min', 'max', 'avg']).replace('data_subtype').build()

class DBMapping(object):
  """
  simple mapping based on a query of the form `select from_id as f, to_id as t from mapping_table where f in :ids`
  """

  def __init__(self, from_idtype, to_idtype, query):
    self.from_idtype = from_idtype
    self.to_idtype = to_idtype
    self.query = query

class DBConnector(object):
  """
  basic connector object
  """

  def __init__(self, views, agg_score=None, mappings=None):
    """
    :param views: the dict of query views
    :param agg_score: optional specify how aggregation should be handled
    :param mappings: optional database mappings
    """
    self.agg_score = agg_score or default_agg_score
    self.views = views
    self.dburl = None
    self.mappings = mappings
    self.statement_timeout = None
    self.statement_timeout_query = None
    self.description = ''

  def dump(self, name):
    from collections import OrderedDict
    return OrderedDict(name=name, description=self.description)
<commit_msg>use ordered dict to preserve column order<commit_after>import logging
import re
from collections import OrderedDict
from phovea_server.security import current_user

__author__ = 'Samuel Gratzl'
_log = logging.getLogger(__name__)
REGEX_TYPE = type(re.compile(''))

def _clean_query(query):
  if callable(query):
    return 'custom function'
  query = query.strip()
  query = query.replace('\n', '')
  # remove two spaces till there are no more
  while '  ' in query:
    query = query.replace('  ', ' ')
  return query

class DBFilterData(object):
  def __init__(self, group, sub_query, join):
    self.group = group
    self.sub_query = sub_query
    self.join = join

class DBView(object):
  def __init__(self, idtype=None, query=None):
    self.description = ''
    self.idtype = idtype
    self.query = query
    self.queries = {}
    self.columns = OrderedDict()
    self.columns_filled_up = None
    self.replacements = []
    self.valid_replacements = {}
    self.arguments = []
    self.filters = {}
    self.table = None
    self.security = None
    self.assign_ids = False

  def needs_to_fill_up_columns(self):
    return self.columns_filled_up is False and self.table is not None

  def dump(self, name):
    from collections import OrderedDict
    r = OrderedDict(name=name, description=self.description)
    if self.idtype:
      r['idType'] = self.idtype
    r['query'] = _clean_query(self.query)
    args = [a for a in self.arguments]
    args.extend(self.replacements)
    r['arguments'] = args
    if self.columns:
      r['columns'] = self.columns.values()
    if self.filters:
      r['filters'] = self.filters.keys()
    if self.queries:
      r['queries'] = {k: _clean_query(v) for k, v in self.queries.items()}
    return r

  def is_valid_filter(self, key):
    if key in self.filters:
      return True
    if key in self.columns:
      return True
    # if not specified and not the columns completed
    return not self.filters and not self.columns_filled_up

  def get_filter_subquery(self, key):
    if key in self.filters and self.filters[key].sub_query is not None:
      return self.filters[key].sub_query
    if ('filter_' + key) in self.queries:  # compatibility
      return self.queries['filter_' + key]
    return key + ' {operator} {value}'

  def get_filter_group(self, key):
    if key in self.filters:
      return self.filters[key].group
    return None

  def get_filter_subjoin(self, key):
    if key in self.filters:
      return self.filters[key].join
    return None

  def filter_groups(self):
    r = set([v.group for v in self.filters.values()])
    if None in r:
      r.remove(None)
    return r

  def is_valid_replacement(self, key, value):
    if key not in self.replacements:
      return False
    if key not in self.valid_replacements:
      return True
    v = self.valid_replacements[key]
    if isinstance(v, list):
      return value in v
    if v == int or v == float:
      return type(value) == v
    if isinstance(v, REGEX_TYPE):
      return v.match(value)
    _log.info(u'unknown %s %s %s', key, value, v)
    return True

  def is_valid_argument(self, key):
    return key in self.arguments

  def can_access(self):
    if self.security is None:
      return True
    if callable(self.security):
      return self.security(current_user())
    role = unicode(self.security)
    return current_user().has_role(role)

class DBViewBuilder(object):
  """
  db view builder pattern implementation
  """

  def __init__(self):
    self.v = DBView()

  def clone(self, view):
    """
    initializes based on a given view
    :param view: the view to copy from
    :return: self
    """
    self.v.idtype = view.idtype
    self.v.description = view.description
    self.v.query = view.query
    self.v.queries = view.queries.copy()
    self.v.columns = view.columns.copy()
    self.v.replacements = list(view.replacements)
    self.v.arguments = list(view.arguments)
    self.v.filters = view.filters.copy()
    self.v.valid_replacements = view.valid_replacements.copy()
    self.v.security = view.security
    return self

  def description(self, desc):
    """
    optional description of this query
    :param desc: the description text
    :return: self
    """
    self.v.description = desc
    return self

  def assign_ids(self):
    """
    defines that before results of this view are returned the unique ids should be assigned based on the contained 'id' field
    :return: self
    """
    self.v.assign_ids = True
    return self

  def idtype(self, idtype):
    """
    specify the IDType of which the resulting rows are
    :param idtype: the idtype
    :return: self
    """
    self.v.idtype = idtype
    return self

  def table(self, table):
    """
    sets the base table name of this query e.g. for generating queries
    :param table: the table name
    :return: self
    """
    self.v.table = table
    return self

  def query(self, key, query=None):
    """
    sets or adds another query to this builder
    :param key: optional the key of this query if not given the default query will be set
    :param query: the sql query
    :return: self
    """
    if query is None:
      query = key
      self.v.query = query
    else:
      self.v.queries[key] = query
    return self

  def callback(self, key, callback=None):
    """
    instead of setting an SQL query setting a callback function that returns the result similar to executing the SQL query
    :param key: optional key
    :param callback: (engine: SQLAlchemyEngine, arguments: dict of query arguments, filters: dict of filters) => array of dicts
    :return:
    """
    if callback is None:
      callback = key
      self.v.query = callback
    else:
      self.v.queries[key] = callback
    return self

  def filters(self, keys, alias=None, table=None, group=None, join=None):
    """
    specify possible filter keys
    :param keys: the list of possible filters
    :param alias: shortcut for just specifying the left hand alias: {alias} {operator} {value}
    :param table: shortcut for just specifying the table alias: {table}.{key} {operator} {value}
    :param group: to inject the filter in another group and_group_where
    :param join: additional join to inject
    :return:
    """
    for key in keys:
      self.filter(key, alias=alias, table=table, group=group, join=join)
    return self

  def filter(self, key, replacement=None, alias=None, table=None, group=None, join=None):
    """
    add a possible filter with replacement strategy of type {key} {operator} {value}
    :param key: filter key
    :param replacement: optional the full replacement string has to inclue {operator} and {value}
    :param alias: shortcut for just specifying the left hand alias: {alias} {operator} {value}
    :param table: shortcut for just specifying the table alias: {table}.{key} {operator} {value}
    :param group: to inject the filter in another group and_group_where
    :param join: additional join to inject
    :return: self
    """
    if table is not None:
      alias = u'{}.{}'.format(table, key)
    if alias is not None:
      replacement = alias + ' {operator} {value}'
    self.v.filters[key] = DBFilterData(group, replacement, join)
    return self

  def append(self, key, query=None):
    """
    append something to the query
    :param key: optional key
    :param query: the subpart to append
    :return: self
    """
    if query is None:
      query = key
      self.v.query += query
    else:
      self.v.queries[key] += query
    return self

  def query_stats(self, query):
    """
    shortcut for query('stats', query)
    :param query: the query
    :return: self
    """
    self.v.queries['stats'] = query
    return self

  def query_categories(self, query):
    """
    shortcut for query('categories', query)
    :param query: the query
    :return: self
    """
    self.v.queries['categories'] = query
    return self

  def derive_columns(self):
    """
    specify that the columns should be automatically derived, requires that 'table' is given
    :return: self
    """
    self.v.columns_filled_up = False
    return self

  def column(self, name, **kwargs):
    """
    specify a column along with a type for the result
    :param name: name of column
    :param kwargs: additional attributes such as label and type
    :return: self
    """
    if 'label' not in kwargs:
      kwargs['label'] = name
    kwargs['column'] = name
    self.v.columns[name] = kwargs
    return self

  def replace(self, replace, valid_replacements=None):
    """
    specify to replace a certain piece of the query (using {replace}) within the query
    :param replace: the key to replace
    :param valid_replacements: optional validation to avoid SQL injection, possible values: a list of strings, a regex, int or float
    :return: self
    """
    self.v.replacements.append(replace)
    if valid_replacements is not None:
      self.v.valid_replacements[replace] = valid_replacements
    return self

  def arg(self, arg):
    """
    adds another argument of this query (using :arg) which will be replaced within SQL
    :param arg: the argument key
    :return: self
    """
    self.v.arguments.append(arg)
    return self

  def call(self, f=None):
    """
    shortcut for f(self)
    :param f: the function to call
    :return: self
    """
    if f is not None:
      f(self)
    return self

  def security(self, security_check):
    """
    adds a security check for this view
    :param security_check: either a string = role_name or a function that will get the user as first argument
    :return: self
    """
    self.v.security = security_check
    return self

  def build(self):
    """
    builds the query and end this builder
    :return: the built query
    """
    return self.v

def limit_offset(builder):
  """
  helper function to append the limit and offset suffix
  :param builder: the current query builder
  :return:
  """
  return builder.append(' LIMIT {limit} OFFSET {offset}').replace('limit', int).replace('offset', int).arg('query')

def inject_where_clause(builder, clause):
  """
    helper function to inject an additional where clause
    :param builder: the current builder
    :param clause: the clause to inject
    :return:
    """
  query = builder.v.query
  if callable(query):
    return builder
  lower = query.lower()
  index = lower.find(' where ')
  if index >= 0:
    index += len(' where ')  # get the end
    builder.query(u'{} ({}) AND {}'.format(query[:index], clause, query[index:]))
  else:
    before = -1
    for before_q in [' order by', ' group by', ' limit', ' offset']:
      before = lower.find(before_q)
      if before >= 0:
        break
    if before < 0:
      # append
      builder.append(' WHERE ').append(clause)
    else:
      builder.query(u'{} WHERE {} {}'.format(query[:index], clause, query[index:]))
  return builder

def inject_where(builder):
  """
  helper function to append to the query the generated where clause
  :param builder: the current builder
  :return:
  """
  query = builder.v.query
  if callable(query):
    return builder
  lower = query.lower()
  where = lower.find(' where ')
  before = -1
  for before_q in [' group by', ' order by', ' limit', ' offset']:
    before = lower.find(before_q)
    if before >= 0:
      break

  if where >= 0:
    if before < 0:
      builder.append(u' {and_where}')
    else:
      builder.query(u'{} {{and_where}} {}'.format(query[:before], query[before:]))
    builder.replace('and_where')
    query = builder.v.query
    builder.query(u'{} {{joins}} {}'.format(query[:where], query[where:]))
    builder.replace('joins')
  else:
    if before < 0:
      builder.append('{joins} {where}')
    else:
      builder.query(u'{} {{joins}} {{where}} {}'.format(query[:before], query[before:]))
    builder.replace('where')
    builder.replace('joins')
  return builder

def add_common_queries(queries, table, idtype, id_query, columns=None, call_function=None, prefix=None,
                       name_column='name'):
  """
  create a set of common queries
  :param queries: dict where the queries should be stored
  :param table: base table name
  :param idtype: idtype of the table
  :param id_query: the snippet to create the required 'id' column
  :param columns: a list of columns for validation
  :param call_function: another call function
  :param prefix: optional prefix instead of the table name
  :param name_column: name of the name column used to verify items
  :return: None
  """
  if prefix is None:
    prefix = table

  queries[prefix + '_items'] = DBViewBuilder().idtype(idtype).table(table).query(u"""
        SELECT {id}, {{column}} AS text
        FROM {table} WHERE LOWER({{column}}) LIKE :query
        ORDER BY {{column}} ASC""".format(id=id_query, table=table)).replace('column', columns).assign_ids().call(call_function).call(limit_offset).arg('query').build()

  queries[prefix + '_items_verify'] = DBViewBuilder().idtype(idtype).table(table).query(u"""
        SELECT {id}, {name} AS text
        FROM {table}""".format(id=id_query, table=table, name=name_column)).assign_ids().call(call_function).call(inject_where).filter(name_column, 'lower({name}) {{operator}} {{value}}'.format(name=name_column)).build()

  queries[prefix + '_unique'] = DBViewBuilder().query(u"""
        SELECT d as id, d as text
        FROM (
          SELECT distinct {{column}} AS d
          FROM {table} WHERE LOWER({{column}}) LIKE :query
          ) as t
        ORDER BY d ASC""".format(table=table)).replace('column', columns).call(limit_offset).arg('query').build()

  queries[prefix + '_unique_all'] = DBViewBuilder().query(u"""
        SELECT distinct {{column}} AS text
        FROM {table} ORDER BY {{column}} ASC """.format(table=table)).replace('column', columns).build()

"""
 default aggregation
"""
default_agg_score = DBViewBuilder().query('{agg}({data_subtype})').replace('agg', ['min', 'max', 'avg']).replace('data_subtype').build()

class DBMapping(object):
  """
  simple mapping based on a query of the form `select from_id as f, to_id as t from mapping_table where f in :ids`
  """

  def __init__(self, from_idtype, to_idtype, query):
    self.from_idtype = from_idtype
    self.to_idtype = to_idtype
    self.query = query

class DBConnector(object):
  """
  basic connector object
  """

  def __init__(self, views, agg_score=None, mappings=None):
    """
    :param views: the dict of query views
    :param agg_score: optional specify how aggregation should be handled
    :param mappings: optional database mappings
    """
    self.agg_score = agg_score or default_agg_score
    self.views = views
    self.dburl = None
    self.mappings = mappings
    self.statement_timeout = None
    self.statement_timeout_query = None
    self.description = ''

  def dump(self, name):
    from collections import OrderedDict
    return OrderedDict(name=name, description=self.description)
<|endoftext|>


<commit_before>"""
tea-party 'party' class.
"""

import os
import yaml

from functools import wraps

from tea_party.log import LOGGER
from tea_party.attendee import Attendee, make_attendees
from tea_party.path import read_path, rmdir
from tea_party.defaults import *
from tea_party.fetchers.callbacks import ProgressBarFetcherCallback
from tea_party.unpackers.callbacks import ProgressBarUnpackerCallback

def load_party_file(path):
    """
    Create a Party instance from a party-file.

    `path` must be a valid party-file name.
    """

    LOGGER.debug('Opening party-file at %s...' % path)

    with open(path) as party_file:
        data = party_file.read()

    values = yaml.load(data)

    party = Party(
        path=path,
        cache_path=values.get('cache'),
        build_path=values.get('build'),
    )

    party.attendees = make_attendees(party, values.get('attendees'))

    return party

class NoSuchAttendeeError(ValueError):

    """
    The specified attendee does not exist.
    """

    def __init__(self, attendee):
        """
        Create an NoSuchAttendeeError for the specified `attendee`.
        """

        super(NoSuchAttendeeError, self).__init__(
            'No such attendee: %s' % attendee
        )

def has_attendees(func):
    """
    Ensures the function has real attendees instances as its `attendees`
    parameter.
    """

    @wraps(func)
    def result(self, *args, **kwargs):
        attendees = kwargs.get('attendees', [])

        if not attendees:
            attendees = self.attendees
        else:
            attendees = map(self.get_attendee_by_name, attendees)

        kwargs['attendees'] = attendees

        return func(self, *args, **kwargs)

    return result

class Party(object):

    """
    A party object is the root object that stores all the information about the
    different attendees (third-party softwares), and the party options.
    """

    def __init__(self, path, cache_path, build_path, **kwargs):
        """
        Create a Party instance.

        `path` is the path to the party file.
        `cache_path` is the root of the cache.
        `build_path` is the root of the build.
        """

        self.path = os.path.abspath(path)
        self.attendees = []
        self.cache_path = read_path(cache_path, os.path.dirname(self.path), DEFAULT_CACHE_PATH)
        self.build_path = read_path(build_path, os.path.dirname(self.path), DEFAULT_BUILD_PATH)
        self.auto_fetch = True
        self.fetcher_callback_class = ProgressBarFetcherCallback
        self.unpacker_callback_class = ProgressBarUnpackerCallback

    def get_attendee_by_name(self, name):
        """
        Get an attendee by name, if it exists.

        If no attendee has the specified name, a NoSuchAttendeeError is raised.
        """

        if isinstance(name, Attendee):
            return name

        for attendee in self.attendees:
            if attendee.name == name:
                return attendee

        raise NoSuchAttendeeError(attendee=attendee)

    @has_attendees
    def clean_cache(self, attendees=[]):
        """
        Clean the cache, optionally for a given list of `attendees` to recover
        disk space.
        """

        map(Attendee.clean_cache, attendees)

    @has_attendees
    def clean_build(self, attendees=[]):
        """
        Clean the build, optionally for a given list of `attendees` to recover
        disk space.
        """

        map(Attendee.clean_build, attendees)

    @has_attendees
    def fetch(self, attendees=[], force=False):
        """
        Fetch the archives.
        """

        if force:
            map(lambda x: x.clean_cache(), attendees)

        attendees_to_fetch = [x for x in attendees if not x.fetched]

        if not attendees_to_fetch:
            LOGGER.info('None of the %s archive(s) needs fetching.', len(attendees))

        else:
            LOGGER.info("Fetching %s/%s archive(s)...", len(attendees_to_fetch), len(self.attendees))

            map(Attendee.fetch, attendees_to_fetch)

            LOGGER.info("Done fetching archives.")

    @has_attendees
    def unpack(self, attendees=[], force=False):
        """
        Unpack the archives.
        """

        if force:
            map(lambda x: x.clean_source(), self.attendees)

        attendees_to_unpack = [x for x in attendees if not x.unpacked]

        if not attendees_to_unpack:
            LOGGER.info('None of the %s archive(s) needs unpacking.', len(self.attendees))

        else:
            LOGGER.info("Unpacking %s/%s archive(s)...", len(attendees_to_unpack), len(self.attendees))

            map(Attendee.unpack, attendees_to_unpack)

            LOGGER.info("Done unpacking archives.")
<commit_msg>Fixed incorrect method call<commit_after>"""
tea-party 'party' class.
"""

import os
import yaml

from functools import wraps

from tea_party.log import LOGGER
from tea_party.attendee import Attendee, make_attendees
from tea_party.path import read_path, rmdir
from tea_party.defaults import *
from tea_party.fetchers.callbacks import ProgressBarFetcherCallback
from tea_party.unpackers.callbacks import ProgressBarUnpackerCallback

def load_party_file(path):
    """
    Create a Party instance from a party-file.

    `path` must be a valid party-file name.
    """

    LOGGER.debug('Opening party-file at %s...' % path)

    with open(path) as party_file:
        data = party_file.read()

    values = yaml.load(data)

    party = Party(
        path=path,
        cache_path=values.get('cache'),
        build_path=values.get('build'),
    )

    party.attendees = make_attendees(party, values.get('attendees'))

    return party

class NoSuchAttendeeError(ValueError):

    """
    The specified attendee does not exist.
    """

    def __init__(self, attendee):
        """
        Create an NoSuchAttendeeError for the specified `attendee`.
        """

        super(NoSuchAttendeeError, self).__init__(
            'No such attendee: %s' % attendee
        )

def has_attendees(func):
    """
    Ensures the function has real attendees instances as its `attendees`
    parameter.
    """

    @wraps(func)
    def result(self, *args, **kwargs):
        attendees = kwargs.get('attendees', [])

        if not attendees:
            attendees = self.attendees
        else:
            attendees = map(self.get_attendee_by_name, attendees)

        kwargs['attendees'] = attendees

        return func(self, *args, **kwargs)

    return result

class Party(object):

    """
    A party object is the root object that stores all the information about the
    different attendees (third-party softwares), and the party options.
    """

    def __init__(self, path, cache_path, build_path, **kwargs):
        """
        Create a Party instance.

        `path` is the path to the party file.
        `cache_path` is the root of the cache.
        `build_path` is the root of the build.
        """

        self.path = os.path.abspath(path)
        self.attendees = []
        self.cache_path = read_path(cache_path, os.path.dirname(self.path), DEFAULT_CACHE_PATH)
        self.build_path = read_path(build_path, os.path.dirname(self.path), DEFAULT_BUILD_PATH)
        self.auto_fetch = True
        self.fetcher_callback_class = ProgressBarFetcherCallback
        self.unpacker_callback_class = ProgressBarUnpackerCallback

    def get_attendee_by_name(self, name):
        """
        Get an attendee by name, if it exists.

        If no attendee has the specified name, a NoSuchAttendeeError is raised.
        """

        if isinstance(name, Attendee):
            return name

        for attendee in self.attendees:
            if attendee.name == name:
                return attendee

        raise NoSuchAttendeeError(attendee=attendee)

    @has_attendees
    def clean_cache(self, attendees=[]):
        """
        Clean the cache, optionally for a given list of `attendees` to recover
        disk space.
        """

        map(Attendee.clean_cache, attendees)

    @has_attendees
    def clean_build(self, attendees=[]):
        """
        Clean the build, optionally for a given list of `attendees` to recover
        disk space.
        """

        map(Attendee.clean_build, attendees)

    @has_attendees
    def fetch(self, attendees=[], force=False):
        """
        Fetch the archives.
        """

        if force:
            map(lambda x: x.clean_cache(), attendees)

        attendees_to_fetch = [x for x in attendees if not x.fetched]

        if not attendees_to_fetch:
            LOGGER.info('None of the %s archive(s) needs fetching.', len(attendees))

        else:
            LOGGER.info("Fetching %s/%s archive(s)...", len(attendees_to_fetch), len(self.attendees))

            map(Attendee.fetch, attendees_to_fetch)

            LOGGER.info("Done fetching archives.")

    @has_attendees
    def unpack(self, attendees=[], force=False):
        """
        Unpack the archives.
        """

        if force:
            map(lambda x: x.clean_build(), self.attendees)

        attendees_to_unpack = [x for x in attendees if not x.unpacked]

        if not attendees_to_unpack:
            LOGGER.info('None of the %s archive(s) needs unpacking.', len(self.attendees))

        else:
            LOGGER.info("Unpacking %s/%s archive(s)...", len(attendees_to_unpack), len(self.attendees))

            map(Attendee.unpack, attendees_to_unpack)

            LOGGER.info("Done unpacking archives.")
<|endoftext|>


<commit_before>from copy import deepcopy
import jsonschema
from decimal import Decimal
import os

from winnow.options import OptionsSet
from winnow import inline
from winnow import utils
from winnow import validation
from winnow.exceptions import OptionsExceptionFailedValidation, OptionsExceptionReferenceError
from winnow.constants import *

def add_doc(target, doc):
    _set_doc(target,  deepcopy(doc))

def allows(source_a, source_b):
    options_a = OptionsSet(source_a.get_options_dict())
    options_b = OptionsSet(source_b.get_options_dict())
    return options_a.allows(options_b)

def merge(source_a, source_b, target, doc):
    doc_b = source_b.get_doc()

    # get the options from bothe sources
    options_a = deepcopy(source_a.get_options_dict())
    options_b = deepcopy(source_b.get_options_dict())

    # expand the options dicts collecting their replaced refs
    ref_hashes = {}
    inline.inline_refs(options_a, source_a, ref_hashes)
    inline.inline_refs(options_b, source_a, ref_hashes)

    # do the merge
    options_a = OptionsSet(options_a)
    options_b = OptionsSet(options_b)
    merged_options = options_a.merge(options_b).store

    # un merge unchanged refs by looking at the ref_hashes
    inline.restore_unchanged_refs(merged_options, ref_hashes)

    # put this merged options into a copy of the doc
    new_doc = deepcopy(doc)
    new_doc[OPTIONS_KEY] = merged_options

    target.clone_history_from(source_a)
    _add_start_if_needed(source_a, target)
    _set_doc(target, new_doc)

    target.add_history_action(action=HISTORY_ACTION_MERGE,
                              input=source_b,
                              output_type=doc.get("type"))

def scope(source, scopes, target, doc):
    new_doc = deepcopy(doc)
    _trim_out_off_scope(new_doc[OPTIONS_KEY], set(scopes))
    target.clone_history_from(source)
    _add_start_if_needed(source, target)
    _set_doc(target, new_doc)

    target.add_history_action(action=HISTORY_ACTION_SCOPE,
                              scope=scopes,
                              output_type=doc.get("type"))

def default_choices(source, scopes):

    #take a copy of the options
    options_dict = deepcopy(source.get_options_dict())

    #expand it
    ref_hashes = {}
    inline.inline_refs(options_dict, source, ref_hashes)

    #scope it
    _trim_out_off_scope(options_dict, set(scopes))

    # wrap it in an options set
    options_set = OptionsSet(options_dict)

    #get default options set
    default = options_set.default()

    return default.store

def quantify(source, target, doc):

    quantity_options = {
        u"quantity": {
            u"type": u"numeric::step",
            u"name": u"Quantity",
            u"default": Decimal("1"),
            u"max": Decimal("10000"),
            u"min": Decimal("1"),
            u"start": Decimal("0"),
            u"step": Decimal("1")
        }
    }

    options_a = OptionsSet(source.get_options_dict())
    options_b = OptionsSet(quantity_options)
    new_doc = deepcopy(doc)
    new_doc[OPTIONS_KEY] = options_a.merge(options_b).store
    target.clone_history_from(source)
    _add_start_if_needed(source, target)
    _set_doc(target, new_doc)

    target.add_history_action(action=HISTORY_ACTION_QUANTIFY,
                              output_type=doc.get("type"))

def _trim_out_off_scope(node, scopes):

    if isinstance(node, dict):
        for key in node.keys():
            child = node[key]
            if isinstance(child, dict):
                if "scopes" in child.keys():
                    child_scopes = set(child["scopes"])
                    if scopes.isdisjoint(child_scopes):
                        del node[key]
                else:
                    _trim_out_off_scope(child, scopes)
            if isinstance(child, list):
                _trim_out_off_scope(child, scopes)

    # recursively trim inside lists
    if isinstance(node, list):
        for i, child in enumerate(node[:]):
            if isinstance(child, dict):
                if "scopes" in child.keys():
                    child_scopes = set(child["scopes"])
                    if scopes.isdisjoint(child_scopes):
                        node.remove(child)
                else:
                    _trim_out_off_scope(child, scopes)
            if isinstance(child, list):
                _trim_out_off_scope(child, scopes)

def filter_allows(filter_source, possible):
    filter_options = OptionsSet(filter_source.get_options_dict())
    return [p for p in possible if filter_options.allows(OptionsSet(p.get_options_dict()))]

def filter_allowed_by(filter_source, possible):
    filter_options = OptionsSet(filter_source.get_options_dict())
    return [p for p in possible if OptionsSet(p.get_options_dict()).allows(filter_options)]

def is_allowed_by(filter_source, possible):
    filter_options = OptionsSet(filter_source.get_options_dict())
    return OptionsSet(possible.get_options_dict()).allows(filter_options)

def expand(source, target):
    new_doc = deepcopy(source.get_doc())
    target.clone_history_from(source)
    ## inline references
    ref_hashes = {}
    inline.inline_refs(new_doc, source, ref_hashes)
    _set_doc(target, new_doc)
    return ref_hashes

def _patch_upstream(source, target, options_set):

    doc = source.get_doc()
    if u"upstream" in doc.keys():
        upstream_delegate = source.get_upstream()
        if upstream_delegate is None:
            raise OptionsExceptionReferenceError("Winnow Reference Error: Cannot find upstream reference %s" % doc[u"upstream"])
    else:
        return options_set

    upstream_options = OptionsSet(upstream_delegate.get_options_dict())
    patched_options_set = options_set.patch(upstream_options)
    if target is not None:
        _add_start_if_needed(source, target)
        target.add_history_action(action=HISTORY_ACTION_PATCH,
                              input=upstream_delegate,
                              output_type=doc.get("type"))

    return _patch_upstream(upstream_delegate, target, patched_options_set)

def _add_start_if_needed(source, target):
    if target.history_is_empty():
        doc = source.get_doc()

        target.add_history_action(action=HISTORY_ACTION_START,
                              input=source,
                              output_type=doc.get("type"))

def asset_props(doc, dl_base=None):
    path = doc.get("path")
    if path is None:
        return []
    relpaths = []
    _walk_dict_for_assets(doc, relpaths)
    return [asset_from_relpath(doc, dl_base=dl_base, rp) for rp in relpaths]

def asset_from_relpath(doc, dl_base=None, relpath):
    return {
        "source": doc['source'],
        "base": doc.get("base", dl_base),
        "path": os.path.normpath("%s/%s" % (doc['path'], relpath)),
        "relpath": relpath
    }

def _walk_dict_for_assets(node, found):

    if isinstance(node, dict):
        if u"asset" in node.keys():
            found.append(node[u"asset"])
        else:
            for k, v in node.iteritems():
                _walk_dict_for_assets(v, found)
    if isinstance(node, list):
        for v in node:
            _walk_dict_for_assets(v, found)

def validate(doc):
    try:
        validation.validate(doc)
    except jsonschema.ValidationError, e:
        raise OptionsExceptionFailedValidation(e)

def _set_doc(target, doc):
    validate(doc)
    target.set_doc(doc)
    target.set_doc_hash(utils.get_doc_hash(utils.json_dumps(doc)))
<commit_msg>ops: fix kw order bug.<commit_after>from copy import deepcopy
import jsonschema
from decimal import Decimal
import os

from winnow.options import OptionsSet
from winnow import inline
from winnow import utils
from winnow import validation
from winnow.exceptions import OptionsExceptionFailedValidation, OptionsExceptionReferenceError
from winnow.constants import *

def add_doc(target, doc):
    _set_doc(target,  deepcopy(doc))

def allows(source_a, source_b):
    options_a = OptionsSet(source_a.get_options_dict())
    options_b = OptionsSet(source_b.get_options_dict())
    return options_a.allows(options_b)

def merge(source_a, source_b, target, doc):
    doc_b = source_b.get_doc()

    # get the options from bothe sources
    options_a = deepcopy(source_a.get_options_dict())
    options_b = deepcopy(source_b.get_options_dict())

    # expand the options dicts collecting their replaced refs
    ref_hashes = {}
    inline.inline_refs(options_a, source_a, ref_hashes)
    inline.inline_refs(options_b, source_a, ref_hashes)

    # do the merge
    options_a = OptionsSet(options_a)
    options_b = OptionsSet(options_b)
    merged_options = options_a.merge(options_b).store

    # un merge unchanged refs by looking at the ref_hashes
    inline.restore_unchanged_refs(merged_options, ref_hashes)

    # put this merged options into a copy of the doc
    new_doc = deepcopy(doc)
    new_doc[OPTIONS_KEY] = merged_options

    target.clone_history_from(source_a)
    _add_start_if_needed(source_a, target)
    _set_doc(target, new_doc)

    target.add_history_action(action=HISTORY_ACTION_MERGE,
                              input=source_b,
                              output_type=doc.get("type"))

def scope(source, scopes, target, doc):
    new_doc = deepcopy(doc)
    _trim_out_off_scope(new_doc[OPTIONS_KEY], set(scopes))
    target.clone_history_from(source)
    _add_start_if_needed(source, target)
    _set_doc(target, new_doc)

    target.add_history_action(action=HISTORY_ACTION_SCOPE,
                              scope=scopes,
                              output_type=doc.get("type"))

def default_choices(source, scopes):

    #take a copy of the options
    options_dict = deepcopy(source.get_options_dict())

    #expand it
    ref_hashes = {}
    inline.inline_refs(options_dict, source, ref_hashes)

    #scope it
    _trim_out_off_scope(options_dict, set(scopes))

    # wrap it in an options set
    options_set = OptionsSet(options_dict)

    #get default options set
    default = options_set.default()

    return default.store

def quantify(source, target, doc):

    quantity_options = {
        u"quantity": {
            u"type": u"numeric::step",
            u"name": u"Quantity",
            u"default": Decimal("1"),
            u"max": Decimal("10000"),
            u"min": Decimal("1"),
            u"start": Decimal("0"),
            u"step": Decimal("1")
        }
    }

    options_a = OptionsSet(source.get_options_dict())
    options_b = OptionsSet(quantity_options)
    new_doc = deepcopy(doc)
    new_doc[OPTIONS_KEY] = options_a.merge(options_b).store
    target.clone_history_from(source)
    _add_start_if_needed(source, target)
    _set_doc(target, new_doc)

    target.add_history_action(action=HISTORY_ACTION_QUANTIFY,
                              output_type=doc.get("type"))

def _trim_out_off_scope(node, scopes):

    if isinstance(node, dict):
        for key in node.keys():
            child = node[key]
            if isinstance(child, dict):
                if "scopes" in child.keys():
                    child_scopes = set(child["scopes"])
                    if scopes.isdisjoint(child_scopes):
                        del node[key]
                else:
                    _trim_out_off_scope(child, scopes)
            if isinstance(child, list):
                _trim_out_off_scope(child, scopes)

    # recursively trim inside lists
    if isinstance(node, list):
        for i, child in enumerate(node[:]):
            if isinstance(child, dict):
                if "scopes" in child.keys():
                    child_scopes = set(child["scopes"])
                    if scopes.isdisjoint(child_scopes):
                        node.remove(child)
                else:
                    _trim_out_off_scope(child, scopes)
            if isinstance(child, list):
                _trim_out_off_scope(child, scopes)

def filter_allows(filter_source, possible):
    filter_options = OptionsSet(filter_source.get_options_dict())
    return [p for p in possible if filter_options.allows(OptionsSet(p.get_options_dict()))]

def filter_allowed_by(filter_source, possible):
    filter_options = OptionsSet(filter_source.get_options_dict())
    return [p for p in possible if OptionsSet(p.get_options_dict()).allows(filter_options)]

def is_allowed_by(filter_source, possible):
    filter_options = OptionsSet(filter_source.get_options_dict())
    return OptionsSet(possible.get_options_dict()).allows(filter_options)

def expand(source, target):
    new_doc = deepcopy(source.get_doc())
    target.clone_history_from(source)
    ## inline references
    ref_hashes = {}
    inline.inline_refs(new_doc, source, ref_hashes)
    _set_doc(target, new_doc)
    return ref_hashes

def _patch_upstream(source, target, options_set):

    doc = source.get_doc()
    if u"upstream" in doc.keys():
        upstream_delegate = source.get_upstream()
        if upstream_delegate is None:
            raise OptionsExceptionReferenceError("Winnow Reference Error: Cannot find upstream reference %s" % doc[u"upstream"])
    else:
        return options_set

    upstream_options = OptionsSet(upstream_delegate.get_options_dict())
    patched_options_set = options_set.patch(upstream_options)
    if target is not None:
        _add_start_if_needed(source, target)
        target.add_history_action(action=HISTORY_ACTION_PATCH,
                              input=upstream_delegate,
                              output_type=doc.get("type"))

    return _patch_upstream(upstream_delegate, target, patched_options_set)

def _add_start_if_needed(source, target):
    if target.history_is_empty():
        doc = source.get_doc()

        target.add_history_action(action=HISTORY_ACTION_START,
                              input=source,
                              output_type=doc.get("type"))

def asset_props(doc, dl_base=None):
    path = doc.get("path")
    if path is None:
        return []
    relpaths = []
    _walk_dict_for_assets(doc, relpaths)
    return [asset_from_relpath(doc, rp, dl_base=dl_base) for rp in relpaths]

def asset_from_relpath(doc, relpath, dl_base=None):
    return {
        "source": doc['source'],
        "base": doc.get("base", dl_base),
        "path": os.path.normpath("%s/%s" % (doc['path'], relpath)),
        "relpath": relpath
    }

def _walk_dict_for_assets(node, found):

    if isinstance(node, dict):
        if u"asset" in node.keys():
            found.append(node[u"asset"])
        else:
            for k, v in node.iteritems():
                _walk_dict_for_assets(v, found)
    if isinstance(node, list):
        for v in node:
            _walk_dict_for_assets(v, found)

def validate(doc):
    try:
        validation.validate(doc)
    except jsonschema.ValidationError, e:
        raise OptionsExceptionFailedValidation(e)

def _set_doc(target, doc):
    validate(doc)
    target.set_doc(doc)
    target.set_doc_hash(utils.get_doc_hash(utils.json_dumps(doc)))
<|endoftext|>