<commit_before>from django.utils.safestring import mark_safe
from corehq.apps.accounting.dispatcher import AccountingAdminInterfaceDispatcher, SubscriptionAdminInterfaceDispatcher
from corehq.apps.accounting.models import BillingAccount, Subscription
from corehq.apps.announcements.forms import HQAnnouncementForm
from corehq.apps.announcements.models import HQAnnouncement
from corehq.apps.crud.interface import BaseCRUDAdminInterface
from corehq.apps.reports.datatables import DataTablesHeader, DataTablesColumn

class AccountingInterface(BaseCRUDAdminInterface):
    section_name = "Accounting"
    base_template = 'reports/base_template.html'
    dispatcher = AccountingAdminInterfaceDispatcher

    crud_form_update_url = "/accounting/form/"

    def validate_document_class(self):
        return True
        #if self.document_class is None or not issubclass(self.document_class, HQAnnouncement):
        #    raise NotImplementedError("document_class must be an HQAnnouncement")

    """
    @property
    def default_report_url(self):
        return reverse("default_announcement_admin")
    """

    @property
    def headers(self):
        return DataTablesHeader(
            DataTablesColumn("Client/Account Name"),
            DataTablesColumn("Billing Contact Name (Email)"),
            DataTablesColumn("Plan Credit"),
            DataTablesColumn("SMS Credit"),
            DataTablesColumn("Users Credit"),
            DataTablesColumn("Account Balance"),
            DataTablesColumn("Active Subscriptions"),
            DataTablesColumn("Edit"),
        )

    @property
    def rows(self):
        rows = []
        for account in BillingAccount.objects.all():
            rows.append([account.name,
                         account.web_user_contact,
                         3,
                         4,
                         5,
                         account.balance,
                         Subscription.objects.filter(account=account,
                                                     is_active=True).count(),
                         mark_safe('Edit' % account.id)])
        return rows

    #######

    name = "View Billing Accounts"
    description = "description of view billing accounts"
    slug = "accounts"

    document_class = HQAnnouncement
    form_class = HQAnnouncementForm

    crud_item_type = "Billing Account"

class SubscriptionInterface(BaseCRUDAdminInterface):
    section_name = "Accounting"
    base_template = 'reports/base_template.html'
    dispatcher = SubscriptionAdminInterfaceDispatcher

    crud_form_update_url = "/accounting/form/"

    def validate_document_class(self):
        return True
        #if self.document_class is None or not issubclass(self.document_class, HQAnnouncement):
        #    raise NotImplementedError("document_class must be an HQAnnouncement")

    """
    @property
    def default_report_url(self):
        return reverse("default_announcement_admin")
    """

    @property
    def headers(self):
        return DataTablesHeader(
            DataTablesColumn("Project Space"),
            DataTablesColumn("Client Name"),
            DataTablesColumn("Version"),
            DataTablesColumn("Description"),
            DataTablesColumn("Account Name"),
            DataTablesColumn("Start Date"),
            DataTablesColumn("End Date"),
            DataTablesColumn("Active Status"),
            DataTablesColumn("Visibility"),
            DataTablesColumn("Is Default for Product?"),
            DataTablesColumn("Edit"),
        )

    @property
    def rows(self):
        rows = []
        for subscription in Subscription.objects.all():
            rows.append([subscription.subscriber.domain,
                         subscription.account.name,
                         subscription.plan.plan.name,
                         subscription.plan.plan.description,
                         mark_safe('%s'
                                   % (subscription.account.id, subscription.account.name)),
                         subscription.date_start,
                         subscription.date_end,
                         subscription.is_active,
                         subscription.plan.plan.visibility,
                         "MISSING VALUE",
                         mark_safe('Edit' % subscription.id)])
        return rows

    #######

    name = "View Subscriptions"
    description = "description of view subscriptions"
    slug = "subscriptions"

    document_class = HQAnnouncement
    form_class = HQAnnouncementForm

    crud_item_type = "Subscription"
<commit_msg>fixing html<commit_after>from django.utils.safestring import mark_safe
from corehq.apps.accounting.dispatcher import AccountingAdminInterfaceDispatcher, SubscriptionAdminInterfaceDispatcher
from corehq.apps.accounting.models import BillingAccount, Subscription
from corehq.apps.announcements.forms import HQAnnouncementForm
from corehq.apps.announcements.models import HQAnnouncement
from corehq.apps.crud.interface import BaseCRUDAdminInterface
from corehq.apps.reports.datatables import DataTablesHeader, DataTablesColumn

class AccountingInterface(BaseCRUDAdminInterface):
    section_name = "Accounting"
    base_template = 'reports/base_template.html'
    dispatcher = AccountingAdminInterfaceDispatcher

    crud_form_update_url = "/accounting/form/"

    def validate_document_class(self):
        return True
        #if self.document_class is None or not issubclass(self.document_class, HQAnnouncement):
        #    raise NotImplementedError("document_class must be an HQAnnouncement")

    """
    @property
    def default_report_url(self):
        return reverse("default_announcement_admin")
    """

    @property
    def headers(self):
        return DataTablesHeader(
            DataTablesColumn("Client/Account Name"),
            DataTablesColumn("Billing Contact Name (Email)"),
            DataTablesColumn("Plan Credit"),
            DataTablesColumn("SMS Credit"),
            DataTablesColumn("Users Credit"),
            DataTablesColumn("Account Balance"),
            DataTablesColumn("Active Subscriptions"),
            DataTablesColumn("Edit"),
        )

    @property
    def rows(self):
        rows = []
        for account in BillingAccount.objects.all():
            rows.append([account.name,
                         account.web_user_contact,
                         3,
                         4,
                         5,
                         account.balance,
                         Subscription.objects.filter(account=account,
                                                     is_active=True).count(),
                         mark_safe('Edit' % account.id)])
        return rows

    #######

    name = "View Billing Accounts"
    description = "description of view billing accounts"
    slug = "accounts"

    document_class = HQAnnouncement
    form_class = HQAnnouncementForm

    crud_item_type = "Billing Account"

class SubscriptionInterface(BaseCRUDAdminInterface):
    section_name = "Accounting"
    base_template = 'reports/base_template.html'
    dispatcher = SubscriptionAdminInterfaceDispatcher

    crud_form_update_url = "/accounting/form/"

    def validate_document_class(self):
        return True
        #if self.document_class is None or not issubclass(self.document_class, HQAnnouncement):
        #    raise NotImplementedError("document_class must be an HQAnnouncement")

    """
    @property
    def default_report_url(self):
        return reverse("default_announcement_admin")
    """

    @property
    def headers(self):
        return DataTablesHeader(
            DataTablesColumn("Project Space"),
            DataTablesColumn("Client Name"),
            DataTablesColumn("Version"),
            DataTablesColumn("Description"),
            DataTablesColumn("Account Name"),
            DataTablesColumn("Start Date"),
            DataTablesColumn("End Date"),
            DataTablesColumn("Active Status"),
            DataTablesColumn("Visibility"),
            DataTablesColumn("Is Default for Product?"),
            DataTablesColumn("Edit"),
        )

    @property
    def rows(self):
        rows = []
        for subscription in Subscription.objects.all():
            rows.append([subscription.subscriber.domain,
                         subscription.account.name,
                         subscription.plan.plan.name,
                         subscription.plan.plan.description,
                         mark_safe('%s'
                                   % (subscription.account.id, subscription.account.name)),
                         subscription.date_start,
                         subscription.date_end,
                         subscription.is_active,
                         subscription.plan.plan.visibility,
                         "MISSING VALUE",
                         mark_safe('Edit' % subscription.id)])
        return rows

    #######

    name = "View Subscriptions"
    description = "description of view subscriptions"
    slug = "subscriptions"

    document_class = HQAnnouncement
    form_class = HQAnnouncementForm

    crud_item_type = "Subscription"
<|endoftext|>


<commit_before>"""
Standardize names of data files on Maryland State Board of Elections and
save to mappings/filenames.csv

File-name convention on MD site (2004-2012):

    general election
        precinct:            countyname_by_precinct_year_general.csv
        state leg. district: state_leg_districts_year_general.csv
        county:              countyname_party_year_general.csv

    primary election
        precinct:            countyname_by_Precinct_party_year_Primary.csv
        state leg. district: state_leg_districts_party_year_primary.csv
        county:              countyname_party_year_primary.csv

    Exceptions: 2000 + 2002

"""
from openelex.base.datasource import DataSource

class MDData(DataSource):

    def __init__(self):
        #TODO: for year in years: get source urls
        #ocd_id = 'ocd-division/country:us/state:md/sldl:all'
        self.base_url = "http://www.elections.state.md.us/elections/%(year)s/election_data/"

    #TODO: Construct target url
    #TODO: Standardize name

    #TODO: datefilter handler; if only years are available, need to either
    # 1) introspect data for dates or
    # 2) notify that only year-based filter is available (and filter for year?)

    @property
    def target_years(self):
        return range(2000, 2014)

    @property
    def target_urls(self):
        urls = []

        # For each year, get urls for primaries and generals
        for year in self.target_years:
            for race_type in ('Primary', 'General'):
                if race_type == 'Primary':
                    for party in ['Democratic', 'Republican']:
                        urls.append(self.state_leg_url(year, race_type, party))
                else:
                    urls.append(self.state_leg_url(year, race_type))
        return urls

    def state_leg_url(self, year, race_type, party=''):
        #TODO: URL examples: "http://www.elections.state.md.us/elections/2009/election_data/State_Legislative_Districts_2009_Democratic_Primary.csv"
        url_tmplt = self.base_url + "State_Legislative_Districts_%(year)s_"
        general['start_date']

        kwargs = {
            'year': year,
            'race_type': race_type,
            'party': party
        }
        # If a party is present, assume it's a primary. this is naive
        if kwargs['party']:
            url_primary = url_tmplt + "%(race_type)s.csv"
            url = url_primary % kwargs
        else:
            url_gen = url_tmplt + "%(party)s_%(race_type)s.csv"
            url = url_gen % kwargs
        return url

    def standardize_name(self, url, race_type, start_date):
        start_date = start_date.replace('-','')
        if 'state_leg' in url.lower():
            bits = (
                start_date
                "__",
                self.state,
                "__general__state_legislative.csv"
            )
            name = "".join(bits)
        elif 'county' in url.lower():
            pass
            name = ""
        return name

        """
        general = [e for e in elections['elections'] if e['election_type'] == 'general'][0]

        urls.append([
            generated_name,
            raw_name,
            ocd_id,
            'State Legislative Districts',
            general['id']
        ])

        primary = [e for e in elections['elections'] if e['election_type'] == 'primary'][0]

        raw_name = self.base_url + "/State_Legislative_Districts_%s_%s_Primary.csv" % (year, party, year)
        for party in ['Democratic', 'Republican']:
            generated_name = primary['start_date'].replace('-','')+"__"+self.state+"__"+party.lower()+"__primary__state_legislative.csv"
            urls.append([
                generated_name,
                raw_name,
                'ocd-division/country:us/state:md/sldl:all',
                'State Legislative Districts',
                primary['id']
            ])
        """
        return urls

    def county_urls(self):
        urls = []
        general = [e for e in elections['elections'] if e['election_type'] == 'general'][0]
        primary = [e for e in elections['elections'] if e['election_type'] == 'primary'][0]
        for jurisdiction in self.jurisdictions():
            county_generated_name = general['start_date'].replace('-','')+"__"+self.state+"__general__%s.csv" % jurisdiction['url_name'].lower()
            county_raw_name = "%s_County_%s_General.csv" % (year, jurisdiction['url_name'], year)
            urls.append([county_generated_name, county_raw_name, jurisdiction['ocd_id'], jurisdiction['name'], general['id']])
            precinct_generated_name = general['start_date'].replace('-','')+"__"+self.state+"__general__%s__precinct.csv" % jurisdiction['url_name'].lower()
            precinct_raw_name = "http://www.elections.state.md.us/elections/%s/election_data/%s_By_Precinct_%s_General.csv" % (year, jurisdiction['url_name'], year)
            urls.append([precinct_generated_name, precinct_raw_name, jurisdiction['ocd_id'], jurisdiction['name'], general['id']])
            for party in ['Democratic', 'Republican']:
                county_party_generated_name = primary['start_date'].replace('-','')+"__"+self.state+"__"+party.lower()+"__primary__%s.csv" % jurisdiction['url_name'].lower()
                county_party_raw_name = "http://www.elections.state.md.us/elections/%s/election_data/%s_County_%s_%s_Primary.csv" % (year, jurisdiction['url_name'], party, year)
                urls.append([county_party_generated_name, county_party_raw_name, jurisdiction['ocd_id'], jurisdiction['name'], primary['id']])
                precinct_party_generated_name = primary['start_date'].replace('-','')+"__"+self.state+"__"+party.lower()+"__primary__%s__precinct.csv" % jurisdiction['url_name'].lower()
                precinct_party_raw_name = "http://www.elections.state.md.us/elections/%s/election_data/%s_By_Precinct_%s_%s_Primary.csv" % (year, jurisdiction['url_name'], party, year)
                urls.append([precinct_party_generated_name, precinct_party_raw_name, jurisdiction['ocd_id'], jurisdiction['name'], primary['id']])
        return urls

    def jurisdictions(self):
        """Maryland counties, plus Baltimore City"""
        m = self.jurisdiction_mappings(('ocd_id','fips','url_name', 'name'))
        return m

if __name__ == '__main__':
    from pprint import pprint as pp
    md =MDData()
    #print md.base_url
    pp(md.target_urls)
<commit_msg>add comma<commit_after>"""
Standardize names of data files on Maryland State Board of Elections and
save to mappings/filenames.csv

File-name convention on MD site (2004-2012):

    general election
        precinct:            countyname_by_precinct_year_general.csv
        state leg. district: state_leg_districts_year_general.csv
        county:              countyname_party_year_general.csv

    primary election
        precinct:            countyname_by_Precinct_party_year_Primary.csv
        state leg. district: state_leg_districts_party_year_primary.csv
        county:              countyname_party_year_primary.csv

    Exceptions: 2000 + 2002

"""
from openelex.base.datasource import DataSource

class MDData(DataSource):

    def __init__(self):
        #TODO: for year in years: get source urls
        #ocd_id = 'ocd-division/country:us/state:md/sldl:all'
        self.base_url = "http://www.elections.state.md.us/elections/%(year)s/election_data/"

    #TODO: Construct target url
    #TODO: Standardize name

    #TODO: datefilter handler; if only years are available, need to either
    # 1) introspect data for dates or
    # 2) notify that only year-based filter is available (and filter for year?)

    @property
    def target_years(self):
        return range(2000, 2014)

    @property
    def target_urls(self):
        urls = []

        # For each year, get urls for primaries and generals
        for year in self.target_years:
            for race_type in ('Primary', 'General'):
                if race_type == 'Primary':
                    for party in ['Democratic', 'Republican']:
                        urls.append(self.state_leg_url(year, race_type, party))
                else:
                    urls.append(self.state_leg_url(year, race_type))
        return urls

    def state_leg_url(self, year, race_type, party=''):
        #TODO: URL examples: "http://www.elections.state.md.us/elections/2009/election_data/State_Legislative_Districts_2009_Democratic_Primary.csv"
        url_tmplt = self.base_url + "State_Legislative_Districts_%(year)s_"
        general['start_date']

        kwargs = {
            'year': year,
            'race_type': race_type,
            'party': party
        }
        # If a party is present, assume it's a primary. this is naive
        if kwargs['party']:
            url_primary = url_tmplt + "%(race_type)s.csv"
            url = url_primary % kwargs
        else:
            url_gen = url_tmplt + "%(party)s_%(race_type)s.csv"
            url = url_gen % kwargs
        return url

    def standardize_name(self, url, race_type, start_date):
        start_date = start_date.replace('-','')
        if 'state_leg' in url.lower():
            bits = (
                start_date,
                "__",
                self.state,
                "__general__state_legislative.csv"
            )
            name = "".join(bits)
        elif 'county' in url.lower():
            pass
            name = ""
        return name

        """
        general = [e for e in elections['elections'] if e['election_type'] == 'general'][0]

        urls.append([
            generated_name,
            raw_name,
            ocd_id,
            'State Legislative Districts',
            general['id']
        ])

        primary = [e for e in elections['elections'] if e['election_type'] == 'primary'][0]

        raw_name = self.base_url + "/State_Legislative_Districts_%s_%s_Primary.csv" % (year, party, year)
        for party in ['Democratic', 'Republican']:
            generated_name = primary['start_date'].replace('-','')+"__"+self.state+"__"+party.lower()+"__primary__state_legislative.csv"
            urls.append([
                generated_name,
                raw_name,
                'ocd-division/country:us/state:md/sldl:all',
                'State Legislative Districts',
                primary['id']
            ])
        """
        return urls

    def county_urls(self):
        urls = []
        general = [e for e in elections['elections'] if e['election_type'] == 'general'][0]
        primary = [e for e in elections['elections'] if e['election_type'] == 'primary'][0]
        for jurisdiction in self.jurisdictions():
            county_generated_name = general['start_date'].replace('-','')+"__"+self.state+"__general__%s.csv" % jurisdiction['url_name'].lower()
            county_raw_name = "%s_County_%s_General.csv" % (year, jurisdiction['url_name'], year)
            urls.append([county_generated_name, county_raw_name, jurisdiction['ocd_id'], jurisdiction['name'], general['id']])
            precinct_generated_name = general['start_date'].replace('-','')+"__"+self.state+"__general__%s__precinct.csv" % jurisdiction['url_name'].lower()
            precinct_raw_name = "http://www.elections.state.md.us/elections/%s/election_data/%s_By_Precinct_%s_General.csv" % (year, jurisdiction['url_name'], year)
            urls.append([precinct_generated_name, precinct_raw_name, jurisdiction['ocd_id'], jurisdiction['name'], general['id']])
            for party in ['Democratic', 'Republican']:
                county_party_generated_name = primary['start_date'].replace('-','')+"__"+self.state+"__"+party.lower()+"__primary__%s.csv" % jurisdiction['url_name'].lower()
                county_party_raw_name = "http://www.elections.state.md.us/elections/%s/election_data/%s_County_%s_%s_Primary.csv" % (year, jurisdiction['url_name'], party, year)
                urls.append([county_party_generated_name, county_party_raw_name, jurisdiction['ocd_id'], jurisdiction['name'], primary['id']])
                precinct_party_generated_name = primary['start_date'].replace('-','')+"__"+self.state+"__"+party.lower()+"__primary__%s__precinct.csv" % jurisdiction['url_name'].lower()
                precinct_party_raw_name = "http://www.elections.state.md.us/elections/%s/election_data/%s_By_Precinct_%s_%s_Primary.csv" % (year, jurisdiction['url_name'], party, year)
                urls.append([precinct_party_generated_name, precinct_party_raw_name, jurisdiction['ocd_id'], jurisdiction['name'], primary['id']])
        return urls

    def jurisdictions(self):
        """Maryland counties, plus Baltimore City"""
        m = self.jurisdiction_mappings(('ocd_id','fips','url_name', 'name'))
        return m

if __name__ == '__main__':
    from pprint import pprint as pp
    md =MDData()
    #print md.base_url
    pp(md.target_urls)
<|endoftext|>


<commit_before><commit_msg>add space to recognised vkey codes<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python

# TODO: right now we assume that input files are pre-split.

import threading

class DataSet(object):
    """Manage input to or output from a map or reduce operation.
    """
    def __len__(self):
        """Number of splits in this DataSet."""
        return 0

    def __getitem__(self, split):
        """Retrieve a seq of URLs for a particular split in this DataSet."""
        raise IndexError

    def ready(self, split=None):
        """Whether or not a split is ready to be retrieved.

        If the split is not specified, ready() tells whether or not
        all splits are ready.
        """
        return True

class FileData(DataSet):
    """A list of static files that can be used as input to an operation.

    >>> urls = ['http://aml.cs.byu.edu/', 'http://www.cs.byu.edu/']
    >>> data = FileData(urls)
    >>> len(data)
    2
    >>> data[1]
    'http://www.cs.byu.edu/'
    >>>
    """
    def __init__(self, urls):
        self._len = len(urls)
        self._urls = tuple(urls)

    def __len__(self):
        """How many files are in the DataSet."""
        return self._len

    def __getitem__(self, i):
        """Retrieve a sequence containing the URL for the specified file."""
        return (self._urls[i],)

class ComputedData(DataSet):
    """Manage input to or output from a map or reduce operation.

    The data are evaluated lazily.  A DataSet knows how to generate or
    regenerate its contents.  It can also decide whether to save the data to
    permanent storage or to leave them in memory on the slaves.
    """
    def __init__(self, input, func, nparts, outdir, parter=None,
            registry=None):
        self.input = input
        self.ntasks = len(self.input)
        self.outdir = outdir
        if registry is None:
            from registry import Registry
            self.registry = Registry()
        else:
            self.registry = registry
        self.nparts = nparts

        self.func_name = self.registry.as_name(func)
        if parter is None:
            self.part_name = ''
        else:
            self.part_name = self.registry.as_name(parter)

        self.tasks_made = False
        self.tasks_todo = []
        self.tasks_done = []
        self.tasks_active = []

        # TODO: store a mapping from tasks to hosts and a map from hosts to
        # tasks.  This way you can know where to find data.  You also know
        # which hosts to restart in case of failure.

    def __len__(self):
        return self.nparts

    def __getitem__(self, split):
        if not self.ready():
            raise IndexError
        else:
            return [task.outurls[split] for task in self.tasks_done
                    if task.outurls[split] != '']

    def ready(self):
        if self.tasks_made and not self.tasks_todo and not self.tasks_active:
            return True
        else:
            return False

    def get_task(self):
        """Return the next available task"""
        if self.tasks_todo:
            task = self.tasks_todo.pop()
            return task
        else:
            return

    def print_status(self):
        active = len(self.tasks_active)
        todo = len(self.tasks_todo)
        done = len(self.tasks_done)
        total = active + todo + done
        print 'Completed: %s/%s, Active: %s' % (done, total, active)

class MapData(ComputedData):
    def __init__(self, input, mapper, nparts, outdir, parter=None,
            registry=None):
        ComputedData.__init__(self, input, mapper, nparts, outdir,
                parter=None, registry=registry)

    def make_tasks(self):
        from mapreduce import MapTask
        for taskid in xrange(self.ntasks):
            task = MapTask(taskid, self.input, self.registry, self.func_name,
                    self.part_name, self.outdir, self.nparts)
            task.dataset = self
            self.tasks_todo.append(task)
        self.tasks_made = True

    def run_serial(self):
        pass
        input_files = [io.openfile(filename) for filename in self.inputs]
        all_input = chain(*input_files)
        map_output = mrs_map(registry['mapper'], all_input)

class ReduceData(ComputedData):
    def __init__(self, input, reducer, nparts, outdir, parter=None,
            registry=None):
        ComputedData.__init__(self, input, reducer, nparts, outdir,
                parter=None, registry=registry)

    def make_tasks(self):
        from mapreduce import ReduceTask
        for taskid in xrange(self.ntasks):
            task = ReduceTask(taskid, self.input, self.registry,
                    self.func_name, self.outdir)
            task.dataset = self
            self.tasks_todo.append(task)
        self.tasks_made = True

# vim: et sw=4 sts=4
<commit_msg>added Bucket with testing<commit_after>#!/usr/bin/env python

# TODO: right now we assume that input files are pre-split.

import threading
from heapq import heappush

from io import fileformat, HexFormat

# TODO: cache data to disk when memory usage is high
class Bucket(object):
    """Hold data from a source or for a split.

    Data can be manually dumped to disk, in which case the data will be saved
    to the given filename with the specified format.  Eventually Bucket will
    be upgraded to automatically cache data to disk if they get too large to
    stay in memory.

    >>> b = Bucket()
    >>> b.append((4, 'test'))
    >>> b.collect([(3, 'a'), (1, 'This'), (2, 'is')])
    >>> ' '.join([value for key, value in b])
    'test a This is'
    >>> b.sort = True
    >>> ' '.join([value for key, value in b])
    'This is a test'
    >>>
    """
    def __init__(self, sort=False, filename=None, format=HexFormat):
        self._data = []
        self.sort = sort

        self.ready = False

    def append(self, x):
        """Collect a single key-value pair
        """
        if self.sort:
            heappush(self._data, x)
        else:
            self._data.append(x)

    def collect(self, pairiter):
        """Collect all key-value pairs from the given iterable

        The collection can be a generator or a Mrs format.  This will block if
        the iterator blocks.
        """
        data = self._data
        if self.sort:
            for kvpair in pairiter:
                heappush(data, kvpair)
                data.append(kvpair)
        else:
            for kvpair in pairiter:
                data.append(kvpair)

    def __iter__(self):
        if self.sort:
            return iter(sorted(self._data))
        else:
            return iter(self._data)

class DataSet(object):
    """Manage input to or output from a map or reduce operation.
    """
    def __len__(self):
        """Number of splits in this DataSet."""
        return 0

    def __getitem__(self, split):
        """Retrieve a seq of URLs for a particular split in this DataSet."""
        raise IndexError

    def ready(self, split=None):
        """Whether or not a split is ready to be retrieved.

        If the split is not specified, ready() tells whether or not
        all splits are ready.
        """
        return True

class FileData(DataSet):
    """A list of static files that can be used as input to an operation.

    >>> urls = ['http://aml.cs.byu.edu/', 'http://www.cs.byu.edu/']
    >>> data = FileData(urls)
    >>> len(data)
    2
    >>> data[1]
    'http://www.cs.byu.edu/'
    >>>
    """
    def __init__(self, urls):
        self._len = len(urls)
        self._urls = tuple(urls)

    def __len__(self):
        """How many files are in the DataSet."""
        return self._len

    def __getitem__(self, i):
        """Retrieve a sequence containing the URL for the specified file."""
        return (self._urls[i],)

class ComputedData(DataSet):
    """Manage input to or output from a map or reduce operation.

    The data are evaluated lazily.  A DataSet knows how to generate or
    regenerate its contents.  It can also decide whether to save the data to
    permanent storage or to leave them in memory on the slaves.
    """
    def __init__(self, input, func, nparts, outdir, parter=None,
            registry=None):
        self.input = input
        self.ntasks = len(self.input)
        self.outdir = outdir
        if registry is None:
            from registry import Registry
            self.registry = Registry()
        else:
            self.registry = registry
        self.nparts = nparts

        self.func_name = self.registry.as_name(func)
        if parter is None:
            self.part_name = ''
        else:
            self.part_name = self.registry.as_name(parter)

        self.tasks_made = False
        self.tasks_todo = []
        self.tasks_done = []
        self.tasks_active = []

        # TODO: store a mapping from tasks to hosts and a map from hosts to
        # tasks.  This way you can know where to find data.  You also know
        # which hosts to restart in case of failure.

    def __len__(self):
        return self.nparts

    def __getitem__(self, split):
        if not self.ready():
            raise IndexError
        else:
            return [task.outurls[split] for task in self.tasks_done
                    if task.outurls[split] != '']

    def ready(self):
        if self.tasks_made and not self.tasks_todo and not self.tasks_active:
            return True
        else:
            return False

    def get_task(self):
        """Return the next available task"""
        if self.tasks_todo:
            task = self.tasks_todo.pop()
            return task
        else:
            return

    def print_status(self):
        active = len(self.tasks_active)
        todo = len(self.tasks_todo)
        done = len(self.tasks_done)
        total = active + todo + done
        print 'Completed: %s/%s, Active: %s' % (done, total, active)

class MapData(ComputedData):
    def __init__(self, input, mapper, nparts, outdir, parter=None,
            registry=None):
        ComputedData.__init__(self, input, mapper, nparts, outdir,
                parter=None, registry=registry)

    def make_tasks(self):
        from mapreduce import MapTask
        for taskid in xrange(self.ntasks):
            task = MapTask(taskid, self.input, self.registry, self.func_name,
                    self.part_name, self.outdir, self.nparts)
            task.dataset = self
            self.tasks_todo.append(task)
        self.tasks_made = True

    def run_serial(self):
        pass
        input_files = [io.openfile(filename) for filename in self.inputs]
        all_input = chain(*input_files)
        map_output = mrs_map(registry['mapper'], all_input)

class ReduceData(ComputedData):
    def __init__(self, input, reducer, nparts, outdir, parter=None,
            registry=None):
        ComputedData.__init__(self, input, reducer, nparts, outdir,
                parter=None, registry=registry)

    def make_tasks(self):
        from mapreduce import ReduceTask
        for taskid in xrange(self.ntasks):
            task = ReduceTask(taskid, self.input, self.registry,
                    self.func_name, self.outdir)
            task.dataset = self
            self.tasks_todo.append(task)
        self.tasks_made = True

def test_datasets():
    import doctest
    doctest.testmod()

if __name__ == "__main__":
    test_datasets()

# vim: et sw=4 sts=4
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Skyspark operation implementations.
"""

import fysom
import hmac
import base64
import hashlib
import re

from ....util import state
from ....util.asyncexc import AsynchronousException

class SkysparkAuthenticateOperation(state.HaystackOperation):
    """
    An implementation of the log-in procedure for Skyspark.  The procedure
    is as follows:

    1. Retrieve the log-in URL (GET request).
    2. Parse the key-values pairs returned, pick up 'username', 'userSalt' and
       'nonce'.
    3. Compute
        mac = Base64(HMAC_SHA1(key=password, msg="${username}:${userSalt}"))
    4. Compute
        digest = Base64(SHA1("${mac}:${nonce}"))
    5. POST to log-in URL:
        nonce: ${nonce}
        digest: ${digest}
    6. Stash received cookies given in the returned body.

    Future requests should the cookies returned.
    """

    _COOKIE_RE = re.compile(r'^cookie[ \t]*:[ \t]*([^=]+)=(.*)$')

    def __init__(self, session, retries=2):
        """
        Attempt to log in to the Skyspark server.

        :param session: Haystack HTTP session object.
        :param retries: Number of retries permitted in case of failure.
        """

        super(SkysparkAuthenticateOperation, self).__init__()
        self._retries = retries
        self._session = session
        self._cookie = None
        self._mac = None
        self._nonce = None
        self._username = None
        self._user_salt = None
        self._digest = None
        self._login_uri = '%s/auth/%s/api' % \
                (session._client.uri, session._project)

        self._state_machine = fysom.Fysom(
                initial='init', final='done',
                events=[
                    # Event             Current State       New State
                    ('get_new_session', 'init',             'newsession'),
                    ('do_login',        'newsession',       'login'),
                    ('login_done',      'login',            'done'),
                    ('exception',       '*',                'failed'),
                    ('retry',           'failed',           'newsession'),
                    ('abort',           'failed',           'done'),
                ], callbacks={
                    'onenternewsession':    self._do_new_session,
                    'onenterlogin':         self._do_login,
                    'onenterfailed':        self._do_fail_retry,
                    'onenterdone':          self._do_done,
                })

    def go(self):
        """
        Start the request.
        """
        # Are we logged in?
        try:
            self._state_machine.get_new_session()
        except: # Catch all exceptions to pass to caller.
            self._state_machine.exception(result=AsynchronousException())

    def _do_new_session(self, event):
        """
        Request the log-in parameters.
        """
        try:
            self._session._get(self._login_uri,
                    callback=self._on_new_session,
                    args={'username': self._session._username},
                    cookies={}, headers={}, exclude_cookies=True,
                    exclude_headers=True, api=False)
        except: # Catch all exceptions to pass to caller.
            self._state_machine.exception(result=AsynchronousException())

    def _on_new_session(self, response):
        """
        Retrieve the log-in parameters.
        """
        try:
            if isinstance(response, AsynchronousException):
                response.reraise()

            login_params = {}
            # TODO: Is this really how they send it, or is it JSON?
            # Can someone dump a sanitised version as a comment?  Seems weird
            # to have to manually parse like this.
            for line in response.body.split('\n'):
                key, value = line.split(':')
                login_params[key] = value

            self._username = login_params['username']
            self._user_salt = login_params['userSalt']
            self._nonce = login_params['nonce']

            self._state_machine.do_login()
        except: # Catch all exceptions to pass to caller.
            self._state_machine.exception(result=AsynchronousException())

    def _do_login(self, event):
        try:
            # Compute log-in body
            mac = hmac.new(key=self._session.password,
                    msg="%s:%s" % (self._username, self._user_salt),
                    digestmod=hashlib.sha1)
            digest = hashlib.sha1()
            digest.update('%s:%s' % (base64.b64encode(mac.digest()),
                        self._nonce))

            # Post
            self._session._post(self._login_uri,
                    callback=self._on_login,
                    body='nonce: %s\ndigest: %s' % \
                            (self._nonce, digest.encode('base64')),
                    body_type='text/plain',
                    headers={}, exclude_cookies=True,
                    exclude_headers=True, api=False)
        except: # Catch all exceptions to pass to caller.
            self._state_machine.exception(result=AsynchronousException())

    def _on_login(self, response):
        """
        See if the login succeeded.
        """
        try:
            if isinstance(response, AsynchronousException):
                response.reraise()

            # Locate the cookie in the response.
            cookie_match = self._COOKIE_RE.match(response.body)
            if not cookie_match:
                raise IOError('No cookie in response, log-in failed.')

            (cookie_name, cookie_value) = cookie_match.groups()

            self._state_machine.login_done(result={cookie_name: cookie_value})
        except: # Catch all exceptions to pass to caller.
            self._state_machine.exception(result=AsynchronousException())

    def _do_fail_retry(self, event):
        """
        Determine whether we retry or fail outright.
        """
        if self._retries > 0:
            self._retries -= 1
            self._state_machine.retry()
        else:
            self._state_machine.abort(result=event.result)

    def _do_done(self, event):
        """
        Return the result from the state machine.
        """
        self._done(event.result)
<commit_msg>client.ops.vendor.skyspark: Use response.text<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Skyspark operation implementations.
"""

import fysom
import hmac
import base64
import hashlib
import re

from ....util import state
from ....util.asyncexc import AsynchronousException

class SkysparkAuthenticateOperation(state.HaystackOperation):
    """
    An implementation of the log-in procedure for Skyspark.  The procedure
    is as follows:

    1. Retrieve the log-in URL (GET request).
    2. Parse the key-values pairs returned, pick up 'username', 'userSalt' and
       'nonce'.
    3. Compute
        mac = Base64(HMAC_SHA1(key=password, msg="${username}:${userSalt}"))
    4. Compute
        digest = Base64(SHA1("${mac}:${nonce}"))
    5. POST to log-in URL:
        nonce: ${nonce}
        digest: ${digest}
    6. Stash received cookies given in the returned body.

    Future requests should the cookies returned.
    """

    _COOKIE_RE = re.compile(r'^cookie[ \t]*:[ \t]*([^=]+)=(.*)$')

    def __init__(self, session, retries=2):
        """
        Attempt to log in to the Skyspark server.

        :param session: Haystack HTTP session object.
        :param retries: Number of retries permitted in case of failure.
        """

        super(SkysparkAuthenticateOperation, self).__init__()
        self._retries = retries
        self._session = session
        self._cookie = None
        self._mac = None
        self._nonce = None
        self._username = None
        self._user_salt = None
        self._digest = None
        self._login_uri = '%s/auth/%s/api' % \
                (session._client.uri, session._project)

        self._state_machine = fysom.Fysom(
                initial='init', final='done',
                events=[
                    # Event             Current State       New State
                    ('get_new_session', 'init',             'newsession'),
                    ('do_login',        'newsession',       'login'),
                    ('login_done',      'login',            'done'),
                    ('exception',       '*',                'failed'),
                    ('retry',           'failed',           'newsession'),
                    ('abort',           'failed',           'done'),
                ], callbacks={
                    'onenternewsession':    self._do_new_session,
                    'onenterlogin':         self._do_login,
                    'onenterfailed':        self._do_fail_retry,
                    'onenterdone':          self._do_done,
                })

    def go(self):
        """
        Start the request.
        """
        # Are we logged in?
        try:
            self._state_machine.get_new_session()
        except: # Catch all exceptions to pass to caller.
            self._state_machine.exception(result=AsynchronousException())

    def _do_new_session(self, event):
        """
        Request the log-in parameters.
        """
        try:
            self._session._get(self._login_uri,
                    callback=self._on_new_session,
                    args={'username': self._session._username},
                    cookies={}, headers={}, exclude_cookies=True,
                    exclude_headers=True, api=False)
        except: # Catch all exceptions to pass to caller.
            self._state_machine.exception(result=AsynchronousException())

    def _on_new_session(self, response):
        """
        Retrieve the log-in parameters.
        """
        try:
            if isinstance(response, AsynchronousException):
                response.reraise()

            login_params = {}
            # TODO: Is this really how they send it, or is it JSON?
            # Can someone dump a sanitised version as a comment?  Seems weird
            # to have to manually parse like this.
            for line in response.text.split('\n'):
                key, value = line.split(':')
                login_params[key] = value

            self._username = login_params['username']
            self._user_salt = login_params['userSalt']
            self._nonce = login_params['nonce']

            self._state_machine.do_login()
        except: # Catch all exceptions to pass to caller.
            self._state_machine.exception(result=AsynchronousException())

    def _do_login(self, event):
        try:
            # Compute log-in body
            mac = hmac.new(key=self._session.password,
                    msg="%s:%s" % (self._username, self._user_salt),
                    digestmod=hashlib.sha1)
            digest = hashlib.sha1()
            digest.update('%s:%s' % (base64.b64encode(mac.digest()),
                        self._nonce))

            # Post
            self._session._post(self._login_uri,
                    callback=self._on_login,
                    body='nonce: %s\ndigest: %s' % \
                            (self._nonce, digest.encode('base64')),
                    body_type='text/plain',
                    headers={}, exclude_cookies=True,
                    exclude_headers=True, api=False)
        except: # Catch all exceptions to pass to caller.
            self._state_machine.exception(result=AsynchronousException())

    def _on_login(self, response):
        """
        See if the login succeeded.
        """
        try:
            if isinstance(response, AsynchronousException):
                response.reraise()

            # Locate the cookie in the response.
            cookie_match = self._COOKIE_RE.match(response.text)
            if not cookie_match:
                raise IOError('No cookie in response, log-in failed.')

            (cookie_name, cookie_value) = cookie_match.groups()

            self._state_machine.login_done(result={cookie_name: cookie_value})
        except: # Catch all exceptions to pass to caller.
            self._state_machine.exception(result=AsynchronousException())

    def _do_fail_retry(self, event):
        """
        Determine whether we retry or fail outright.
        """
        if self._retries > 0:
            self._retries -= 1
            self._state_machine.retry()
        else:
            self._state_machine.abort(result=event.result)

    def _do_done(self, event):
        """
        Return the result from the state machine.
        """
        self._done(event.result)
<|endoftext|>


<commit_before>from django.conf.urls import *

from corehq.apps.receiverwrapper.views import post, secure_post

urlpatterns = patterns('corehq.apps.receiverwrapper.views',
    url(r'^/?$', post, name='receiver_post'),
    url(r'^/secure/(?P<app_id>[\w-]+)/$', secure_post, name='receiver_secure_post_with_app_id'),
    url(r'^/secure/$', secure_post, name='receiver_secure_post'),

    # odk urls
    url(r'^/submission/?$',  post, name="receiver_odk_post"),
    url(r'^/(?P<app_id>[\w-]+)/$', post, name='receiver_post_with_app_id'),
)
<commit_msg>remove extra space<commit_after>from django.conf.urls import *

from corehq.apps.receiverwrapper.views import post, secure_post

urlpatterns = patterns('corehq.apps.receiverwrapper.views',
    url(r'^/?$', post, name='receiver_post'),
    url(r'^/secure/(?P<app_id>[\w-]+)/$', secure_post, name='receiver_secure_post_with_app_id'),
    url(r'^/secure/$', secure_post, name='receiver_secure_post'),

    # odk urls
    url(r'^/submission/?$', post, name="receiver_odk_post"),
    url(r'^/(?P<app_id>[\w-]+)/$', post, name='receiver_post_with_app_id'),
)
<|endoftext|>


<commit_before>from __future__ import absolute_import

from memoized import memoized
from corehq.apps.app_manager.app_translations import (
    expected_bulk_app_sheet_headers,
    expected_bulk_app_sheet_rows,
    get_unicode_dicts,
)
from corehq.apps.app_manager.app_translations.const import MODULES_AND_FORMS_SHEET_NAME
from django.utils.translation import ugettext as _

COLUMNS_TO_COMPARE = {
    'module_and_form': ['Type', 'sheet_name'],
    'module': ['case_property', 'list_or_detail'],
    'form': ['label'],
}
LESS_ROW_COUNT_MISMATCH_MESSAGE = _("We found less rows than expected. Validating what was uploaded.")
MORE_ROW_COUNT_MISMATCH_MESSAGE = _("We found more rows than expected. Validating what was uploaded.")

class UploadedTranslationsValidator(object):
    """
    this compares the excel sheet uploaded with translations with what would be generated
    with current app state and flags any discrepancies found between the two
    """
    def __init__(self, app, uploaded_workbook, lang_prefix='default_'):
        self.app = app
        self.uploaded_workbook = uploaded_workbook
        self.headers = None
        self.expected_rows = None
        self.lang_prefix = lang_prefix
        self.default_language_column = self.lang_prefix + self.app.default_language

    def _generate_expected_headers_and_rows(self):
        self.headers = {h[0]: h[1] for h in expected_bulk_app_sheet_headers(self.app)}
        self.expected_rows = expected_bulk_app_sheet_rows(self.app)

    @memoized
    def _get_header_index(self, sheet_name, header):
        for index, _column_name in enumerate(self.headers[sheet_name]):
            if _column_name == header:
                return index

    def _compare_sheet(self, sheet_name, uploaded_rows, for_type):
        """
        :param uploaded_rows: dict
        :param for_type: type of sheet, module_and_forms, module, form
        :return: list of errors messages if any
        """
        columns_to_compare = COLUMNS_TO_COMPARE[for_type] + [self.default_language_column]
        expected_rows = self.expected_rows[sheet_name]
        msgs = []
        number_of_uploaded_rows = len(uploaded_rows)
        number_of_expected_rows = len(expected_rows)
        if number_of_uploaded_rows < number_of_expected_rows:
            msgs.append(LESS_ROW_COUNT_MISMATCH_MESSAGE)
        elif number_of_uploaded_rows > number_of_expected_rows:
            msgs.append(MORE_ROW_COUNT_MISMATCH_MESSAGE)

        iterate_on = [expected_rows, uploaded_rows]
        # 2 to account for the sheet header as well
        for i, (expected_row, uploaded_row) in enumerate(zip(*iterate_on), 2):
            for column_name in columns_to_compare:
                uploaded_value = uploaded_row.get(column_name)
                expected_value = expected_row[self._get_header_index(sheet_name, column_name)]
                if expected_value != uploaded_value:

                    msgs.append("For {}:{}  uploaded '{}' but expected '{}'.".format(
                        i + 2, column_name, uploaded_value, expected_value
                    ))
        return msgs

    def compare(self):
        msgs = {}
        self._generate_expected_headers_and_rows()
        for sheet in self.uploaded_workbook.worksheets:
            rows = get_unicode_dicts(sheet)
            sheet_name = sheet.worksheet.title
            if sheet_name == MODULES_AND_FORMS_SHEET_NAME:
                error_msgs = self._compare_sheet(sheet_name, rows, 'module_and_form')
            elif 'module' in sheet_name and 'form' not in sheet_name:
                error_msgs = self._compare_sheet(sheet_name, rows, 'module')
            elif 'module' in sheet_name and 'form' in sheet_name:
                error_msgs = self._compare_sheet(sheet_name, rows, 'form')
            else:
                raise Exception("Got unexpected sheet name %s" % sheet_name)
            if error_msgs:
                msgs[sheet_name] = error_msgs
        return msgs
<commit_msg>use ugettext_lazy<commit_after>from __future__ import absolute_import

from memoized import memoized
from corehq.apps.app_manager.app_translations import (
    expected_bulk_app_sheet_headers,
    expected_bulk_app_sheet_rows,
    get_unicode_dicts,
)
from corehq.apps.app_manager.app_translations.const import MODULES_AND_FORMS_SHEET_NAME
from django.utils.translation import ugettext_lazy as _

COLUMNS_TO_COMPARE = {
    'module_and_form': ['Type', 'sheet_name'],
    'module': ['case_property', 'list_or_detail'],
    'form': ['label'],
}
LESS_ROW_COUNT_MISMATCH_MESSAGE = _("We found less rows than expected. Validating what was uploaded.")
MORE_ROW_COUNT_MISMATCH_MESSAGE = _("We found more rows than expected. Validating what was uploaded.")

class UploadedTranslationsValidator(object):
    """
    this compares the excel sheet uploaded with translations with what would be generated
    with current app state and flags any discrepancies found between the two
    """
    def __init__(self, app, uploaded_workbook, lang_prefix='default_'):
        self.app = app
        self.uploaded_workbook = uploaded_workbook
        self.headers = None
        self.expected_rows = None
        self.lang_prefix = lang_prefix
        self.default_language_column = self.lang_prefix + self.app.default_language

    def _generate_expected_headers_and_rows(self):
        self.headers = {h[0]: h[1] for h in expected_bulk_app_sheet_headers(self.app)}
        self.expected_rows = expected_bulk_app_sheet_rows(self.app)

    @memoized
    def _get_header_index(self, sheet_name, header):
        for index, _column_name in enumerate(self.headers[sheet_name]):
            if _column_name == header:
                return index

    def _compare_sheet(self, sheet_name, uploaded_rows, for_type):
        """
        :param uploaded_rows: dict
        :param for_type: type of sheet, module_and_forms, module, form
        :return: list of errors messages if any
        """
        columns_to_compare = COLUMNS_TO_COMPARE[for_type] + [self.default_language_column]
        expected_rows = self.expected_rows[sheet_name]
        msgs = []
        number_of_uploaded_rows = len(uploaded_rows)
        number_of_expected_rows = len(expected_rows)
        if number_of_uploaded_rows < number_of_expected_rows:
            msgs.append(LESS_ROW_COUNT_MISMATCH_MESSAGE)
        elif number_of_uploaded_rows > number_of_expected_rows:
            msgs.append(MORE_ROW_COUNT_MISMATCH_MESSAGE)

        iterate_on = [expected_rows, uploaded_rows]
        # 2 to account for the sheet header as well
        for i, (expected_row, uploaded_row) in enumerate(zip(*iterate_on), 2):
            for column_name in columns_to_compare:
                uploaded_value = uploaded_row.get(column_name)
                expected_value = expected_row[self._get_header_index(sheet_name, column_name)]
                if expected_value != uploaded_value:

                    msgs.append("For {}:{}  uploaded '{}' but expected '{}'.".format(
                        i + 2, column_name, uploaded_value, expected_value
                    ))
        return msgs

    def compare(self):
        msgs = {}
        self._generate_expected_headers_and_rows()
        for sheet in self.uploaded_workbook.worksheets:
            rows = get_unicode_dicts(sheet)
            sheet_name = sheet.worksheet.title
            if sheet_name == MODULES_AND_FORMS_SHEET_NAME:
                error_msgs = self._compare_sheet(sheet_name, rows, 'module_and_form')
            elif 'module' in sheet_name and 'form' not in sheet_name:
                error_msgs = self._compare_sheet(sheet_name, rows, 'module')
            elif 'module' in sheet_name and 'form' in sheet_name:
                error_msgs = self._compare_sheet(sheet_name, rows, 'form')
            else:
                raise Exception("Got unexpected sheet name %s" % sheet_name)
            if error_msgs:
                msgs[sheet_name] = error_msgs
        return msgs
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Jun 25 10:18:40 2016

Morphology detector provided by Sergey Burnos.
Reprogrammed to python by Jan Cimbalnik..

If you use this detector please cite!

@author: jan_cimbalnik
"""
import pickle,os

from scipy.signal import butter, filtfilt, hilbert
import numpy as np

from ..signal_transform import *
from ..feature_extraction import *
from ..thresholds import *
from ..io.data_operations import *

# TODO - finish function help

# %% Presets - Sergey's custom filter
mod_dir = os.path.split(__file__)[0]
filter_coefs = pickle.load(open(mod_dir+'/Morphology_detector.pkl','rb'))

# %% Morphology detector

def morphology_detect(data, fs, low_fc, high_fc, mark,
                      bl_mu = .9, cdf_rms = .95, cdf_filt = .99, bs_dur = 30,
                      dur_th = .99, time_th = .02, max_noise_uv = 10.,
                      max_amp = 30., max_gap = .02, min_N_osc = 6):
    """
    Morphology detector by Sergey Burnos et al.\n
    Please cite!\n
    \n
    Burnos et al., 2016 Clinical Neurophysiology\n
    \n

    Parameters:
    -----------
    data(1-d numpy array) - raw data\n
    fs(int) - sampling frequency\n
    low_fc(float) - low cut-off frequency, allows only 80, 250\n
    high_fc(float) - high cut-off frequency, allows onlt 250, 500\n
    bl_mu(float) - level for maximum entropy\n
    cdf_rms(float) - percentile of detected baselines, incr in trheshold\n
    cdf_filt(float) - percentile of detected baselines, incr in trheshold\n
    bs_dur(float) - number of seconds for baseline detection\n
    dur_th(float) - duraiton threshold

    max_amp(float) - max event amplitude - artifact rejection\n
    max_gap(float) - interval for joining events
    min_N_osc(float) - minimum number of oscillations

    Returns:
    --------
    df_out(pandas.DataFrame) - output dataframe with detections\n
    """

    # Create output dataframe

    df_out = create_output_df(fields=['peak','peak_amp'])

    # Define some additional parameters -could go into parameters
    time_th = np.ceil(time_th * fs)
    smooth_window = 1 / high_fc  # RMS smoothing window
    bl_border = .02  # Ignore bordeers because of ST
    bl_mindist = 10*fs/1e3  # Min distance interval from baseline

    if  mark == 'Ripple':
        max_amp_filt = 30
    elif mark == 'FastRipple':
        max_amp_filt = 20

    # 1) Filtering
    if low_fc == 80:
        b = filter_coefs['Rb']
        a = filter_coefs['Ra']
    elif low_fc == 250:
        b = filter_coefs['FRb']
        a = filter_coefs['FRa']
    filt_data = filtfilt(b, a, data)

    # 2) Envelope
    env = smooth(np.abs(hilbert(filt_data)),smooth_window * fs)

    # 3) threshold
    thr, thr_filt, indHighEntr = baseline_threshold(data, filt_data, env,
                                                   bs_dur, bl_mu, bl_border,
                                                   bl_mindist, max_noise_uv,
                                                   fs, low_fc, high_fc)

    # Display warning if baseline is too short
    if len(indHighEntr) < 2*fs:
        print('!!!!Short baseline!!!!')

    # 4) Stage 1 - detection of EoIs

    env[0] = env[-1] = 0
    pred_env = np.zeros(len(env))
    pred_env[1:] = env[:-1]
    pred_env[0] = pred_env[1]

    t1 = np.where(pred_env < (thr * dur_th) & env >= (thr * dur_thr))
    t2 = np.where(pred_env > (thr * dur_th) & env <= (thr * dur_thr))

    trig = np.where(pred_env < thr & env >= thr)
    trig_end = np.where(pred_enc >= thr & env < thr)

    det_cnt = 0

    for trigs in zip(trig, trig_end):
        if trigs[1] - trigs[0] >= time_th:

            k = np.where(t1 <= trigs[0] & t2 >= trigs[0])

            if t1[k] > 0:
                det_start = t1[k]
            else:
                det_start = 0

            if t2[k] <= len(env):
                det_stop = t2[k]
            else:
                det_stop = len(env)

            peak_amp = np.max(env[t1[k]:t2[k]])
            peak_ind = np.argmax(env[t1[k]:t2[k]])

            if peak_amp > max_amp_filt:
                continue

            df_out.loc[det_cnt] = [det_start, det_stop, peak_ind, peak_amp]
            det_cnt += 1

    if det_cnt:
        df_out = check_oscillations(df_out, filt_data, thr_filt, min_N_osc) #FIXME - should this be a general function?
        df_out = join_detections(df_out, max_gap) #FIXME - this should be general function

    return df_out

# %% Helper functions for morphology detect

#TODO - check that this functions corresponds with matlab smooth function.

def check_oscillations(df, data, thr_filt, min_N_osc):
    """
    Function to reject oscillations that have less than 8 peaks.
    """

    rejected_idcs = []
    for row in df.iterrows():
        # Detrend data
        to_detrend = 0

        # Get eeg interval
        interval_eeg = data[row[1].event_start:row[1].event_stop] - to_detrend

        # Compute absolute values for oscillations interval
        abs_eeg = np.abs(interval_eeg)

        # Look for zeros
        zero_vec = np.argmax(interval_eeg[:-1] * interval_eeg[1:] < 0)
        N_zeros = len(zero_vec)

        N_max_counter = np.zeros(N_zeros)

        if N_zeros:
            for zi, z in enumerate(zero_vec[:-1]):
                cross_start = z
                cross_stop = zero_vec[zi+1]
                d_max = np.max(abs_eeg[cross_start:cross_stop])

                if d_max > thr_filt:
                    N_max_counter[zi] = 1
                else:
                    N_max_counter[zi] = 0

        N_max_counter = np.concatenate([[0],N_max_counter,[0]])

        # NOTE: inversed logic from the original code - is more readable this way
        if not True in (np.diff(np.where(N_max_counter==0)) > min_N_osc):
            rejected_idcs.append(row[0])

    # Get rid of detections that did not have enough oscillations
    df = df.loc[~df.index.isin(rejected_idcs)]
    df = df.reindex()

    return df

def join_detections(df, max_gap):
    """
    Function to join detections.
    """

    N_det_c = 0

    joined_df = df[0:1]

    for row in df[1:].iterrows():
        # Join detection
        if row[1].event_start > joined_df.loc[N_det_c,'event_stop']:
            n_diff = row[1].event_start - joined_df.loc[N_det_c,'event_stop']

            if n_diff < max_gap:
                joined_df.loc[N_det_c,'event_stop'] = row[1].event_stop

            if joined_df.loc[N_det_c,'peak_amp'] < row[1].peak_amp:
                joined_df.loc[N_det_c,'peak_amp'] = row[1].peak_amp
                joined_df.loc[N_det_c,'peak'] = row[1].peak

        else:
            N_det_c += 1
            joined_df.loc[N_det_c] = row[1]

    return joined_df

def ecdf(x):
    xs = np.sort(x)
    ys = np.arange(1, len(xs)+1)/float(len(xs))
    return xs, ys

def smooth(data,N):
    """
    Function to smooth the signal by running mean.

    Parameters:
    -----------
    data(array) - data to be smoothed.\n
    N(int) - sliding window size\n

    Returns:
    --------
    smoothed_data(array)\n
    """

    cumsum = np.cumsum(np.insert(data, 0, 0))
    return (cumsum[N:] - cumsum[:-N]) / N

def baseline_threshold(data, filt_data, env,
                       bs_dur, bl_mu, bl_border, bl_mindist, maxNoiseuV,
                       cdf_level_filt,
                       fs, low_fc, high_fc):
    """
    distinguish background activity from spikes-HFO-artifacts
    according to Stockwell entrophy
    ref: wavelet entrophy: a new tool for analysis...
    Osvaldo a. Rosso et al, journal of neuroscience methods, 2000

    Parameters:
    -----------
    data(array) - original data\n
    filt_data(array) - filtered data\n
    env(array) - amplitude envelope\n

    Returns:
    --------

    """

    # Parameters
    indHighEntr=np.array([])
    S=np.zeros(fs)

    # Check duration

    if bs_dur > len(data) / fs:
        bs_dur = np.floor(len(data) / fs)

    for sec in range(len(np.floor(len(data) / fs))):

        signal = data[sec * fs : (sec + 1) * fs]

        # S transform
        ST_data = stockwell_transform(signal, low_fc+1, high_fc, 1/fs, 1)
        stda = np.square(np.abs(ST_data))

        # Stockwell entropy
        std_total = np.sum(stda)  # Total energy
        std_rel = stda / std_total

        # Total entropy
        S = []
        for ifr in std_rel:
            S.append(-sum(ifr) * np.log(ifr))

        Smax=np.log(np.size(stda, 1)) # maximum entrophy = log(f_ST), /mu in mni,

        ### Threshold and baseline ###

        thr=bl_mu*Smax # threshold at mu*Smax, in mni BLmu=0.67
        indAboveThr=(S>thr) # find pt with high entrophy

        if indAboveThr:
            # FIXME - could we eliminate one if?
            # dont take border points because of stockwell transf
            indAboveThr=indAboveThr[~indAboveThr<(fs * bl_border)]
            indAboveThr=indAboveThr[~indAboveThr>(fs * (1-bl_border))]

            if indAboveThr:

                # check for the length
                indAboveThrN = indAboveThr[1:]
                indBrake = np.where(indAboveThrN - indAboveThrN[:-1] > 1)

                # check if it starts already above or the last point is abover the threshold

                if indAboveThr[0] == fs * bl_border: #??? This should not happen!!!
                    indBrake = indBrake[1:]
                if indAboveThr[-1] == fs * (1-bl_border): #??? Neither should this happen!!!
                    indBrake = indBrake[:-1]

                if np.size(indBrake) == 0:
                    indBrake = len(indAboveThr)

            for iper in range(indBrake[:-1]):
                j = np.arange(indBrake[iper]+1,indBrake[iper+1])
                if len(j) >= bl_mindist:
                    indAboveThr[j] = indAboveThr[j] + (sec-1) *fs
                    if sum(abs(filt_data[indAboveThr[j]])) <= maxNoiseuV:  # % check that filtered signal is below max Noise level
                        indHighEntr = np.concatenate([indHighEntr,indAboveThr[j]])

    ### check one more time if the lentgh of baseline is too small ###
    if len(indHighEntr) < 2*fs:
        print('Baseline length < 2 sec, calculating for 5 min ')

        for sec in range(len(np.floor(len(data) / fs))):

            signal = data[sec * fs : (sec + 1) * fs]

            # S transform
            ST_data = stockwell_transform(signal, low_fc+1, high_fc, 1/fs, 1)
            stda = np.square(np.abs(ST_data))

            # Stockwell entropy
            std_total = np.sum(stda)  # Total energy
            std_rel = stda / std_total

            # Total entropy
            S = []
            for ifr in std_rel:
                S.append(-sum(ifr) * np.log(ifr))

            Smax=np.log(np.size(stda, 1)) # maximum entrophy = log(f_ST), /mu in mni,

            ### Threshold and baseline ###

            thr=bl_mu*Smax # threshold at mu*Smax, in mni BLmu=0.67
            indAboveThr=(S>thr) # find pt with high entrophy

            if indAboveThr:
                # FIXME - could we eliminate one if?
                # dont take border points because of stockwell transf
                indAboveThr=indAboveThr[~indAboveThr<(fs * bl_border)]
                indAboveThr=indAboveThr[~indAboveThr>(fs * (1-bl_border))]

                if indAboveThr:

                    # check for the length
                    indAboveThrN = indAboveThr[1:]
                    indBrake = np.where(indAboveThrN - indAboveThrN[:-1] > 1)

                    # check if it starts already above or the last point is abover the threshold

                    if indAboveThr[0] == fs * bl_border: #??? This should not happen!!!
                        indBrake = indBrake[1:]
                    if indAboveThr[-1] == fs * (1-bl_border): #??? Neither should this happen!!!
                        indBrake = indBrake[:-1]

                    if np.size(indBrake) == 0:
                        indBrake = len(indAboveThr)

                for iper in range(indBrake[:-1]):
                    j = np.arange(indBrake[iper]+1,indBrake[iper+1])
                    if len(j) >= bl_mindist:
                        indAboveThr[j] = indAboveThr[j] + (sec-1) *fs
                        if sum(abs(filt_data[indAboveThr[j]])) <= maxNoiseuV:  # % check that filtered signal is below max Noise level
                            indHighEntr = np.concatenate([indHighEntr,indAboveThr[j]])

    print('For '+str(npfloor(len(data)/fs))+' sec, baseline length '+str(len(indHighEntr)/fs)+' sec')

    if len(indHighEntr):
        xs, ys = ecdf(env[indHighEntr])
        thr_cdf = xs[ys>cdf_level_filt]
    else:
        thr_cdf = 1000

    thr = thr_cdf

    if len(indHighEntr):
        xs, ys = ecdf(filt_data(indHighEntr))
        thr_cdf = xs[ys>cdf_level_filt]
    else:
        thr_cdf = 1000

    thr_filt = thr_cdf

    # show thresholds
    print('ThrEnv = '+str(thr)+', ThrFiltSig = '+str(thr_filt))

    return thr, thr_filt, indHighEntr<commit_msg>Removed unnecessary smooth function. Using convolution instead.<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Jun 25 10:18:40 2016

Morphology detector provided by Sergey Burnos.
Reprogrammed to python by Jan Cimbalnik..

If you use this detector please cite!

@author: jan_cimbalnik
"""
import pickle,os

from scipy.signal import butter, filtfilt, hilbert
import numpy as np

from ..signal_transform import *
from ..feature_extraction import *
from ..thresholds import *
from ..io.data_operations import *

# TODO - finish function help

# %% Presets - Sergey's custom filter
mod_dir = os.path.split(__file__)[0]
filter_coefs = pickle.load(open(mod_dir+'/Morphology_detector.pkl','rb'))

# %% Morphology detector

def morphology_detect(data, fs, low_fc, high_fc, mark,
                      bl_mu = .9, cdf_rms = .95, cdf_filt = .99, bs_dur = 30,
                      dur_th = .99, time_th = .02, max_noise_uv = 10.,
                      max_amp = 30., max_gap = .02, min_N_osc = 6):
    """
    Morphology detector by Sergey Burnos et al.\n
    Please cite!\n
    \n
    Burnos et al., 2016 Clinical Neurophysiology\n
    \n

    Parameters:
    -----------
    data(1-d numpy array) - raw data\n
    fs(int) - sampling frequency\n
    low_fc(float) - low cut-off frequency, allows only 80, 250\n
    high_fc(float) - high cut-off frequency, allows onlt 250, 500\n
    bl_mu(float) - level for maximum entropy\n
    cdf_rms(float) - percentile of detected baselines, incr in trheshold\n
    cdf_filt(float) - percentile of detected baselines, incr in trheshold\n
    bs_dur(float) - number of seconds for baseline detection\n
    dur_th(float) - duraiton threshold

    max_amp(float) - max event amplitude - artifact rejection\n
    max_gap(float) - interval for joining events
    min_N_osc(float) - minimum number of oscillations

    Returns:
    --------
    df_out(pandas.DataFrame) - output dataframe with detections\n
    """

    # Create output dataframe

    df_out = create_output_df(fields=['peak','peak_amp'])

    # Define some additional parameters -could go into parameters
    time_th = np.ceil(time_th * fs)
    smooth_window = 1 / high_fc  # RMS smoothing window
    bl_border = .02  # Ignore bordeers because of ST
    bl_mindist = 10*fs/1e3  # Min distance interval from baseline

    if  mark == 'Ripple':
        max_amp_filt = 30
    elif mark == 'FastRipple':
        max_amp_filt = 20

    # 1) Filtering
    if low_fc == 80:
        b = filter_coefs['Rb']
        a = filter_coefs['Ra']
    elif low_fc == 250:
        b = filter_coefs['FRb']
        a = filter_coefs['FRa']
    filt_data = filtfilt(b, a, data)

    # 2) Envelope
    smooth_N = int(smooth_window * fs)
    env = np.abs(hilbert(filt_data))
    env = np.convolve(env, np.ones((smooth_N,))/smooth_N, mode='same')

    # 3) threshold
    thr, thr_filt, indHighEntr = baseline_threshold(data, filt_data, env,
                                                   bs_dur, bl_mu, bl_border,
                                                   bl_mindist, max_noise_uv,
                                                   fs, low_fc, high_fc)

    # Display warning if baseline is too short
    if len(indHighEntr) < 2*fs:
        print('!!!!Short baseline!!!!')

    # 4) Stage 1 - detection of EoIs

    env[0] = env[-1] = 0
    pred_env = np.zeros(len(env))
    pred_env[1:] = env[:-1]
    pred_env[0] = pred_env[1]

    t1 = np.where(pred_env < (thr * dur_th) & env >= (thr * dur_thr))
    t2 = np.where(pred_env > (thr * dur_th) & env <= (thr * dur_thr))

    trig = np.where(pred_env < thr & env >= thr)
    trig_end = np.where(pred_enc >= thr & env < thr)

    det_cnt = 0

    for trigs in zip(trig, trig_end):
        if trigs[1] - trigs[0] >= time_th:

            k = np.where(t1 <= trigs[0] & t2 >= trigs[0])

            if t1[k] > 0:
                det_start = t1[k]
            else:
                det_start = 0

            if t2[k] <= len(env):
                det_stop = t2[k]
            else:
                det_stop = len(env)

            peak_amp = np.max(env[t1[k]:t2[k]])
            peak_ind = np.argmax(env[t1[k]:t2[k]])

            if peak_amp > max_amp_filt:
                continue

            df_out.loc[det_cnt] = [det_start, det_stop, peak_ind, peak_amp]
            det_cnt += 1

    if det_cnt:
        df_out = check_oscillations(df_out, filt_data, thr_filt, min_N_osc) #FIXME - should this be a general function?
        df_out = join_detections(df_out, max_gap) #FIXME - this should be general function

    return df_out

# %% Helper functions for morphology detect

#TODO - check that this functions corresponds with matlab smooth function.

def check_oscillations(df, data, thr_filt, min_N_osc):
    """
    Function to reject oscillations that have less than 8 peaks.
    """

    rejected_idcs = []
    for row in df.iterrows():
        # Detrend data
        to_detrend = 0

        # Get eeg interval
        interval_eeg = data[row[1].event_start:row[1].event_stop] - to_detrend

        # Compute absolute values for oscillations interval
        abs_eeg = np.abs(interval_eeg)

        # Look for zeros
        zero_vec = np.argmax(interval_eeg[:-1] * interval_eeg[1:] < 0)
        N_zeros = len(zero_vec)

        N_max_counter = np.zeros(N_zeros)

        if N_zeros:
            for zi, z in enumerate(zero_vec[:-1]):
                cross_start = z
                cross_stop = zero_vec[zi+1]
                d_max = np.max(abs_eeg[cross_start:cross_stop])

                if d_max > thr_filt:
                    N_max_counter[zi] = 1
                else:
                    N_max_counter[zi] = 0

        N_max_counter = np.concatenate([[0],N_max_counter,[0]])

        # NOTE: inversed logic from the original code - is more readable this way
        if not True in (np.diff(np.where(N_max_counter==0)) > min_N_osc):
            rejected_idcs.append(row[0])

    # Get rid of detections that did not have enough oscillations
    df = df.loc[~df.index.isin(rejected_idcs)]
    df = df.reindex()

    return df

def join_detections(df, max_gap):
    """
    Function to join detections.
    """

    N_det_c = 0

    joined_df = df[0:1]

    for row in df[1:].iterrows():
        # Join detection
        if row[1].event_start > joined_df.loc[N_det_c,'event_stop']:
            n_diff = row[1].event_start - joined_df.loc[N_det_c,'event_stop']

            if n_diff < max_gap:
                joined_df.loc[N_det_c,'event_stop'] = row[1].event_stop

            if joined_df.loc[N_det_c,'peak_amp'] < row[1].peak_amp:
                joined_df.loc[N_det_c,'peak_amp'] = row[1].peak_amp
                joined_df.loc[N_det_c,'peak'] = row[1].peak

        else:
            N_det_c += 1
            joined_df.loc[N_det_c] = row[1]

    return joined_df

def ecdf(x):
    xs = np.sort(x)
    ys = np.arange(1, len(xs)+1)/float(len(xs))
    return xs, ys

def baseline_threshold(data, filt_data, env,
                       bs_dur, bl_mu, bl_border, bl_mindist, maxNoiseuV,
                       cdf_level_filt,
                       fs, low_fc, high_fc):
    """
    distinguish background activity from spikes-HFO-artifacts
    according to Stockwell entrophy
    ref: wavelet entrophy: a new tool for analysis...
    Osvaldo a. Rosso et al, journal of neuroscience methods, 2000

    Parameters:
    -----------
    data(array) - original data\n
    filt_data(array) - filtered data\n
    env(array) - amplitude envelope\n

    Returns:
    --------

    """

    # Parameters
    indHighEntr=np.array([])
    S=np.zeros(fs)

    # Check duration

    if bs_dur > len(data) / fs:
        bs_dur = np.floor(len(data) / fs)

    for sec in range(len(np.floor(len(data) / fs))):

        signal = data[sec * fs : (sec + 1) * fs]

        # S transform
        ST_data = stockwell_transform(signal, low_fc+1, high_fc, 1/fs, 1)
        stda = np.square(np.abs(ST_data))

        # Stockwell entropy
        std_total = np.sum(stda)  # Total energy
        std_rel = stda / std_total

        # Total entropy
        S = []
        for ifr in std_rel:
            S.append(-sum(ifr) * np.log(ifr))

        Smax=np.log(np.size(stda, 1)) # maximum entrophy = log(f_ST), /mu in mni,

        ### Threshold and baseline ###

        thr=bl_mu*Smax # threshold at mu*Smax, in mni BLmu=0.67
        indAboveThr=(S>thr) # find pt with high entrophy

        if indAboveThr:
            # FIXME - could we eliminate one if?
            # dont take border points because of stockwell transf
            indAboveThr=indAboveThr[~indAboveThr<(fs * bl_border)]
            indAboveThr=indAboveThr[~indAboveThr>(fs * (1-bl_border))]

            if indAboveThr:

                # check for the length
                indAboveThrN = indAboveThr[1:]
                indBrake = np.where(indAboveThrN - indAboveThrN[:-1] > 1)

                # check if it starts already above or the last point is abover the threshold

                if indAboveThr[0] == fs * bl_border: #??? This should not happen!!!
                    indBrake = indBrake[1:]
                if indAboveThr[-1] == fs * (1-bl_border): #??? Neither should this happen!!!
                    indBrake = indBrake[:-1]

                if np.size(indBrake) == 0:
                    indBrake = len(indAboveThr)

            for iper in range(indBrake[:-1]):
                j = np.arange(indBrake[iper]+1,indBrake[iper+1])
                if len(j) >= bl_mindist:
                    indAboveThr[j] = indAboveThr[j] + (sec-1) *fs
                    if sum(abs(filt_data[indAboveThr[j]])) <= maxNoiseuV:  # % check that filtered signal is below max Noise level
                        indHighEntr = np.concatenate([indHighEntr,indAboveThr[j]])

    ### check one more time if the lentgh of baseline is too small ###
    if len(indHighEntr) < 2*fs:
        print('Baseline length < 2 sec, calculating for 5 min ')

        for sec in range(len(np.floor(len(data) / fs))):

            signal = data[sec * fs : (sec + 1) * fs]

            # S transform
            ST_data = stockwell_transform(signal, low_fc+1, high_fc, 1/fs, 1)
            stda = np.square(np.abs(ST_data))

            # Stockwell entropy
            std_total = np.sum(stda)  # Total energy
            std_rel = stda / std_total

            # Total entropy
            S = []
            for ifr in std_rel:
                S.append(-sum(ifr) * np.log(ifr))

            Smax=np.log(np.size(stda, 1)) # maximum entrophy = log(f_ST), /mu in mni,

            ### Threshold and baseline ###

            thr=bl_mu*Smax # threshold at mu*Smax, in mni BLmu=0.67
            indAboveThr=(S>thr) # find pt with high entrophy

            if indAboveThr:
                # FIXME - could we eliminate one if?
                # dont take border points because of stockwell transf
                indAboveThr=indAboveThr[~indAboveThr<(fs * bl_border)]
                indAboveThr=indAboveThr[~indAboveThr>(fs * (1-bl_border))]

                if indAboveThr:

                    # check for the length
                    indAboveThrN = indAboveThr[1:]
                    indBrake = np.where(indAboveThrN - indAboveThrN[:-1] > 1)

                    # check if it starts already above or the last point is abover the threshold

                    if indAboveThr[0] == fs * bl_border: #??? This should not happen!!!
                        indBrake = indBrake[1:]
                    if indAboveThr[-1] == fs * (1-bl_border): #??? Neither should this happen!!!
                        indBrake = indBrake[:-1]

                    if np.size(indBrake) == 0:
                        indBrake = len(indAboveThr)

                for iper in range(indBrake[:-1]):
                    j = np.arange(indBrake[iper]+1,indBrake[iper+1])
                    if len(j) >= bl_mindist:
                        indAboveThr[j] = indAboveThr[j] + (sec-1) *fs
                        if sum(abs(filt_data[indAboveThr[j]])) <= maxNoiseuV:  # % check that filtered signal is below max Noise level
                            indHighEntr = np.concatenate([indHighEntr,indAboveThr[j]])

    print('For '+str(npfloor(len(data)/fs))+' sec, baseline length '+str(len(indHighEntr)/fs)+' sec')

    if len(indHighEntr):
        xs, ys = ecdf(env[indHighEntr])
        thr_cdf = xs[ys>cdf_level_filt]
    else:
        thr_cdf = 1000

    thr = thr_cdf

    if len(indHighEntr):
        xs, ys = ecdf(filt_data(indHighEntr))
        thr_cdf = xs[ys>cdf_level_filt]
    else:
        thr_cdf = 1000

    thr_filt = thr_cdf

    # show thresholds
    print('ThrEnv = '+str(thr)+', ThrFiltSig = '+str(thr_filt))

    return thr, thr_filt, indHighEntr<|endoftext|>


<commit_before>from collections import namedtuple
from datetime import datetime, time
from corehq.apps.reports_core.exceptions import FilterValueException
from corehq.apps.userreports.expressions.getters import transform_from_datatype
from corehq.apps.userreports.reports.filters.values import SHOW_ALL_CHOICE, CHOICE_DELIMITER
from corehq.apps.userreports.util import localize
from corehq.util.dates import iso_string_to_date

from dimagi.utils.dates import DateSpan
from dimagi.utils.decorators.memoized import memoized
from django.utils.translation import ugettext_lazy as _

FilterParam = namedtuple('FilterParam', ['name', 'required'])

class BaseFilter(object):
    """
    Base object for filters.
    """

    def __init__(self, name, params=None):
        self.name = name
        self.params = params or []

    def get_value(self, context):
        if self.check_context(context):
            kwargs = {param.name: context[param.name] for param in self.params if param.name in context}
            return self.value(**kwargs)
        else:
            return self.default_value()

    def check_context(self, context):
        return all(slug.name in context for slug in self.params if slug.required)

    def value(self, **kwargs):
        """
        Override this and return the value. This method will only be called if all required
        parameters are present in the filter context. All the parameters present in the context
        will be passed in as keyword arguments.

        If any of the parameters are invalid a FilterValueException should be raised.

        This method should generally be memoized.
        """
        return None

    def default_value(self):
        """
        If the filter is not marked as required and the user does not supply any / all necessary parameters
        this value will be used instead.
        """
        return None

    def context(self, value, lang=None):
        """
        Context for rendering the filter.
        """
        context = {
            'label': localize(self.label, lang),
            'css_id': self.css_id,
            'value': value,
        }
        context.update(self.filter_context())
        return context

    def filter_context(self):
        """
        Override to supply additional context.
        """
        return {}

class DatespanFilter(BaseFilter):
    template = 'reports_core/filters/datespan_filter/datespan_filter.html'
    javascript_template = 'reports_core/filters/datespan_filter/datespan_filter.js'

    def __init__(self, name, label='Datespan Filter',
                 css_id=None):
        self.label = label
        self.css_id = css_id or name
        params = [
            FilterParam(self.startdate_param_name, True),
            FilterParam(self.enddate_param_name, True),
            FilterParam('date_range_inclusive', False),
        ]
        super(DatespanFilter, self).__init__(name=name, params=params)

    @property
    def startdate_param_name(self):
        return '{}-start'.format(self.css_id)

    @property
    def enddate_param_name(self):
        return '{}-end'.format(self.css_id)

    @memoized
    def value(self, **kwargs):
        startdate = kwargs[self.startdate_param_name]
        enddate = kwargs[self.enddate_param_name]
        date_range_inclusive = kwargs.get('date_range_inclusive', True)

        def date_or_nothing(param):
            if param:
                return datetime.combine(iso_string_to_date(param), time())
            else:
                return None
        try:
            startdate = date_or_nothing(startdate)
            enddate = date_or_nothing(enddate)
        except (ValueError, TypeError) as e:
            raise FilterValueException('Error parsing date parameters: {}'.format(e.message))

        if startdate or enddate:
            return DateSpan(startdate, enddate, inclusive=date_range_inclusive)

    def default_value(self):
        # default to "Show All Dates"
        return None

    def filter_context(self):
        return {
            'timezone': None
        }

class NumericFilter(BaseFilter):
    template = "reports_core/filters/numeric_filter.html"

    def __init__(self, name, label=_('Numeric Filter'), css_id=None):
        self.label = label
        self.css_id = css_id or name
        params = [
            FilterParam(self.operator_param_name, True),
            FilterParam(self.operand_param_name, True),
        ]
        super(NumericFilter, self).__init__(name=name, params=params)

    @property
    def operator_param_name(self):
        return "{}-operator".format(self.css_id)

    @property
    def operand_param_name(self):
        return "{}-operand".format(self.css_id)

    @memoized
    def value(self, **kwargs):
        operator = kwargs[self.operator_param_name]
        operand = kwargs[self.operand_param_name]
        if operand == "":
            return None
        try:
            assert operator in ["=", "!=", "<", "<=", ">", ">="]
            assert isinstance(operand, float) or isinstance(operand, int)
        except AssertionError as e:
            raise FilterValueException('Error parsing numeric filter parameters: {}'.format(e.message))

        return {"operator": operator, "operand": operand}

    def default_value(self):
        return None

Choice = namedtuple('Choice', ['value', 'display'])

class ChoiceListFilter(BaseFilter):
    """
    Filter for a list of choices. Each choice should be a Choice object as per above.
    """

    def __init__(self, name, datatype, label='Choice List Filter',
                 template='reports_core/filters/choice_list_filter.html',
                 css_id=None, choices=None):
        params = [
            FilterParam(name, True),
        ]
        super(ChoiceListFilter, self).__init__(name=name, params=params)
        self.datatype = datatype
        self.label = label
        self.template = template
        self.css_id = css_id or self.name
        self.choices = choices or []

    def value(self, **kwargs):
        raw_value = kwargs[self.name]
        choice = transform_from_datatype(self.datatype)(raw_value) if raw_value != SHOW_ALL_CHOICE else raw_value
        choice_values = map(lambda c: c.value, self.choices)
        if choice not in choice_values:
            raise FilterValueException(_(u'Choice "{choice}" not found in choices: {choices}')
                                       .format(choice=choice,
                                               choices=choice_values))
        return next(choice_obj for choice_obj in self.choices if choice_obj.value == choice)

    def default_value(self):
        return self.choices[0]

class DynamicChoiceListFilter(BaseFilter):
    """
    Filter for a list of choices.

    The choices are generated dynamically based on the database.
    """
    template = 'reports_core/filters/dynamic_choice_list_filter/dynamic_choice_list.html'
    javascript_template = 'reports_core/filters/dynamic_choice_list_filter/dynamic_choice_list.js'

    def __init__(self, name, field, datatype, label, show_all, url_generator, choice_provider, css_id=None):
        """
        url_generator should be a callable that takes a domain, report, and filter and returns a url.
        see userreports.reports.filters.dynamic_choice_list_url for an example.
        """
        params = [
            FilterParam(name, True),
        ]
        super(DynamicChoiceListFilter, self).__init__(name=name, params=params)
        self.datatype = datatype
        self.field = field
        self.label = label
        self.show_all = show_all
        self.css_id = css_id or self.name
        self.url_generator = url_generator
        self.choice_provider = choice_provider

    def value(self, **kwargs):
        selection = unicode(kwargs.get(self.name, ""))
        if selection:
            choices = selection.split(CHOICE_DELIMITER)
            typed_choices = [transform_from_datatype(self.datatype)(c) for c in choices]
            return [Choice(tc, c) for (tc, c) in zip(typed_choices, choices)]
        return self.default_value()

    def default_value(self):
        return [Choice(SHOW_ALL_CHOICE, '')]
<commit_msg>cleanup templates<commit_after>from collections import namedtuple
from datetime import datetime, time
from corehq.apps.reports_core.exceptions import FilterValueException
from corehq.apps.userreports.expressions.getters import transform_from_datatype
from corehq.apps.userreports.reports.filters.values import SHOW_ALL_CHOICE, CHOICE_DELIMITER
from corehq.apps.userreports.util import localize
from corehq.util.dates import iso_string_to_date

from dimagi.utils.dates import DateSpan
from dimagi.utils.decorators.memoized import memoized
from django.utils.translation import ugettext_lazy as _

FilterParam = namedtuple('FilterParam', ['name', 'required'])

class BaseFilter(object):
    """
    Base object for filters.
    """
    template = None
    javascript_template = None

    def __init__(self, name, params=None):
        self.name = name
        self.params = params or []

    def get_value(self, context):
        if self.check_context(context):
            kwargs = {param.name: context[param.name] for param in self.params if param.name in context}
            return self.value(**kwargs)
        else:
            return self.default_value()

    def check_context(self, context):
        return all(slug.name in context for slug in self.params if slug.required)

    def value(self, **kwargs):
        """
        Override this and return the value. This method will only be called if all required
        parameters are present in the filter context. All the parameters present in the context
        will be passed in as keyword arguments.

        If any of the parameters are invalid a FilterValueException should be raised.

        This method should generally be memoized.
        """
        return None

    def default_value(self):
        """
        If the filter is not marked as required and the user does not supply any / all necessary parameters
        this value will be used instead.
        """
        return None

    def context(self, value, lang=None):
        """
        Context for rendering the filter.
        """
        context = {
            'label': localize(self.label, lang),
            'css_id': self.css_id,
            'value': value,
        }
        context.update(self.filter_context())
        return context

    def filter_context(self):
        """
        Override to supply additional context.
        """
        return {}

class DatespanFilter(BaseFilter):
    template = 'reports_core/filters/datespan_filter/datespan_filter.html'
    javascript_template = 'reports_core/filters/datespan_filter/datespan_filter.js'

    def __init__(self, name, label='Datespan Filter',
                 css_id=None):
        self.label = label
        self.css_id = css_id or name
        params = [
            FilterParam(self.startdate_param_name, True),
            FilterParam(self.enddate_param_name, True),
            FilterParam('date_range_inclusive', False),
        ]
        super(DatespanFilter, self).__init__(name=name, params=params)

    @property
    def startdate_param_name(self):
        return '{}-start'.format(self.css_id)

    @property
    def enddate_param_name(self):
        return '{}-end'.format(self.css_id)

    @memoized
    def value(self, **kwargs):
        startdate = kwargs[self.startdate_param_name]
        enddate = kwargs[self.enddate_param_name]
        date_range_inclusive = kwargs.get('date_range_inclusive', True)

        def date_or_nothing(param):
            if param:
                return datetime.combine(iso_string_to_date(param), time())
            else:
                return None
        try:
            startdate = date_or_nothing(startdate)
            enddate = date_or_nothing(enddate)
        except (ValueError, TypeError) as e:
            raise FilterValueException('Error parsing date parameters: {}'.format(e.message))

        if startdate or enddate:
            return DateSpan(startdate, enddate, inclusive=date_range_inclusive)

    def default_value(self):
        # default to "Show All Dates"
        return None

    def filter_context(self):
        return {
            'timezone': None
        }

class NumericFilter(BaseFilter):
    template = "reports_core/filters/numeric_filter.html"

    def __init__(self, name, label=_('Numeric Filter'), css_id=None):
        self.label = label
        self.css_id = css_id or name
        params = [
            FilterParam(self.operator_param_name, True),
            FilterParam(self.operand_param_name, True),
        ]
        super(NumericFilter, self).__init__(name=name, params=params)

    @property
    def operator_param_name(self):
        return "{}-operator".format(self.css_id)

    @property
    def operand_param_name(self):
        return "{}-operand".format(self.css_id)

    @memoized
    def value(self, **kwargs):
        operator = kwargs[self.operator_param_name]
        operand = kwargs[self.operand_param_name]
        if operand == "":
            return None
        try:
            assert operator in ["=", "!=", "<", "<=", ">", ">="]
            assert isinstance(operand, float) or isinstance(operand, int)
        except AssertionError as e:
            raise FilterValueException('Error parsing numeric filter parameters: {}'.format(e.message))

        return {"operator": operator, "operand": operand}

    def default_value(self):
        return None

Choice = namedtuple('Choice', ['value', 'display'])

class ChoiceListFilter(BaseFilter):
    """
    Filter for a list of choices. Each choice should be a Choice object as per above.
    """
    template = 'reports_core/filters/choice_list_filter.html'

    def __init__(self, name, datatype, label='Choice List Filter',
                 css_id=None, choices=None):
        params = [
            FilterParam(name, True),
        ]
        super(ChoiceListFilter, self).__init__(name=name, params=params)
        self.datatype = datatype
        self.label = label
        self.css_id = css_id or self.name
        self.choices = choices or []

    def value(self, **kwargs):
        raw_value = kwargs[self.name]
        choice = transform_from_datatype(self.datatype)(raw_value) if raw_value != SHOW_ALL_CHOICE else raw_value
        choice_values = map(lambda c: c.value, self.choices)
        if choice not in choice_values:
            raise FilterValueException(_(u'Choice "{choice}" not found in choices: {choices}')
                                       .format(choice=choice,
                                               choices=choice_values))
        return next(choice_obj for choice_obj in self.choices if choice_obj.value == choice)

    def default_value(self):
        return self.choices[0]

class DynamicChoiceListFilter(BaseFilter):
    """
    Filter for a list of choices.

    The choices are generated dynamically based on the database.
    """
    template = 'reports_core/filters/dynamic_choice_list_filter/dynamic_choice_list.html'
    javascript_template = 'reports_core/filters/dynamic_choice_list_filter/dynamic_choice_list.js'

    def __init__(self, name, field, datatype, label, show_all, url_generator, choice_provider, css_id=None):
        """
        url_generator should be a callable that takes a domain, report, and filter and returns a url.
        see userreports.reports.filters.dynamic_choice_list_url for an example.
        """
        params = [
            FilterParam(name, True),
        ]
        super(DynamicChoiceListFilter, self).__init__(name=name, params=params)
        self.datatype = datatype
        self.field = field
        self.label = label
        self.show_all = show_all
        self.css_id = css_id or self.name
        self.url_generator = url_generator
        self.choice_provider = choice_provider

    def value(self, **kwargs):
        selection = unicode(kwargs.get(self.name, ""))
        if selection:
            choices = selection.split(CHOICE_DELIMITER)
            typed_choices = [transform_from_datatype(self.datatype)(c) for c in choices]
            return [Choice(tc, c) for (tc, c) in zip(typed_choices, choices)]
        return self.default_value()

    def default_value(self):
        return [Choice(SHOW_ALL_CHOICE, '')]
<|endoftext|>