<commit_before>from __future__ import absolute_import
from collections import defaultdict, namedtuple
from datetime import datetime, date, time
import logging
import re
from lxml import etree
import os
from django.conf import settings
from corehq.apps.app_manager.util import all_apps_by_domain
from corehq.util.quickcache import quickcache
from couchforms.models import XFormDeprecated

logger = logging.Logger(__name__)

class OpenClinicaIntegrationError(Exception):
    pass

Item = namedtuple('Item', ('study_event_oid', 'form_oid', 'item_group_oid', 'item_oid'))
AdminDataUser = namedtuple('AdminDataUser', ('user_id', 'first_name', 'last_name'))
OpenClinicaUser = namedtuple('OpenClinicaUser', ('user_id', 'first_name', 'last_name', 'username', 'full_name'))

# CDISC OMD XML namespace map
odm_nsmap = {
    'odm': "http://www.cdisc.org/ns/odm/v1.3",
    'OpenClinica': "http://www.openclinica.org/ns/odm_ext_v130/v3.1",
    'OpenClinicaRules': "http://www.openclinica.org/ns/rules/v3.1",
    'xsi': "http://www.w3.org/2001/XMLSchema-instance",
}

@quickcache(['domain'])
def get_question_items(domain):
    """
    Return a map of CommCare form questions to OpenClinica form items
    """

    def get_item_prefix(form_oid, ig_oid):
        """
        OpenClinica item OIDs are prefixed with "I_<prefix>_" where <prefix> is derived from the item's form OID

        (Dropping "I_<prefix>_" will give us the CommCare question name in upper case)
        """
        form_name = form_oid[2:]  # Drop "F_"
        ig_name = ig_oid[3:]  # Drop "IG_"
        prefix = os.path.commonprefix((form_name, ig_name))
        if prefix.endswith('_'):
            prefix = prefix[:-1]
        return prefix

    def read_question_item_map(odm):
        """
        Return a dictionary of {question: (study_event_oid, form_oid, item_group_oid, item_oid)}
        """
        question_item_map = {}  # A dictionary of question: (study_event_oid, form_oid, item_group_oid, item_oid)

        meta_e = odm.xpath('./odm:Study/odm:MetaDataVersion', namespaces=odm_nsmap)[0]

        for se_ref in meta_e.xpath('./odm:Protocol/odm:StudyEventRef', namespaces=odm_nsmap):
            se_oid = se_ref.get('StudyEventOID')
            for form_ref in meta_e.xpath('./odm:StudyEventDef[@OID="{}"]/odm:FormRef'.format(se_oid),
                                         namespaces=odm_nsmap):
                form_oid = form_ref.get('FormOID')
                for ig_ref in meta_e.xpath('./odm:FormDef[@OID="{}"]/odm:ItemGroupRef'.format(form_oid),
                                           namespaces=odm_nsmap):
                    ig_oid = ig_ref.get('ItemGroupOID')
                    prefix = get_item_prefix(form_oid, ig_oid)
                    prefix_len = len(prefix) + 3  # len of "I_<prefix>_"
                    for item_ref in meta_e.xpath('./odm:ItemGroupDef[@OID="{}"]/odm:ItemRef'.format(ig_oid),
                                                 namespaces=odm_nsmap):
                        item_oid = item_ref.get('ItemOID')
                        question = item_oid[prefix_len:].lower()
                        question_item_map[question] = Item(se_oid, form_oid, ig_oid, item_oid)
        return question_item_map

    def read_forms(question_item_map):
        data = defaultdict(dict)
        for domain_, pymodule in settings.DOMAIN_MODULE_MAP.iteritems():
            if pymodule == 'custom.openclinica':
                for app in all_apps_by_domain(domain_):
                    for ccmodule in app.get_modules():
                        for ccform in ccmodule.get_forms():
                            form = data[ccform.xmlns]
                            form['app'] = app.name
                            form['module'] = ccmodule.name['en']
                            form['name'] = ccform.name['en']
                            form['questions'] = {}
                            for question in ccform.get_questions(['en']):
                                name = question['value'].split('/')[-1]
                                form['questions'][name] = question_item_map.get(name)
        return data

    metadata_xml = get_study_metadata(domain)
    map_ = read_question_item_map(metadata_xml)
    question_items = read_forms(map_)
    return question_items

def get_question_item(domain, form_xmlns, question):
    """
    Returns an Item namedtuple given a CommCare form and question name
    """
    question_items = get_question_items(domain)
    try:
        se_oid, form_oid, ig_oid, item_oid = question_items[form_xmlns]['questions'][question]
        return Item(se_oid, form_oid, ig_oid, item_oid)
    except KeyError:
        # Did an old form set the value of a question that no longer exists? Best to check that out.
        logger.error('Unknown CommCare question "{}" found in form "{}"'.format(question, form_xmlns))
        return None
    except TypeError:
        # CommCare question does not match an OpenClinica item. This happens with CommCare-only forms
        return None

@quickcache(['domain'])
def get_study_metadata_string(domain):
    """
    Return the study metadata for the given domain as an XML string
    """
    # For this first OpenClinica integration project, for the sake of simplicity, we are just fetching
    # metadata from custom/openclinica/study_metadata.xml. In future, we can fetch it from the web service
    # (See branch openclinica_ws)
    metadata_filename = os.path.join(settings.BASE_DIR, 'custom', 'openclinica', 'study_metadata.xml')
    with open(metadata_filename) as metadata_file:
        return metadata_file.read()

def get_study_metadata(domain):
    """
    Return the study metadata for the given domain as an ElementTree
    """
    # We can't cache an ElementTree instance. Split this function from get_study_metadata_string() to cache the
    # return value of get_study_metadata_string() when fetching via web service.
    return etree.fromstring(get_study_metadata_string(domain))

def get_study_constant(domain, name):
    """
    Return the study metadata of the given name for the given domain
    """
    xpath_text = lambda xml, xpath: xml.xpath(xpath, namespaces=odm_nsmap)[0].text
    xpath_xml = lambda xml, xpath: etree.tostring(xml.xpath(xpath, namespaces=odm_nsmap)[0])
    func = {
        'study_oid': lambda xml: xml.xpath('./odm:Study', namespaces=odm_nsmap)[0].get('OID'),
        'study_name': lambda xml: xpath_text(xml, './odm:Study/odm:GlobalVariables/odm:StudyName'),
        'study_description': lambda xml: xpath_text(xml, './odm:Study/odm:GlobalVariables/odm:StudyDescription'),
        'protocol_name': lambda xml: xpath_text(xml, './odm:Study/odm:GlobalVariables/odm:ProtocolName'),
        'study_xml': lambda xml: xpath_xml(xml, './odm:Study'),
        'admin_data_xml': lambda xml: xpath_xml(xml, './odm:AdminData'),
    }[name]
    metadata_xml = get_study_metadata(domain)
    return func(metadata_xml)

def get_item_measurement_unit(domain, item):
    """
    Return the measurement unit OID for the given Item, or None
    """
    xml = get_study_metadata(domain)
    mu_ref = xml.xpath(
        './odm:Study/odm:MetaDataVersion/odm:ItemDef[@OID="{}"]/odm:MeasurementUnitRef'.format(item.item_oid),
        namespaces=odm_nsmap)
    return mu_ref[0].get('MeasurementUnitOID') if mu_ref else None

def get_study_event_name(domain, oid):
    xml = get_study_metadata(domain)
    return xml.xpath('./odm:Study/odm:MetaDataVersion/odm:StudyEventDef[@OID="{}"]'.format(oid),
                     namespaces=odm_nsmap)[0].get('Name')

def is_study_event_repeating(domain, oid):
    xml = get_study_metadata(domain)
    return xml.xpath('./odm:Study/odm:MetaDataVersion/odm:StudyEventDef[@OID="{}"]'.format(oid),
                     namespaces=odm_nsmap)[0].get('Repeating') == 'Yes'

def is_item_group_repeating(domain, oid):
    xml = get_study_metadata(domain)
    return xml.xpath('./odm:Study/odm:MetaDataVersion/odm:ItemGroupDef[@OID="{}"]'.format(oid),
                     namespaces=odm_nsmap)[0].get('Repeating') == 'Yes'

def mk_oc_username(cc_username):
    """
    Makes a username that meets OpenClinica requirements from a CommCare username.

    Strips off "@domain.name", replaces non-alphanumerics, and pads with "_" if less than 5 characters

    >>> mk_oc_username('eric.idle@montypython.com')
    'eric_idle'
    >>> mk_oc_username('eric')
    'eric_'
    >>> mk_oc_username('I3#')
    'I3___'

    """
    username = cc_username.split('@')[0]
    username = re.sub(r'[^\w]', '_', username)
    if len(username) < 5:
        username += '_' * (5 - len(username))
    return username

@quickcache(['domain'])
def get_oc_users_by_name(domain):
    # We have to look up OpenClinica users by name because usernames are excluded from study metadata
    oc_users_by_name = {}
    xml = get_study_metadata(domain)
    admin = xml.xpath('./odm:AdminData', namespaces=odm_nsmap)[0]
    for user_e in admin:
        try:
            first_name = user_e.xpath('./odm:FirstName', namespaces=odm_nsmap)[0].text
        except IndexError:
            first_name = None
        try:
            last_name = user_e.xpath('./odm:LastName', namespaces=odm_nsmap)[0].text
        except IndexError:
            last_name = None
        user_id = user_e.get('OID')
        oc_users_by_name[(first_name, last_name)] = AdminDataUser(user_id, first_name, last_name)
    return oc_users_by_name

def get_oc_user(domain, cc_user):
    """
    Returns OpenClinica user details for corresponding CommCare user (CouchUser)
    """
    oc_users_by_name = get_oc_users_by_name(domain)
    oc_user = oc_users_by_name.get((cc_user.first_name, cc_user.last_name))
    return OpenClinicaUser(
        user_id=oc_user.user_id,
        username=mk_oc_username(cc_user.username),
        first_name=oc_user.first_name,
        last_name=oc_user.last_name,
        full_name=' '.join((oc_user.first_name, oc_user.last_name)),
    ) if oc_user else None

def oc_format_date(answer):
    """
    Format CommCare datetime answers for OpenClinica

    >>> from datetime import datetime
    >>> answer = datetime(2015, 8, 19, 19, 8, 15)
    >>> oc_format_date(answer)
    '2015-08-19 19:08:15'

    """
    if isinstance(answer, datetime):
        return answer.isoformat(sep=' ')
    if isinstance(answer, date) or isinstance(answer, time):
        return answer.isoformat()
    return answer

def originals_first(forms):
    """
    Return original (deprecated) forms before edited versions
    """
    def get_previous_versions(form_id):
        form_ = XFormDeprecated.get(form_id)
        if getattr(form_, 'deprecated_form_id', None):
            return get_previous_versions(form_.deprecated_form_id) + [form_]
        else:
            return [form_]

    for form in forms:
        if getattr(form, 'deprecated_form_id', None):
            for previous in get_previous_versions(form.deprecated_form_id):
                yield previous
        yield form
<commit_msg>Use full item OID as question name<commit_after>from __future__ import absolute_import
from collections import defaultdict, namedtuple
from datetime import datetime, date, time
import logging
import re
from lxml import etree
import os
from django.conf import settings
from corehq.apps.app_manager.util import all_apps_by_domain
from corehq.util.quickcache import quickcache
from couchforms.models import XFormDeprecated

logger = logging.Logger(__name__)

class OpenClinicaIntegrationError(Exception):
    pass

Item = namedtuple('Item', ('study_event_oid', 'form_oid', 'item_group_oid', 'item_oid'))
AdminDataUser = namedtuple('AdminDataUser', ('user_id', 'first_name', 'last_name'))
OpenClinicaUser = namedtuple('OpenClinicaUser', ('user_id', 'first_name', 'last_name', 'username', 'full_name'))

# CDISC OMD XML namespace map
odm_nsmap = {
    'odm': "http://www.cdisc.org/ns/odm/v1.3",
    'OpenClinica': "http://www.openclinica.org/ns/odm_ext_v130/v3.1",
    'OpenClinicaRules': "http://www.openclinica.org/ns/rules/v3.1",
    'xsi': "http://www.w3.org/2001/XMLSchema-instance",
}

@quickcache(['domain'])
def get_question_items(domain):
    """
    Return a map of CommCare form questions to OpenClinica form items
    """

    def read_question_item_map(odm):
        """
        Return a dictionary of {question: (study_event_oid, form_oid, item_group_oid, item_oid)}
        """
        question_item_map = {}  # A dictionary of question: (study_event_oid, form_oid, item_group_oid, item_oid)

        meta_e = odm.xpath('./odm:Study/odm:MetaDataVersion', namespaces=odm_nsmap)[0]

        for se_ref in meta_e.xpath('./odm:Protocol/odm:StudyEventRef', namespaces=odm_nsmap):
            se_oid = se_ref.get('StudyEventOID')
            for form_ref in meta_e.xpath('./odm:StudyEventDef[@OID="{}"]/odm:FormRef'.format(se_oid),
                                         namespaces=odm_nsmap):
                form_oid = form_ref.get('FormOID')
                for ig_ref in meta_e.xpath('./odm:FormDef[@OID="{}"]/odm:ItemGroupRef'.format(form_oid),
                                           namespaces=odm_nsmap):
                    ig_oid = ig_ref.get('ItemGroupOID')
                    for item_ref in meta_e.xpath('./odm:ItemGroupDef[@OID="{}"]/odm:ItemRef'.format(ig_oid),
                                                 namespaces=odm_nsmap):
                        item_oid = item_ref.get('ItemOID')
                        question = item_oid.lower()
                        question_item_map[question] = (se_oid, form_oid, ig_oid, item_oid)
        return question_item_map

    def read_forms(question_item_map):
        data = defaultdict(dict)
        for domain_, pymodule in settings.DOMAIN_MODULE_MAP.iteritems():
            if pymodule == 'custom.openclinica':
                for app in all_apps_by_domain(domain_):
                    for ccmodule in app.get_modules():
                        for ccform in ccmodule.get_forms():
                            form = data[ccform.xmlns]
                            form['app'] = app.name
                            form['module'] = ccmodule.name['en']
                            form['name'] = ccform.name['en']
                            form['questions'] = {}
                            for question in ccform.get_questions(['en']):
                                name = question['value'].split('/')[-1]
                                form['questions'][name] = question_item_map.get(name)
        return data

    metadata_xml = get_study_metadata(domain)
    map_ = read_question_item_map(metadata_xml)
    question_items = read_forms(map_)
    return question_items

def get_question_item(domain, form_xmlns, question):
    """
    Returns an Item namedtuple given a CommCare form and question name
    """
    question_items = get_question_items(domain)
    try:
        se_oid, form_oid, ig_oid, item_oid = question_items[form_xmlns]['questions'][question]
        return Item(se_oid, form_oid, ig_oid, item_oid)
    except KeyError:
        # Did an old form set the value of a question that no longer exists? Best to check that out.
        logger.error('Unknown CommCare question "{}" found in form "{}"'.format(question, form_xmlns))
        return None
    except TypeError:
        # CommCare question does not match an OpenClinica item. This happens with CommCare-only forms
        return None

@quickcache(['domain'])
def get_study_metadata_string(domain):
    """
    Return the study metadata for the given domain as an XML string
    """
    # For this first OpenClinica integration project, for the sake of simplicity, we are just fetching
    # metadata from custom/openclinica/study_metadata.xml. In future, we can fetch it from the web service
    # (See branch openclinica_ws)
    metadata_filename = os.path.join(settings.BASE_DIR, 'custom', 'openclinica', 'study_metadata.xml')
    with open(metadata_filename) as metadata_file:
        return metadata_file.read()

def get_study_metadata(domain):
    """
    Return the study metadata for the given domain as an ElementTree
    """
    # We can't cache an ElementTree instance. Split this function from get_study_metadata_string() to cache the
    # return value of get_study_metadata_string() when fetching via web service.
    return etree.fromstring(get_study_metadata_string(domain))

def get_study_constant(domain, name):
    """
    Return the study metadata of the given name for the given domain
    """
    xpath_text = lambda xml, xpath: xml.xpath(xpath, namespaces=odm_nsmap)[0].text
    xpath_xml = lambda xml, xpath: etree.tostring(xml.xpath(xpath, namespaces=odm_nsmap)[0])
    func = {
        'study_oid': lambda xml: xml.xpath('./odm:Study', namespaces=odm_nsmap)[0].get('OID'),
        'study_name': lambda xml: xpath_text(xml, './odm:Study/odm:GlobalVariables/odm:StudyName'),
        'study_description': lambda xml: xpath_text(xml, './odm:Study/odm:GlobalVariables/odm:StudyDescription'),
        'protocol_name': lambda xml: xpath_text(xml, './odm:Study/odm:GlobalVariables/odm:ProtocolName'),
        'study_xml': lambda xml: xpath_xml(xml, './odm:Study'),
        'admin_data_xml': lambda xml: xpath_xml(xml, './odm:AdminData'),
    }[name]
    metadata_xml = get_study_metadata(domain)
    return func(metadata_xml)

def get_item_measurement_unit(domain, item):
    """
    Return the measurement unit OID for the given Item, or None
    """
    xml = get_study_metadata(domain)
    mu_ref = xml.xpath(
        './odm:Study/odm:MetaDataVersion/odm:ItemDef[@OID="{}"]/odm:MeasurementUnitRef'.format(item.item_oid),
        namespaces=odm_nsmap)
    return mu_ref[0].get('MeasurementUnitOID') if mu_ref else None

def get_study_event_name(domain, oid):
    xml = get_study_metadata(domain)
    return xml.xpath('./odm:Study/odm:MetaDataVersion/odm:StudyEventDef[@OID="{}"]'.format(oid),
                     namespaces=odm_nsmap)[0].get('Name')

def is_study_event_repeating(domain, oid):
    xml = get_study_metadata(domain)
    return xml.xpath('./odm:Study/odm:MetaDataVersion/odm:StudyEventDef[@OID="{}"]'.format(oid),
                     namespaces=odm_nsmap)[0].get('Repeating') == 'Yes'

def is_item_group_repeating(domain, oid):
    xml = get_study_metadata(domain)
    return xml.xpath('./odm:Study/odm:MetaDataVersion/odm:ItemGroupDef[@OID="{}"]'.format(oid),
                     namespaces=odm_nsmap)[0].get('Repeating') == 'Yes'

def mk_oc_username(cc_username):
    """
    Makes a username that meets OpenClinica requirements from a CommCare username.

    Strips off "@domain.name", replaces non-alphanumerics, and pads with "_" if less than 5 characters

    >>> mk_oc_username('eric.idle@montypython.com')
    'eric_idle'
    >>> mk_oc_username('eric')
    'eric_'
    >>> mk_oc_username('I3#')
    'I3___'

    """
    username = cc_username.split('@')[0]
    username = re.sub(r'[^\w]', '_', username)
    if len(username) < 5:
        username += '_' * (5 - len(username))
    return username

@quickcache(['domain'])
def get_oc_users_by_name(domain):
    # We have to look up OpenClinica users by name because usernames are excluded from study metadata
    oc_users_by_name = {}
    xml = get_study_metadata(domain)
    admin = xml.xpath('./odm:AdminData', namespaces=odm_nsmap)[0]
    for user_e in admin:
        try:
            first_name = user_e.xpath('./odm:FirstName', namespaces=odm_nsmap)[0].text
        except IndexError:
            first_name = None
        try:
            last_name = user_e.xpath('./odm:LastName', namespaces=odm_nsmap)[0].text
        except IndexError:
            last_name = None
        user_id = user_e.get('OID')
        oc_users_by_name[(first_name, last_name)] = AdminDataUser(user_id, first_name, last_name)
    return oc_users_by_name

def get_oc_user(domain, cc_user):
    """
    Returns OpenClinica user details for corresponding CommCare user (CouchUser)
    """
    oc_users_by_name = get_oc_users_by_name(domain)
    oc_user = oc_users_by_name.get((cc_user.first_name, cc_user.last_name))
    return OpenClinicaUser(
        user_id=oc_user.user_id,
        username=mk_oc_username(cc_user.username),
        first_name=oc_user.first_name,
        last_name=oc_user.last_name,
        full_name=' '.join((oc_user.first_name, oc_user.last_name)),
    ) if oc_user else None

def oc_format_date(answer):
    """
    Format CommCare datetime answers for OpenClinica

    >>> from datetime import datetime
    >>> answer = datetime(2015, 8, 19, 19, 8, 15)
    >>> oc_format_date(answer)
    '2015-08-19 19:08:15'

    """
    if isinstance(answer, datetime):
        return answer.isoformat(sep=' ')
    if isinstance(answer, date) or isinstance(answer, time):
        return answer.isoformat()
    return answer

def originals_first(forms):
    """
    Return original (deprecated) forms before edited versions
    """
    def get_previous_versions(form_id):
        form_ = XFormDeprecated.get(form_id)
        if getattr(form_, 'deprecated_form_id', None):
            return get_previous_versions(form_.deprecated_form_id) + [form_]
        else:
            return [form_]

    for form in forms:
        if getattr(form, 'deprecated_form_id', None):
            for previous in get_previous_versions(form.deprecated_form_id):
                yield previous
        yield form
<|endoftext|>


<commit_before>import datetime

from corehq.apps.accounting.invoicing import (
    CustomerAccountInvoiceFactory,
    DomainInvoiceFactory,
    should_create_invoice,
)
from corehq.apps.accounting.models import (
    BillingAccount,
    DefaultProductPlan,
    SoftwarePlanEdition,
    Subscription,
)
from corehq.apps.accounting.tests import generator
from corehq.apps.accounting.tests.base_tests import BaseAccountingTest
from corehq.apps.accounting.utils import clear_plan_version_cache
from corehq.util.dates import get_previous_month_date_range

class TestDomainInvoiceFactory(BaseAccountingTest):

    @classmethod
    def setUpClass(cls):
        super(TestDomainInvoiceFactory, cls).setUpClass()
        generator.bootstrap_test_software_plan_versions()

    def setUp(self):
        super(TestDomainInvoiceFactory, self).setUp()
        self.invoice_start, self.invoice_end = get_previous_month_date_range()

        self.domain = generator.arbitrary_domain()
        self.account = BillingAccount.get_or_create_account_by_domain(
            domain=self.domain.name, created_by="TEST"
        )[0]
        self.community = DefaultProductPlan.get_default_plan_version()
        generator.arbitrary_commcare_users_for_domain(
            self.domain.name, self.community.user_limit + 1
        )

        self.invoice_factory = DomainInvoiceFactory(
            self.invoice_start, self.invoice_end, self.domain
        )

    def tearDown(self):
        self.domain.delete()
        super(TestDomainInvoiceFactory, self).tearDown()

    @classmethod
    def tearDownClass(cls):
        clear_plan_version_cache()
        super(TestDomainInvoiceFactory, cls).tearDownClass()

    def test_feature_charges(self):
        domain_under_limits = generator.arbitrary_domain()
        self.assertTrue(self.community.feature_charges_exist_for_domain(self.domain))
        self.assertFalse(self.community.feature_charges_exist_for_domain(domain_under_limits))
        domain_under_limits.delete()

    def test_incomplete_starting_coverage(self):
        some_plan = generator.subscribable_plan_version()
        subscription = Subscription.new_domain_subscription(
            self.account, self.domain.name, some_plan,
            date_start=self.invoice_start + datetime.timedelta(days=3)
        )
        subscriptions = self.invoice_factory._get_subscriptions()
        community_ranges = self.invoice_factory._get_community_ranges(subscriptions)
        self.assertEqual(len(community_ranges), 1)
        self.assertEqual(community_ranges[0][0], self.invoice_start)
        self.assertEqual(community_ranges[0][1], subscription.date_start)

    def test_incomplete_ending_coverage(self):
        some_plan = generator.subscribable_plan_version()
        subscription = Subscription.new_domain_subscription(
            self.account, self.domain.name, some_plan,
            date_start=self.invoice_start,
            date_end=self.invoice_end - datetime.timedelta(days=3)
        )
        subscriptions = self.invoice_factory._get_subscriptions()
        community_ranges = self.invoice_factory._get_community_ranges(subscriptions)
        self.assertEqual(len(community_ranges), 1)
        self.assertEqual(community_ranges[0][0], subscription.date_end)
        self.assertEqual(community_ranges[0][1],
                         self.invoice_end + datetime.timedelta(days=1))

    def test_patchy_coverage(self):
        some_plan = generator.subscribable_plan_version()
        middle_date = self.invoice_end - datetime.timedelta(days=15)
        Subscription.new_domain_subscription(
            self.account, self.domain.name, some_plan,
            date_start=self.invoice_start + datetime.timedelta(days=1),
            date_end=middle_date
        )
        next_start = middle_date + datetime.timedelta(days=2)
        next_end = next_start + datetime.timedelta(days=2)
        Subscription.new_domain_subscription(
            self.account, self.domain.name, some_plan,
            date_start=next_start,
            date_end=next_end,
        )
        final_start = next_end + datetime.timedelta(days=2)
        Subscription.new_domain_subscription(
            self.account, self.domain.name, some_plan,
            date_start=final_start,
            date_end=self.invoice_end - datetime.timedelta(days=1),
        )
        subscriptions = self.invoice_factory._get_subscriptions()
        self.assertEqual(len(subscriptions), 3)
        community_ranges = self.invoice_factory._get_community_ranges(subscriptions)
        self.assertEqual(len(community_ranges), 4)

    def test_full_coverage(self):
        some_plan = generator.subscribable_plan_version()
        Subscription.new_domain_subscription(
            self.account, self.domain.name, some_plan,
            date_start=self.invoice_start,
            date_end=self.invoice_end + datetime.timedelta(days=1),
        )
        subscriptions = self.invoice_factory._get_subscriptions()
        community_ranges = self.invoice_factory._get_community_ranges(subscriptions)
        self.assertEqual(len(community_ranges), 0)

    def test_no_coverage(self):
        subscriptions = self.invoice_factory._get_subscriptions()
        self.assertEqual(len(subscriptions), 0)
        community_ranges = self.invoice_factory._get_community_ranges(subscriptions)
        self.assertEqual(community_ranges, [(self.invoice_start, self.invoice_end + datetime.timedelta(days=1))])

class TestInvoicingMethods(BaseAccountingTest):

    def setUp(self):
        super(TestInvoicingMethods, self).setUp()
        self.invoice_start = datetime.date(2018, 5, 1)
        self.invoice_end = datetime.date(2018, 5, 31)

        self.domain = generator.arbitrary_domain()
        self.account = BillingAccount.get_or_create_account_by_domain(
            domain=self.domain, created_by="TEST"
        )[0]
        self.account.is_customer_billing_account = True
        self.account.save()
        self.invoice_factory = CustomerAccountInvoiceFactory(self.invoice_start, self.invoice_end, self.account)
        self.advanced_plan = DefaultProductPlan.get_default_plan_version(edition=SoftwarePlanEdition.ADVANCED)
        self.advanced_plan.plan.is_customer_software_plan = True
        self.pro_plan = DefaultProductPlan.get_default_plan_version(edition=SoftwarePlanEdition.PRO)
        self.pro_plan.plan.is_customer_software_plan = True
        self.subscription = Subscription.new_domain_subscription(
            self.account,
            self.domain.name,
            self.advanced_plan,
            date_start=self.invoice_start,
            date_end=self.invoice_end
        )

    def tearDown(self):
        self.domain.delete()
        super(TestInvoicingMethods, self).tearDown()

    def test_should_not_invoice_trial(self):
        trial_domain = generator.arbitrary_domain()
        subscription = Subscription.new_domain_subscription(
            self.account, trial_domain.name, self.advanced_plan, date_start=self.invoice_start
        )
        subscription.is_trial = True
        self.assertFalse(should_create_invoice(
            subscription=subscription,
            domain=subscription.subscriber.domain,
            invoice_start=self.invoice_start,
            invoice_end=self.invoice_end
        ))
        trial_domain.delete()

    def test_should_not_invoice_without_subscription_charges(self):
        feature_charge_domain = generator.arbitrary_domain()
        subscription = Subscription.new_domain_subscription(
            self.account, feature_charge_domain.name, self.advanced_plan, date_start=self.invoice_start
        )
        subscription.skip_invoicing_if_no_feature_charges = True
        self.assertFalse(should_create_invoice(
            subscription=subscription,
            domain=subscription.subscriber.domain,
            invoice_start=self.invoice_start,
            invoice_end=self.invoice_end
        ))
        feature_charge_domain.delete()

    def test_should_not_invoice_after_end(self):
        invoice_start = datetime.date(2018, 4, 1)
        invoice_end = datetime.date(2018, 4, 30)
        self.assertFalse(should_create_invoice(
            subscription=self.subscription,
            domain=self.subscription.subscriber.domain,
            invoice_start=invoice_start,
            invoice_end=invoice_end
        ))

    def test_should_not_invoice_before_start(self):
        invoice_start = datetime.date(2018, 6, 1)
        invoice_end = datetime.date(2018, 6, 30)
        self.assertFalse(should_create_invoice(
            subscription=self.subscription,
            domain=self.subscription.subscriber.domain,
            invoice_start=invoice_start,
            invoice_end=invoice_end
        ))
<commit_msg>add test for domain invoice factory with paused plans<commit_after>import datetime

from corehq.apps.accounting.invoicing import (
    CustomerAccountInvoiceFactory,
    DomainInvoiceFactory,
    should_create_invoice,
)
from corehq.apps.accounting.models import (
    BillingAccount,
    DefaultProductPlan,
    SoftwarePlanEdition,
    Subscription,
)
from corehq.apps.accounting.tests import generator
from corehq.apps.accounting.tests.base_tests import BaseAccountingTest
from corehq.apps.accounting.utils import clear_plan_version_cache
from corehq.util.dates import get_previous_month_date_range

class TestDomainInvoiceFactory(BaseAccountingTest):

    @classmethod
    def setUpClass(cls):
        super(TestDomainInvoiceFactory, cls).setUpClass()
        generator.bootstrap_test_software_plan_versions()

    def setUp(self):
        super(TestDomainInvoiceFactory, self).setUp()
        self.invoice_start, self.invoice_end = get_previous_month_date_range()

        self.domain = generator.arbitrary_domain()
        self.account = BillingAccount.get_or_create_account_by_domain(
            domain=self.domain.name, created_by="TEST"
        )[0]
        self.community = DefaultProductPlan.get_default_plan_version()
        generator.arbitrary_commcare_users_for_domain(
            self.domain.name, self.community.user_limit + 1
        )

        self.invoice_factory = DomainInvoiceFactory(
            self.invoice_start, self.invoice_end, self.domain
        )

    def tearDown(self):
        self.domain.delete()
        super(TestDomainInvoiceFactory, self).tearDown()

    @classmethod
    def tearDownClass(cls):
        clear_plan_version_cache()
        super(TestDomainInvoiceFactory, cls).tearDownClass()

    def test_feature_charges(self):
        domain_under_limits = generator.arbitrary_domain()
        self.assertTrue(self.community.feature_charges_exist_for_domain(self.domain))
        self.assertFalse(self.community.feature_charges_exist_for_domain(domain_under_limits))
        domain_under_limits.delete()

    def test_incomplete_starting_coverage(self):
        some_plan = generator.subscribable_plan_version()
        subscription = Subscription.new_domain_subscription(
            self.account, self.domain.name, some_plan,
            date_start=self.invoice_start + datetime.timedelta(days=3)
        )
        subscriptions = self.invoice_factory._get_subscriptions()
        community_ranges = self.invoice_factory._get_community_ranges(subscriptions)
        self.assertEqual(len(community_ranges), 1)
        self.assertEqual(community_ranges[0][0], self.invoice_start)
        self.assertEqual(community_ranges[0][1], subscription.date_start)

    def test_incomplete_ending_coverage(self):
        some_plan = generator.subscribable_plan_version()
        subscription = Subscription.new_domain_subscription(
            self.account, self.domain.name, some_plan,
            date_start=self.invoice_start,
            date_end=self.invoice_end - datetime.timedelta(days=3)
        )
        subscriptions = self.invoice_factory._get_subscriptions()
        community_ranges = self.invoice_factory._get_community_ranges(subscriptions)
        self.assertEqual(len(community_ranges), 1)
        self.assertEqual(community_ranges[0][0], subscription.date_end)
        self.assertEqual(community_ranges[0][1],
                         self.invoice_end + datetime.timedelta(days=1))

    def test_patchy_coverage(self):
        some_plan = generator.subscribable_plan_version()
        middle_date = self.invoice_end - datetime.timedelta(days=15)
        Subscription.new_domain_subscription(
            self.account, self.domain.name, some_plan,
            date_start=self.invoice_start + datetime.timedelta(days=1),
            date_end=middle_date
        )
        next_start = middle_date + datetime.timedelta(days=2)
        next_end = next_start + datetime.timedelta(days=2)
        Subscription.new_domain_subscription(
            self.account, self.domain.name, some_plan,
            date_start=next_start,
            date_end=next_end,
        )
        final_start = next_end + datetime.timedelta(days=2)
        Subscription.new_domain_subscription(
            self.account, self.domain.name, some_plan,
            date_start=final_start,
            date_end=self.invoice_end - datetime.timedelta(days=1),
        )
        subscriptions = self.invoice_factory._get_subscriptions()
        self.assertEqual(len(subscriptions), 3)
        community_ranges = self.invoice_factory._get_community_ranges(subscriptions)
        self.assertEqual(len(community_ranges), 4)

    def test_full_coverage(self):
        some_plan = generator.subscribable_plan_version()
        Subscription.new_domain_subscription(
            self.account, self.domain.name, some_plan,
            date_start=self.invoice_start,
            date_end=self.invoice_end + datetime.timedelta(days=1),
        )
        subscriptions = self.invoice_factory._get_subscriptions()
        community_ranges = self.invoice_factory._get_community_ranges(subscriptions)
        self.assertEqual(len(community_ranges), 0)

    def test_no_coverage(self):
        subscriptions = self.invoice_factory._get_subscriptions()
        self.assertEqual(len(subscriptions), 0)
        community_ranges = self.invoice_factory._get_community_ranges(subscriptions)
        self.assertEqual(community_ranges, [(self.invoice_start, self.invoice_end + datetime.timedelta(days=1))])

    def test_paused_plan_generates_no_invoice(self):
        """
        Ensure that paused plans do not generate invoices.
        """
        paused_plan = generator.subscribable_plan_version(
            edition=SoftwarePlanEdition.PAUSED
        )
        Subscription.new_domain_subscription(
            self.account, self.domain.name, paused_plan,
            date_start=self.invoice_start,
            date_end=self.invoice_end + datetime.timedelta(days=1),

        )
        self.assertListEqual(self.invoice_factory._get_subscriptions(), [])

class TestInvoicingMethods(BaseAccountingTest):

    def setUp(self):
        super(TestInvoicingMethods, self).setUp()
        self.invoice_start = datetime.date(2018, 5, 1)
        self.invoice_end = datetime.date(2018, 5, 31)

        self.domain = generator.arbitrary_domain()
        self.account = BillingAccount.get_or_create_account_by_domain(
            domain=self.domain, created_by="TEST"
        )[0]
        self.account.is_customer_billing_account = True
        self.account.save()
        self.invoice_factory = CustomerAccountInvoiceFactory(self.invoice_start, self.invoice_end, self.account)
        self.advanced_plan = DefaultProductPlan.get_default_plan_version(edition=SoftwarePlanEdition.ADVANCED)
        self.advanced_plan.plan.is_customer_software_plan = True
        self.pro_plan = DefaultProductPlan.get_default_plan_version(edition=SoftwarePlanEdition.PRO)
        self.pro_plan.plan.is_customer_software_plan = True
        self.subscription = Subscription.new_domain_subscription(
            self.account,
            self.domain.name,
            self.advanced_plan,
            date_start=self.invoice_start,
            date_end=self.invoice_end
        )

    def tearDown(self):
        self.domain.delete()
        super(TestInvoicingMethods, self).tearDown()

    def test_should_not_invoice_trial(self):
        trial_domain = generator.arbitrary_domain()
        subscription = Subscription.new_domain_subscription(
            self.account, trial_domain.name, self.advanced_plan, date_start=self.invoice_start
        )
        subscription.is_trial = True
        self.assertFalse(should_create_invoice(
            subscription=subscription,
            domain=subscription.subscriber.domain,
            invoice_start=self.invoice_start,
            invoice_end=self.invoice_end
        ))
        trial_domain.delete()

    def test_should_not_invoice_without_subscription_charges(self):
        feature_charge_domain = generator.arbitrary_domain()
        subscription = Subscription.new_domain_subscription(
            self.account, feature_charge_domain.name, self.advanced_plan, date_start=self.invoice_start
        )
        subscription.skip_invoicing_if_no_feature_charges = True
        self.assertFalse(should_create_invoice(
            subscription=subscription,
            domain=subscription.subscriber.domain,
            invoice_start=self.invoice_start,
            invoice_end=self.invoice_end
        ))
        feature_charge_domain.delete()

    def test_should_not_invoice_after_end(self):
        invoice_start = datetime.date(2018, 4, 1)
        invoice_end = datetime.date(2018, 4, 30)
        self.assertFalse(should_create_invoice(
            subscription=self.subscription,
            domain=self.subscription.subscriber.domain,
            invoice_start=invoice_start,
            invoice_end=invoice_end
        ))

    def test_should_not_invoice_before_start(self):
        invoice_start = datetime.date(2018, 6, 1)
        invoice_end = datetime.date(2018, 6, 30)
        self.assertFalse(should_create_invoice(
            subscription=self.subscription,
            domain=self.subscription.subscriber.domain,
            invoice_start=invoice_start,
            invoice_end=invoice_end
        ))
<|endoftext|>


<commit_before>from conjureup import events, utils
from conjureup.app_config import app

from . import common

class CredentialsController(common.BaseCredentialsController):
    def render(self):
        if app.provider.cloud_type == 'localhost':
            # no credentials required for localhost
            self.finish()
        elif not self.credentials:
            utils.warning("You attempted to do an install against a cloud "
                          "that requires credentials that could not be "
                          "found.  If you wish to supply those "
                          "credentials please run "
                          "`juju add-credential "
                          "{}`.".format(app.provider.cloud))
            events.Shutdown.set(1)
        elif not app.provider.credential:
            utils.warning("You attempted to install against a cloud with "
                          "multiple credentials and no default credentials "
                          "set.  Please set a default credential with:\n"
                          "\n"
                          "    juju set-default-credential {} <credential>")
            events.Shutdown.set(1)
        else:
            self.finish()

_controller_class = CredentialsController
<commit_msg>Correct cloud type checker in TUI for localhost<commit_after>from conjureup import events, utils
from conjureup.app_config import app

from . import common

class CredentialsController(common.BaseCredentialsController):
    def render(self):
        if app.provider.cloud_type == 'lxd':
            # no credentials required for localhost
            self.finish()
        elif not self.credentials:
            utils.warning("You attempted to do an install against a cloud "
                          "that requires credentials that could not be "
                          "found.  If you wish to supply those "
                          "credentials please run "
                          "`juju add-credential "
                          "{}`.".format(app.provider.cloud))
            events.Shutdown.set(1)
        elif not app.provider.credential:
            utils.warning("You attempted to install against a cloud with "
                          "multiple credentials and no default credentials "
                          "set.  Please set a default credential with:\n"
                          "\n"
                          "    juju set-default-credential {} <credential>")
            events.Shutdown.set(1)
        else:
            self.finish()

_controller_class = CredentialsController
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import re
import requests
import time
import datetime
import logging
import logging.handlers
import subprocess

from subprocess import CalledProcessError
from requests import RequestException
from settings import DeviceToken

import settings
import settings_vendor as config

W1_DEVICES = '/sys/bus/w1/devices/'
W1_SENSOR_PATTERN = re.compile('(10|22|28)-.+', re.IGNORECASE)

#CPU_USAGE_CMD = "top -n2 -d.1 | awk -F ',' '/Cpu\(s\):/ {print $1}'"
CPU_TEMPERATURE_CMD = "vcgencmd measure_temp"

ANSI_ESCAPE = re.compile(r'\x1b[^m]*m')

LOG_FILE_PATH = os.path.join('/', 'var', 'log', 'cloud4rpid.log')

REQUEST_TIMEOUT_SECONDS = 3 * 60 + 0.05

def create_logger():
    logger = logging.getLogger(__name__)
    logger.setLevel(logging.INFO)
    config_logging_to_console(logger)
    return logger

def config_logging_to_console(logger):
    console = logging.StreamHandler()
    console.setFormatter(logging.Formatter('%(message)s'))
    logger.addHandler(console)

def config_logging_to_file(logger):
    log_file = logging.handlers.RotatingFileHandler(LOG_FILE_PATH, maxBytes=1024 * 1024)
    log_file.setFormatter(logging.Formatter('%(asctime)s: %(message)s'))
    logger.addHandler(log_file)

log = create_logger()

def get_system_parameters():
    # cpu_usage = get_cpu_usage()
    cpu_temperature = get_cpu_temperature()
    return {
        # 'cpuUsage': cpu_usage,
        'cpuTemperature': cpu_temperature
    }

# def get_cpu_usage():
#     cpu_usage_str = subprocess.check_output(CPU_USAGE_CMD, shell=True).splitlines()[-1]
#     stripped = strip_escape_codes(cpu_usage_str)
#     return extract_usage(stripped)

def strip_escape_codes(s):
    return ANSI_ESCAPE.sub('', s)

# def extract_usage(s):
#     return float(s.lstrip('%Cpu(s): ').rstrip(' us'))

def get_cpu_temperature():
    cpu_temperature_str = subprocess.check_output(CPU_TEMPERATURE_CMD, shell=True) \
        .lstrip("temp=").rstrip("'C\n")

    cpu_temperature = float(cpu_temperature_str)
    return cpu_temperature

class MutableDatetime(datetime.datetime):
    @classmethod
    def utcnow(cls):
        return super(MutableDatetime, cls).utcnow()

datetime.datetime = MutableDatetime

def sensor_full_path(sensor):
    return os.path.join(W1_DEVICES, sensor, 'w1_slave')

def find_sensors():
    return [x for x in os.listdir(W1_DEVICES)
            if W1_SENSOR_PATTERN.match(x) and os.path.isfile(sensor_full_path(x))]

def read_sensor(address):
    readings = read_whole_file(sensor_full_path(address))
    temp_token = 't='
    temp_index = readings.find(temp_token)
    if temp_index < 0:
        return 0.0
    temp = readings[temp_index + len(temp_token):]
    return address, float(temp) / 1000

def read_whole_file(path):
    with open(path, 'r') as f:
        return f.read()

def read_sensors():
    return [read_sensor(x) for x in find_sensors()]

def request_headers(token):
    return {'api_key': token}

def device_request_url(token):
    return '{0}/devices/{1}/'.format(config.baseApiUrl, token)

def stream_request_url(token):
    return '{0}/devices/{1}/streams/'.format(config.baseApiUrl, token)

def system_parameters_request_url(token):
    return '{0}/devices/{1}/params/'.format(config.baseApiUrl, token)

def get_device(token):
    res = requests.get(device_request_url(token),
                       headers=request_headers(token),
                       timeout=REQUEST_TIMEOUT_SECONDS)
    check_response(res)
    return ServerDevice(res.json())

def put_device(token, device):
    log.info('Sending device configuration...')
    deviceJSON = device.dump()
    log.info(deviceJSON)
    res = requests.put(device_request_url(token),
                       headers=request_headers(token),
                       json=deviceJSON,
                       timeout=REQUEST_TIMEOUT_SECONDS)
    check_response(res)
    if res.status_code != 200:
        log.error("Can't register sensor. Status: {0}".format(res.status_code))

    return ServerDevice(res.json())

def post_stream(token, stream):
    log.info('sending {0}'.format(stream))

    res = requests.post(stream_request_url(token),
                        headers=request_headers(token),
                        json=stream,
                        timeout=REQUEST_TIMEOUT_SECONDS)
    check_response(res)
    return res.json()

def post_system_parameters(token, params):
    log.info('sending {0}'.format(params))

    res = requests.post(system_parameters_request_url(token),
                        headers=request_headers(token),
                        json=params,
                        timeout=REQUEST_TIMEOUT_SECONDS)
    check_response(res)
    return res.json()

def check_response(res):
    log.info(res.status_code)
    if res.status_code == 401:
        raise AuthenticationError

def verify_token(token):
    r = re.compile('[0-9a-f]{24}')
    return len(token) == 24 and r.match(token)

class AuthenticationError(Exception):
    pass

class InvalidTokenError(Exception):
    pass

class NoSensorsError(Exception):
    pass

class ServerDevice(object):
    def __init__(self, device_json):
        self.json = device_json
        self.addresses = None
        self.sensor_index = None
        self.__build_sensor_index()
        self.__extract_addresses()

    def sensor_addrs(self):
        return self.addresses

    def add_sensors(self, sensors):
        self.json['sensors'] += [{'name': x, 'address': x} for x in sensors]
        self.__extract_addresses()

    def whats_new(self, sensors):
        existing = self.sensor_addrs()
        return set(sensors) - set(existing)

    def dump(self):
        return self.json

    def map_sensors(self, readings):
        index = self.sensor_index
        return {index[address]: reading for address, reading in readings if address in index}

    def set_type(self, new_type):
        self.json['type'] = new_type

    def __extract_addresses(self):
        self.addresses = self.sensor_index.keys()

    def __build_sensor_index(self):
        self.sensor_index = {sensor['address']: sensor['_id'] for sensor in self.json['sensors']}

class RpiDaemon(object):
    def __init__(self, token):
        self.sensors = None
        self.me = None
        if not verify_token(token):
            raise InvalidTokenError
        self.token = token

    def run(self):
        self.prepare_sensors()
        self.ensure_there_are_sensors()
        self.poll()

    def prepare_sensors(self):
        self.know_thyself()
        self.find_sensors()
        self.send_device_config()

    def know_thyself(self):
        log.info('Getting device configuration...')
        self.me = get_device(self.token)

    def find_sensors(self):
        self.sensors = find_sensors()

    def send_device_config(self):
        self.register_new_sensors()
        self.me.set_type('Raspberry PI')
        self.me = put_device(self.token, self.me)

    def register_new_sensors(self):
        new_sensors = self.me.whats_new(self.sensors)
        if len(new_sensors) > 0:
            log.info('New sensors found: {0}'.format(list(new_sensors)))
            self.me.add_sensors(sorted(new_sensors))

    def ensure_there_are_sensors(self):
        if len(self.sensors) == 0:
            raise NoSensorsError

    def poll(self):
        while True:
            self.tick()
            time.sleep(settings.scanIntervalSeconds)

    def tick(self):
        self.send_stream()
        self.send_system_parameters()

    def send_stream(self):
        stream = self.create_stream()
        try:
            post_stream(self.token, stream)
        except RequestException:
            log.error('Failed. Skipping...')

    def create_stream(self):
        ts = datetime.datetime.utcnow().isoformat()
        readings = read_sensors()
        payload = self.me.map_sensors(readings)
        return {
            'ts': ts,
            'payload': payload
        }

    def send_system_parameters(self):
        try:
            params = get_system_parameters()
            post_system_parameters(self.token, params)
        except (CalledProcessError, RequestException):
            log.error('Failed. Skipping...')

def modprobe(module):
    cmd = 'modprobe {0}'.format(module)
    ret = os.system(cmd)
    if ret != 0:
        raise CalledProcessError(ret, cmd)

def safeRunDaemon():
    maxTryCount = 5
    waitSecs = 10
    n = 0
    while n < maxTryCount:
        try:
            daemon.run()
            break
        except requests.ConnectionError as ex:
            log.exception('Daemon running ERROR: {0}'.format(ex.message))
            log.exception('Waiting for {5} sec...').format(waitSecs)
            time.sleep(waitSecs)
            n += 1
            waitSecs *= 2

if __name__ == "__main__":
    try:
        modprobe('w1-gpio')
        modprobe('w1-therm')

        config_logging_to_file(log)

        log.info('Starting...')

        daemon = RpiDaemon(DeviceToken)
        safeRunDaemon()

    except RequestException as e:
        log.exception('Connection failed. Please try again later. Error: {0}'.format(e.message))
        exit(1)
    except CalledProcessError:
        log.exception('Try "sudo python cloud4rpi.py"')
        exit(1)
    except InvalidTokenError:
        log.exception('Device Access Token {0} is incorrect. Please verify it.'.format(DeviceToken))
        exit(1)
    except AuthenticationError:
        log.exception('Authentication failed. Check your device token.')
        exit(1)
    except NoSensorsError:
        log.exception('No sensors found... Exiting')
        exit(1)
    except Exception as e:
        log.exception('Unexpected error: {0}'.format(e.message))
        exit(1)
    except KeyboardInterrupt:
        log.info('Interrupted')
        exit(1)
<commit_msg>fix format bug<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import re
import requests
import time
import datetime
import logging
import logging.handlers
import subprocess

from subprocess import CalledProcessError
from requests import RequestException
from settings import DeviceToken

import settings
import settings_vendor as config

W1_DEVICES = '/sys/bus/w1/devices/'
W1_SENSOR_PATTERN = re.compile('(10|22|28)-.+', re.IGNORECASE)

#CPU_USAGE_CMD = "top -n2 -d.1 | awk -F ',' '/Cpu\(s\):/ {print $1}'"
CPU_TEMPERATURE_CMD = "vcgencmd measure_temp"

ANSI_ESCAPE = re.compile(r'\x1b[^m]*m')

LOG_FILE_PATH = os.path.join('/', 'var', 'log', 'cloud4rpid.log')

REQUEST_TIMEOUT_SECONDS = 3 * 60 + 0.05

def create_logger():
    logger = logging.getLogger(__name__)
    logger.setLevel(logging.INFO)
    config_logging_to_console(logger)
    return logger

def config_logging_to_console(logger):
    console = logging.StreamHandler()
    console.setFormatter(logging.Formatter('%(message)s'))
    logger.addHandler(console)

def config_logging_to_file(logger):
    log_file = logging.handlers.RotatingFileHandler(LOG_FILE_PATH, maxBytes=1024 * 1024)
    log_file.setFormatter(logging.Formatter('%(asctime)s: %(message)s'))
    logger.addHandler(log_file)

log = create_logger()

def get_system_parameters():
    # cpu_usage = get_cpu_usage()
    cpu_temperature = get_cpu_temperature()
    return {
        # 'cpuUsage': cpu_usage,
        'cpuTemperature': cpu_temperature
    }

# def get_cpu_usage():
#     cpu_usage_str = subprocess.check_output(CPU_USAGE_CMD, shell=True).splitlines()[-1]
#     stripped = strip_escape_codes(cpu_usage_str)
#     return extract_usage(stripped)

def strip_escape_codes(s):
    return ANSI_ESCAPE.sub('', s)

# def extract_usage(s):
#     return float(s.lstrip('%Cpu(s): ').rstrip(' us'))

def get_cpu_temperature():
    cpu_temperature_str = subprocess.check_output(CPU_TEMPERATURE_CMD, shell=True) \
        .lstrip("temp=").rstrip("'C\n")

    cpu_temperature = float(cpu_temperature_str)
    return cpu_temperature

class MutableDatetime(datetime.datetime):
    @classmethod
    def utcnow(cls):
        return super(MutableDatetime, cls).utcnow()

datetime.datetime = MutableDatetime

def sensor_full_path(sensor):
    return os.path.join(W1_DEVICES, sensor, 'w1_slave')

def find_sensors():
    return [x for x in os.listdir(W1_DEVICES)
            if W1_SENSOR_PATTERN.match(x) and os.path.isfile(sensor_full_path(x))]

def read_sensor(address):
    readings = read_whole_file(sensor_full_path(address))
    temp_token = 't='
    temp_index = readings.find(temp_token)
    if temp_index < 0:
        return 0.0
    temp = readings[temp_index + len(temp_token):]
    return address, float(temp) / 1000

def read_whole_file(path):
    with open(path, 'r') as f:
        return f.read()

def read_sensors():
    return [read_sensor(x) for x in find_sensors()]

def request_headers(token):
    return {'api_key': token}

def device_request_url(token):
    return '{0}/devices/{1}/'.format(config.baseApiUrl, token)

def stream_request_url(token):
    return '{0}/devices/{1}/streams/'.format(config.baseApiUrl, token)

def system_parameters_request_url(token):
    return '{0}/devices/{1}/params/'.format(config.baseApiUrl, token)

def get_device(token):
    res = requests.get(device_request_url(token),
                       headers=request_headers(token),
                       timeout=REQUEST_TIMEOUT_SECONDS)
    check_response(res)
    return ServerDevice(res.json())

def put_device(token, device):
    log.info('Sending device configuration...')
    deviceJSON = device.dump()
    log.info(deviceJSON)
    res = requests.put(device_request_url(token),
                       headers=request_headers(token),
                       json=deviceJSON,
                       timeout=REQUEST_TIMEOUT_SECONDS)
    check_response(res)
    if res.status_code != 200:
        log.error("Can't register sensor. Status: {0}".format(res.status_code))

    return ServerDevice(res.json())

def post_stream(token, stream):
    log.info('sending {0}'.format(stream))

    res = requests.post(stream_request_url(token),
                        headers=request_headers(token),
                        json=stream,
                        timeout=REQUEST_TIMEOUT_SECONDS)
    check_response(res)
    return res.json()

def post_system_parameters(token, params):
    log.info('sending {0}'.format(params))

    res = requests.post(system_parameters_request_url(token),
                        headers=request_headers(token),
                        json=params,
                        timeout=REQUEST_TIMEOUT_SECONDS)
    check_response(res)
    return res.json()

def check_response(res):
    log.info(res.status_code)
    if res.status_code == 401:
        raise AuthenticationError

def verify_token(token):
    r = re.compile('[0-9a-f]{24}')
    return len(token) == 24 and r.match(token)

class AuthenticationError(Exception):
    pass

class InvalidTokenError(Exception):
    pass

class NoSensorsError(Exception):
    pass

class ServerDevice(object):
    def __init__(self, device_json):
        self.json = device_json
        self.addresses = None
        self.sensor_index = None
        self.__build_sensor_index()
        self.__extract_addresses()

    def sensor_addrs(self):
        return self.addresses

    def add_sensors(self, sensors):
        self.json['sensors'] += [{'name': x, 'address': x} for x in sensors]
        self.__extract_addresses()

    def whats_new(self, sensors):
        existing = self.sensor_addrs()
        return set(sensors) - set(existing)

    def dump(self):
        return self.json

    def map_sensors(self, readings):
        index = self.sensor_index
        return {index[address]: reading for address, reading in readings if address in index}

    def set_type(self, new_type):
        self.json['type'] = new_type

    def __extract_addresses(self):
        self.addresses = self.sensor_index.keys()

    def __build_sensor_index(self):
        self.sensor_index = {sensor['address']: sensor['_id'] for sensor in self.json['sensors']}

class RpiDaemon(object):
    def __init__(self, token):
        self.sensors = None
        self.me = None
        if not verify_token(token):
            raise InvalidTokenError
        self.token = token

    def run(self):
        self.prepare_sensors()
        self.ensure_there_are_sensors()
        self.poll()

    def prepare_sensors(self):
        self.know_thyself()
        self.find_sensors()
        self.send_device_config()

    def know_thyself(self):
        log.info('Getting device configuration...')
        self.me = get_device(self.token)

    def find_sensors(self):
        self.sensors = find_sensors()

    def send_device_config(self):
        self.register_new_sensors()
        self.me.set_type('Raspberry PI')
        self.me = put_device(self.token, self.me)

    def register_new_sensors(self):
        new_sensors = self.me.whats_new(self.sensors)
        if len(new_sensors) > 0:
            log.info('New sensors found: {0}'.format(list(new_sensors)))
            self.me.add_sensors(sorted(new_sensors))

    def ensure_there_are_sensors(self):
        if len(self.sensors) == 0:
            raise NoSensorsError

    def poll(self):
        while True:
            self.tick()
            time.sleep(settings.scanIntervalSeconds)

    def tick(self):
        self.send_stream()
        self.send_system_parameters()

    def send_stream(self):
        stream = self.create_stream()
        try:
            post_stream(self.token, stream)
        except RequestException:
            log.error('Failed. Skipping...')

    def create_stream(self):
        ts = datetime.datetime.utcnow().isoformat()
        readings = read_sensors()
        payload = self.me.map_sensors(readings)
        return {
            'ts': ts,
            'payload': payload
        }

    def send_system_parameters(self):
        try:
            params = get_system_parameters()
            post_system_parameters(self.token, params)
        except (CalledProcessError, RequestException):
            log.error('Failed. Skipping...')

def modprobe(module):
    cmd = 'modprobe {0}'.format(module)
    ret = os.system(cmd)
    if ret != 0:
        raise CalledProcessError(ret, cmd)

def safeRunDaemon():
    maxTryCount = 5
    waitSecs = 10
    n = 0
    while n < maxTryCount:
        try:
            daemon.run()
            break
        except requests.ConnectionError as ex:
            log.exception('Daemon running ERROR: {0}'.format(ex.message))
            log.exception('Waiting for {0} sec...').format(waitSecs)
            time.sleep(waitSecs)
            n += 1
            waitSecs *= 2

if __name__ == "__main__":
    try:
        modprobe('w1-gpio')
        modprobe('w1-therm')

        config_logging_to_file(log)

        log.info('Starting...')

        daemon = RpiDaemon(DeviceToken)
        safeRunDaemon()

    except RequestException as e:
        log.exception('Connection failed. Please try again later. Error: {0}'.format(e.message))
        exit(1)
    except CalledProcessError:
        log.exception('Try "sudo python cloud4rpi.py"')
        exit(1)
    except InvalidTokenError:
        log.exception('Device Access Token {0} is incorrect. Please verify it.'.format(DeviceToken))
        exit(1)
    except AuthenticationError:
        log.exception('Authentication failed. Check your device token.')
        exit(1)
    except NoSensorsError:
        log.exception('No sensors found... Exiting')
        exit(1)
    except Exception as e:
        log.exception('Unexpected error: {0}'.format(e.message))
        exit(1)
    except KeyboardInterrupt:
        log.info('Interrupted')
        exit(1)
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import unicode_literals

import uuid

from datetime import datetime

from django.test import TestCase
from sqlalchemy import Date, Integer, SmallInteger, UnicodeText

from casexml.apps.case.mock import CaseBlock
from casexml.apps.case.tests.util import delete_all_cases, delete_all_xforms
from casexml.apps.case.util import post_case_blocks
from corehq.apps.aggregate_ucrs.importer import import_aggregation_models_from_spec
from corehq.apps.aggregate_ucrs.ingestion import populate_aggregate_table_data, get_aggregation_start_period, \
    get_aggregation_end_period
from corehq.apps.aggregate_ucrs.models import AggregateTableDefinition
from corehq.apps.aggregate_ucrs.sql.adapter import AggregateIndicatorSqlAdapter
from corehq.apps.aggregate_ucrs.tests.base import AggregationBaseTestMixin
from corehq.apps.app_manager.tests.app_factory import AppFactory
from corehq.apps.app_manager.tests.util import delete_all_apps
from corehq.apps.app_manager.xform_builder import XFormBuilder
from corehq.apps.receiverwrapper.util import submit_form_locally
from corehq.apps.userreports.app_manager.helpers import get_form_data_source, get_case_data_source
from corehq.apps.userreports.tasks import _iteratively_build_table
from corehq.apps.userreports.util import get_indicator_adapter
from corehq.form_processor.utils.xform import FormSubmissionBuilder, TestFormMetadata

class UCRAggregationTest(TestCase, AggregationBaseTestMixin):
    domain = 'agg'
    case_type = 'agg-cases'
    case_name = 'Rowena Luk'
    case_date_opened = datetime(2017, 12, 19)
    case_properties = (
        ('first_name', 'First Name', 'string', 'Rowena'),
        ('last_name', 'Last Name', 'string', 'Luk'),
        ('children', 'Children', 'int', '1'),
        ('lmp', 'Pregnancy Start Date', 'date', '2018-01-21'),
        ('delivery_date', 'Pregnancy End Date', 'date', '2018-10-27'),
    )
    fu_visit_dates = (
        datetime(2018, 3, 16),
        datetime(2018, 4, 11),
        datetime(2018, 4, 16),
    )
    parent_case_type = 'parent'
    parent_name = 'Mama'

    @classmethod
    def setUpClass(cls):
        super(UCRAggregationTest, cls).setUpClass()
        # cleanup any previous data
        cls._cleanup_data()

        # setup app
        factory = AppFactory(domain=cls.domain)
        # parent case module, incl opening child cases of main type
        m_parent, f_parent = factory.new_basic_module('Parent Module', cls.parent_case_type)
        factory.form_opens_case(f_parent, case_type=cls.parent_case_type)
        factory.form_opens_case(f_parent, case_type=cls.case_type, is_subcase=True)

        # main module
        m0, f0 = factory.new_basic_module('A Module', cls.case_type)
        f1 = factory.new_form(m0)
        f1.source = cls._get_xform()
        factory.form_requires_case(f1, case_type=cls.case_type, update={
            cp[0]: '/data/{}'.format(cp[0]) for cp in cls.case_properties
        })
        cls.followup_form = f1

        cls.app = factory.app
        cls.app.save()

        # create form and case ucrs
        cls.form_data_source = get_form_data_source(cls.app, cls.followup_form)
        cls.case_data_source = get_case_data_source(cls.app, cls.case_type)
        cls.parent_case_data_source = get_case_data_source(cls.app, cls.parent_case_type)

        # create some data - first just create the case
        cls.parent_case_id = cls._create_parent_case(cls.parent_name)
        cls.case_id = cls._create_case(cls.parent_case_id)
        for fu_date in cls.fu_visit_dates:
            cls._submit_followup_form(cls.case_id, received_on=fu_date)

        # populate the UCRs with the data we just created
        cls.form_adapter = get_indicator_adapter(cls.form_data_source)
        cls.case_adapter = get_indicator_adapter(cls.case_data_source)
        cls.parent_case_adapter = get_indicator_adapter(cls.parent_case_data_source)

        cls.form_adapter.rebuild_table()
        cls.case_adapter.rebuild_table()
        cls.parent_case_adapter.rebuild_table()

        _iteratively_build_table(cls.form_data_source)
        _iteratively_build_table(cls.case_data_source)
        _iteratively_build_table(cls.parent_case_data_source)

        # setup AggregateTableDefinition
        cls.monthly_aggregate_table_definition = cls._get_monthly_aggregate_table_definition()
        cls.basic_aggregate_table_definition = cls._get_basic_aggregate_table_definition()

    @classmethod
    def tearDownClass(cls):
        super(UCRAggregationTest, cls).tearDownClass()
        cls._cleanup_data()

    @classmethod
    def _cleanup_data(cls):
        delete_all_cases()
        delete_all_xforms()
        delete_all_apps()
        AggregateTableDefinition.objects.all().delete()

    def setUp(self):
        # confirm that our setupClass function properly did its job
        self.assertEqual(3, self.form_adapter.get_query_object().count())
        self.assertEqual(1, self.case_adapter.get_query_object().count())
        self.assertEqual(1, self.parent_case_adapter.get_query_object().count())

    @classmethod
    def _get_xform(cls):
        xform = XFormBuilder()
        for prop_name, prop_text, datatype, _ in cls.case_properties:
            xform.new_question(prop_name, prop_text, data_type=datatype)
        return xform.tostring()

    @classmethod
    def _get_case_property_values(cls):
        return {
            cp[0]: cp[-1] for cp in cls.case_properties
        }

    @classmethod
    def _create_case(cls, parent_id):
        case_id = uuid.uuid4().hex
        caseblock = CaseBlock(
            case_id=case_id,
            case_type=cls.case_type,
            date_opened=cls.case_date_opened,
            case_name=cls.case_name,
            update=cls._get_case_property_values(),
            index={
                'parent': (cls.case_type, parent_id)
            }
        )
        post_case_blocks([caseblock.as_xml()], domain=cls.domain)
        return case_id

    @classmethod
    def _create_parent_case(cls, case_name):
        parent_id = uuid.uuid4().hex
        post_case_blocks(
            [
                CaseBlock(
                    create=True,
                    case_id=parent_id,
                    case_name=case_name,
                    case_type=cls.parent_case_type,
                ).as_xml()
            ], domain=cls.domain
        )
        return parent_id

    @classmethod
    def _submit_followup_form(cls, case_id, received_on):
        form_id = uuid.uuid4().hex
        form_meta = TestFormMetadata(
            domain=cls.domain,
            xmlns=cls.followup_form.xmlns,
            app_id=cls.app._id,
            received_on=received_on,
        )
        properties = cls._get_case_property_values()
        caseblock = CaseBlock(
            case_id=case_id,
            update=properties,
        )
        form_builder = FormSubmissionBuilder(
            form_id=form_id,
            metadata=form_meta,
            case_blocks=[caseblock],
            form_properties=properties,
        )
        submit_form_locally(form_builder.as_xml_string(), cls.domain, received_on=received_on)
        return form_id

    @classmethod
    def _get_monthly_aggregate_table_definition(cls):
        spec = cls.get_monthly_config_spec()
        spec.primary_table.data_source_id = cls.case_data_source._id
        spec.secondary_tables[0].data_source_id = cls.form_data_source._id
        return import_aggregation_models_from_spec(spec)

    @classmethod
    def _get_basic_aggregate_table_definition(cls):
        spec = cls.get_basic_config_spec()
        spec.primary_table.data_source_id = cls.case_data_source._id
        spec.secondary_tables[0].data_source_id = cls.form_data_source._id
        spec.secondary_tables[1].data_source_id = cls.parent_case_data_source._id
        return import_aggregation_models_from_spec(spec)

    def test_aggregate_table(self):
        adapter = AggregateIndicatorSqlAdapter(self.monthly_aggregate_table_definition)
        table = adapter.get_table()
        id_column = table.columns['doc_id']

        # basic checks on primary columns
        self.assertEqual(UnicodeText, type(id_column.type))
        self.assertEqual(True, id_column.primary_key)
        self.assertEqual(Date, type(table.columns['month'].type))
        # todo: pregnancy_start_date should eventually be a date, but nothing in the bootstrap
        # code tells the engine that this is the case. eventually this should be done through
        # the data dictionary.
        self.assertEqual(UnicodeText, type(table.columns['pregnancy_start_date'].type))
        self.assertEqual(Integer, type(table.columns['open_in_month'].type))
        self.assertEqual(SmallInteger, type(table.columns['pregnant_in_month'].type))

        # basic check on secondary column
        self.assertEqual(Integer, type(table.columns['fu_forms_in_month'].type))

    def test_get_aggregation_start_period(self):
        self.assertEqual(self.case_date_opened,
                         get_aggregation_start_period(self.monthly_aggregate_table_definition))

    def test_get_aggregation_end_period(self):
        self.assertEqual(datetime.utcnow().date(),
                         get_aggregation_end_period(self.monthly_aggregate_table_definition).date())

    def test_basic_aggregation(self):
        # next generate our table
        aggregate_table_adapter = AggregateIndicatorSqlAdapter(self.basic_aggregate_table_definition)
        aggregate_table_adapter.rebuild_table()

        populate_aggregate_table_data(aggregate_table_adapter)
        self._check_basic_results()

        # confirm it's also idempotent
        populate_aggregate_table_data(aggregate_table_adapter)
        self._check_basic_results()

    def _check_basic_results(self):
        aggregate_table_adapter = AggregateIndicatorSqlAdapter(self.basic_aggregate_table_definition)
        aggregate_table = aggregate_table_adapter.get_table()
        aggregate_query = aggregate_table_adapter.get_query_object()

        doc_id_column = aggregate_table.c['doc_id']

        # before december the case should not exist
        self.assertEqual(1, aggregate_query.filter(
            doc_id_column == self.case_id,
        ).count())

        row = aggregate_query.filter(
            doc_id_column == self.case_id,
        ).one()
        self.assertEqual(self.case_name, row.name)
        self.assertEqual('2018-01-21', row.pregnancy_start_date)
        self.assertEqual(3, row.fu_forms)

    def test_monthly_aggregation(self):
        # generate our table
        aggregate_table_adapter = AggregateIndicatorSqlAdapter(self.monthly_aggregate_table_definition)
        aggregate_table_adapter.rebuild_table()

        populate_aggregate_table_data(aggregate_table_adapter)
        self._check_results()

        # confirm it's also idempotent
        populate_aggregate_table_data(aggregate_table_adapter)
        self._check_results()

    def _check_results(self):
        aggregate_table_adapter = AggregateIndicatorSqlAdapter(self.monthly_aggregate_table_definition)
        aggregate_table = aggregate_table_adapter.get_table()
        aggregate_query = aggregate_table_adapter.get_query_object()

        doc_id_column = aggregate_table.c['doc_id']
        month_column = aggregate_table.c['month']

        # before december the case should not exist
        self.assertEqual(0, aggregate_query.filter(
            doc_id_column == self.case_id,
            month_column <= '2017-11-01'
        ).count())

        # in december the case should exist, but should not be flagged as pregnant
        row = aggregate_query.filter(
            doc_id_column == self.case_id,
            month_column == '2017-12-01'
        ).one()
        self.assertEqual(self.case_name, row.name)
        self.assertEqual(1, row.open_in_month)
        self.assertEqual(0, row.pregnant_in_month)
        self.assertEqual(None, row.fu_forms_in_month)

        # in january the case should exist, and be flagged as pregnant
        row = aggregate_query.filter(
            doc_id_column == self.case_id,
            month_column == '2018-01-01'
        ).one()
        self.assertEqual(1, row.open_in_month)
        self.assertEqual(1, row.pregnant_in_month)
        self.assertEqual(None, row.fu_forms_in_month)

        # in march the case should exist, be flagged as pregnant, and there is a form
        row = aggregate_query.filter(
            doc_id_column == self.case_id,
            month_column == '2018-03-01'
        ).one()
        self.assertEqual(1, row.open_in_month)
        self.assertEqual(1, row.pregnant_in_month)
        self.assertEqual(1, row.fu_forms_in_month)

        # in april the case should exist, be flagged as pregnant, and there are 2 forms
        row = aggregate_query.filter(
            doc_id_column == self.case_id,
            month_column == '2018-04-01'
        ).one()
        self.assertEqual(1, row.open_in_month)
        self.assertEqual(1, row.pregnant_in_month)
        self.assertEqual(2, row.fu_forms_in_month)
<commit_msg>call super teardown last<commit_after>from __future__ import absolute_import
from __future__ import unicode_literals

import uuid

from datetime import datetime

from django.test import TestCase
from sqlalchemy import Date, Integer, SmallInteger, UnicodeText

from casexml.apps.case.mock import CaseBlock
from casexml.apps.case.tests.util import delete_all_cases, delete_all_xforms
from casexml.apps.case.util import post_case_blocks
from corehq.apps.aggregate_ucrs.importer import import_aggregation_models_from_spec
from corehq.apps.aggregate_ucrs.ingestion import populate_aggregate_table_data, get_aggregation_start_period, \
    get_aggregation_end_period
from corehq.apps.aggregate_ucrs.models import AggregateTableDefinition
from corehq.apps.aggregate_ucrs.sql.adapter import AggregateIndicatorSqlAdapter
from corehq.apps.aggregate_ucrs.tests.base import AggregationBaseTestMixin
from corehq.apps.app_manager.tests.app_factory import AppFactory
from corehq.apps.app_manager.tests.util import delete_all_apps
from corehq.apps.app_manager.xform_builder import XFormBuilder
from corehq.apps.receiverwrapper.util import submit_form_locally
from corehq.apps.userreports.app_manager.helpers import get_form_data_source, get_case_data_source
from corehq.apps.userreports.tasks import _iteratively_build_table
from corehq.apps.userreports.util import get_indicator_adapter
from corehq.form_processor.utils.xform import FormSubmissionBuilder, TestFormMetadata

class UCRAggregationTest(TestCase, AggregationBaseTestMixin):
    domain = 'agg'
    case_type = 'agg-cases'
    case_name = 'Rowena Luk'
    case_date_opened = datetime(2017, 12, 19)
    case_properties = (
        ('first_name', 'First Name', 'string', 'Rowena'),
        ('last_name', 'Last Name', 'string', 'Luk'),
        ('children', 'Children', 'int', '1'),
        ('lmp', 'Pregnancy Start Date', 'date', '2018-01-21'),
        ('delivery_date', 'Pregnancy End Date', 'date', '2018-10-27'),
    )
    fu_visit_dates = (
        datetime(2018, 3, 16),
        datetime(2018, 4, 11),
        datetime(2018, 4, 16),
    )
    parent_case_type = 'parent'
    parent_name = 'Mama'

    @classmethod
    def setUpClass(cls):
        super(UCRAggregationTest, cls).setUpClass()
        # cleanup any previous data
        cls._cleanup_data()

        # setup app
        factory = AppFactory(domain=cls.domain)
        # parent case module, incl opening child cases of main type
        m_parent, f_parent = factory.new_basic_module('Parent Module', cls.parent_case_type)
        factory.form_opens_case(f_parent, case_type=cls.parent_case_type)
        factory.form_opens_case(f_parent, case_type=cls.case_type, is_subcase=True)

        # main module
        m0, f0 = factory.new_basic_module('A Module', cls.case_type)
        f1 = factory.new_form(m0)
        f1.source = cls._get_xform()
        factory.form_requires_case(f1, case_type=cls.case_type, update={
            cp[0]: '/data/{}'.format(cp[0]) for cp in cls.case_properties
        })
        cls.followup_form = f1

        cls.app = factory.app
        cls.app.save()

        # create form and case ucrs
        cls.form_data_source = get_form_data_source(cls.app, cls.followup_form)
        cls.case_data_source = get_case_data_source(cls.app, cls.case_type)
        cls.parent_case_data_source = get_case_data_source(cls.app, cls.parent_case_type)

        # create some data - first just create the case
        cls.parent_case_id = cls._create_parent_case(cls.parent_name)
        cls.case_id = cls._create_case(cls.parent_case_id)
        for fu_date in cls.fu_visit_dates:
            cls._submit_followup_form(cls.case_id, received_on=fu_date)

        # populate the UCRs with the data we just created
        cls.form_adapter = get_indicator_adapter(cls.form_data_source)
        cls.case_adapter = get_indicator_adapter(cls.case_data_source)
        cls.parent_case_adapter = get_indicator_adapter(cls.parent_case_data_source)

        cls.form_adapter.rebuild_table()
        cls.case_adapter.rebuild_table()
        cls.parent_case_adapter.rebuild_table()

        _iteratively_build_table(cls.form_data_source)
        _iteratively_build_table(cls.case_data_source)
        _iteratively_build_table(cls.parent_case_data_source)

        # setup AggregateTableDefinition
        cls.monthly_aggregate_table_definition = cls._get_monthly_aggregate_table_definition()
        cls.basic_aggregate_table_definition = cls._get_basic_aggregate_table_definition()

    @classmethod
    def tearDownClass(cls):
        cls._cleanup_data()
        super(UCRAggregationTest, cls).tearDownClass()

    @classmethod
    def _cleanup_data(cls):
        delete_all_cases()
        delete_all_xforms()
        delete_all_apps()
        AggregateTableDefinition.objects.all().delete()

    def setUp(self):
        # confirm that our setupClass function properly did its job
        self.assertEqual(3, self.form_adapter.get_query_object().count())
        self.assertEqual(1, self.case_adapter.get_query_object().count())
        self.assertEqual(1, self.parent_case_adapter.get_query_object().count())

    @classmethod
    def _get_xform(cls):
        xform = XFormBuilder()
        for prop_name, prop_text, datatype, _ in cls.case_properties:
            xform.new_question(prop_name, prop_text, data_type=datatype)
        return xform.tostring()

    @classmethod
    def _get_case_property_values(cls):
        return {
            cp[0]: cp[-1] for cp in cls.case_properties
        }

    @classmethod
    def _create_case(cls, parent_id):
        case_id = uuid.uuid4().hex
        caseblock = CaseBlock(
            case_id=case_id,
            case_type=cls.case_type,
            date_opened=cls.case_date_opened,
            case_name=cls.case_name,
            update=cls._get_case_property_values(),
            index={
                'parent': (cls.case_type, parent_id)
            }
        )
        post_case_blocks([caseblock.as_xml()], domain=cls.domain)
        return case_id

    @classmethod
    def _create_parent_case(cls, case_name):
        parent_id = uuid.uuid4().hex
        post_case_blocks(
            [
                CaseBlock(
                    create=True,
                    case_id=parent_id,
                    case_name=case_name,
                    case_type=cls.parent_case_type,
                ).as_xml()
            ], domain=cls.domain
        )
        return parent_id

    @classmethod
    def _submit_followup_form(cls, case_id, received_on):
        form_id = uuid.uuid4().hex
        form_meta = TestFormMetadata(
            domain=cls.domain,
            xmlns=cls.followup_form.xmlns,
            app_id=cls.app._id,
            received_on=received_on,
        )
        properties = cls._get_case_property_values()
        caseblock = CaseBlock(
            case_id=case_id,
            update=properties,
        )
        form_builder = FormSubmissionBuilder(
            form_id=form_id,
            metadata=form_meta,
            case_blocks=[caseblock],
            form_properties=properties,
        )
        submit_form_locally(form_builder.as_xml_string(), cls.domain, received_on=received_on)
        return form_id

    @classmethod
    def _get_monthly_aggregate_table_definition(cls):
        spec = cls.get_monthly_config_spec()
        spec.primary_table.data_source_id = cls.case_data_source._id
        spec.secondary_tables[0].data_source_id = cls.form_data_source._id
        return import_aggregation_models_from_spec(spec)

    @classmethod
    def _get_basic_aggregate_table_definition(cls):
        spec = cls.get_basic_config_spec()
        spec.primary_table.data_source_id = cls.case_data_source._id
        spec.secondary_tables[0].data_source_id = cls.form_data_source._id
        spec.secondary_tables[1].data_source_id = cls.parent_case_data_source._id
        return import_aggregation_models_from_spec(spec)

    def test_aggregate_table(self):
        adapter = AggregateIndicatorSqlAdapter(self.monthly_aggregate_table_definition)
        table = adapter.get_table()
        id_column = table.columns['doc_id']

        # basic checks on primary columns
        self.assertEqual(UnicodeText, type(id_column.type))
        self.assertEqual(True, id_column.primary_key)
        self.assertEqual(Date, type(table.columns['month'].type))
        # todo: pregnancy_start_date should eventually be a date, but nothing in the bootstrap
        # code tells the engine that this is the case. eventually this should be done through
        # the data dictionary.
        self.assertEqual(UnicodeText, type(table.columns['pregnancy_start_date'].type))
        self.assertEqual(Integer, type(table.columns['open_in_month'].type))
        self.assertEqual(SmallInteger, type(table.columns['pregnant_in_month'].type))

        # basic check on secondary column
        self.assertEqual(Integer, type(table.columns['fu_forms_in_month'].type))

    def test_get_aggregation_start_period(self):
        self.assertEqual(self.case_date_opened,
                         get_aggregation_start_period(self.monthly_aggregate_table_definition))

    def test_get_aggregation_end_period(self):
        self.assertEqual(datetime.utcnow().date(),
                         get_aggregation_end_period(self.monthly_aggregate_table_definition).date())

    def test_basic_aggregation(self):
        # next generate our table
        aggregate_table_adapter = AggregateIndicatorSqlAdapter(self.basic_aggregate_table_definition)
        aggregate_table_adapter.rebuild_table()

        populate_aggregate_table_data(aggregate_table_adapter)
        self._check_basic_results()

        # confirm it's also idempotent
        populate_aggregate_table_data(aggregate_table_adapter)
        self._check_basic_results()

    def _check_basic_results(self):
        aggregate_table_adapter = AggregateIndicatorSqlAdapter(self.basic_aggregate_table_definition)
        aggregate_table = aggregate_table_adapter.get_table()
        aggregate_query = aggregate_table_adapter.get_query_object()

        doc_id_column = aggregate_table.c['doc_id']

        # before december the case should not exist
        self.assertEqual(1, aggregate_query.filter(
            doc_id_column == self.case_id,
        ).count())

        row = aggregate_query.filter(
            doc_id_column == self.case_id,
        ).one()
        self.assertEqual(self.case_name, row.name)
        self.assertEqual('2018-01-21', row.pregnancy_start_date)
        self.assertEqual(3, row.fu_forms)

    def test_monthly_aggregation(self):
        # generate our table
        aggregate_table_adapter = AggregateIndicatorSqlAdapter(self.monthly_aggregate_table_definition)
        aggregate_table_adapter.rebuild_table()

        populate_aggregate_table_data(aggregate_table_adapter)
        self._check_results()

        # confirm it's also idempotent
        populate_aggregate_table_data(aggregate_table_adapter)
        self._check_results()

    def _check_results(self):
        aggregate_table_adapter = AggregateIndicatorSqlAdapter(self.monthly_aggregate_table_definition)
        aggregate_table = aggregate_table_adapter.get_table()
        aggregate_query = aggregate_table_adapter.get_query_object()

        doc_id_column = aggregate_table.c['doc_id']
        month_column = aggregate_table.c['month']

        # before december the case should not exist
        self.assertEqual(0, aggregate_query.filter(
            doc_id_column == self.case_id,
            month_column <= '2017-11-01'
        ).count())

        # in december the case should exist, but should not be flagged as pregnant
        row = aggregate_query.filter(
            doc_id_column == self.case_id,
            month_column == '2017-12-01'
        ).one()
        self.assertEqual(self.case_name, row.name)
        self.assertEqual(1, row.open_in_month)
        self.assertEqual(0, row.pregnant_in_month)
        self.assertEqual(None, row.fu_forms_in_month)

        # in january the case should exist, and be flagged as pregnant
        row = aggregate_query.filter(
            doc_id_column == self.case_id,
            month_column == '2018-01-01'
        ).one()
        self.assertEqual(1, row.open_in_month)
        self.assertEqual(1, row.pregnant_in_month)
        self.assertEqual(None, row.fu_forms_in_month)

        # in march the case should exist, be flagged as pregnant, and there is a form
        row = aggregate_query.filter(
            doc_id_column == self.case_id,
            month_column == '2018-03-01'
        ).one()
        self.assertEqual(1, row.open_in_month)
        self.assertEqual(1, row.pregnant_in_month)
        self.assertEqual(1, row.fu_forms_in_month)

        # in april the case should exist, be flagged as pregnant, and there are 2 forms
        row = aggregate_query.filter(
            doc_id_column == self.case_id,
            month_column == '2018-04-01'
        ).one()
        self.assertEqual(1, row.open_in_month)
        self.assertEqual(1, row.pregnant_in_month)
        self.assertEqual(2, row.fu_forms_in_month)
<|endoftext|>


<commit_before>import numpy as np
import os

import chainer

from chainercv.chainer_experimental.datasets.sliceable import GetterDataset
from chainercv.utils import read_image

def directory_parsing_label_names(root, numerical_sort=False):
    """Get label names from the directories that are named by them.

    The label names are the names of the directories that locate a
    layer below the root directory.

    The label names can be used together with
    :class:`~chainercv.datasets.DirectoryParsingLabelDataset`.
    The index of a label name corresponds to the label id
    that is used by the dataset to refer the label.

    Args:
        root (str): The root directory.
        numerical_sort (bool): Label names are sorted numerically.
            This means that label :obj:`2` is before label :obj:`10`,
            which is not the case when string sort is used.
            The default value is :obj:`False`.

    Returns:
        list of strings:
        Sorted names of classes.

    """
    label_names = [d for d in os.listdir(root)
                   if os.path.isdir(os.path.join(root, d))]

    if not numerical_sort:
        label_names.sort()
    else:
        label_names = sorted(label_names, key=int)
    return label_names

def _check_img_ext(path):
    img_extensions = ['.jpg', '.jpeg', '.png', '.ppm', '.bmp']
    return any(os.path.splitext(path)[1].lower() == extension for
               extension in img_extensions)

def _parse_label_dataset(root, label_names,
                         check_img_file=_check_img_ext):
    img_paths = []
    labels = []
    for label, label_name in enumerate(label_names):
        label_dir = os.path.join(root, label_name)
        if not os.path.isdir(label_dir):
            continue

        walk_dir = sorted(os.walk(label_dir), key=lambda x: x[0])
        for cur_dir, _, names in walk_dir:
            names = sorted(names)
            for name in names:
                img_path = os.path.join(cur_dir, name)
                if check_img_file(img_path):
                    img_paths.append(img_path)
                    labels.append(label)

    return img_paths, np.array(labels, np.int32)

class DirectoryParsingLabelDataset(GetterDataset):
    """A label dataset whose label names are the names of the subdirectories.

    The label names are the names of the directories that locate a layer below
    the root directory.
    All images locating under the subdirectoies will be categorized to classes
    with subdirectory names.
    An image is parsed only when the function :obj:`check_img_file`
    returns :obj:`True` by taking the path to the image as an argument.
    If :obj:`check_img_file` is :obj:`None`,
    the path with any image extensions will be parsed.

    Example:

        A directory structure should be one like below.

        .. code::

            root
            |-- class_0
            |   |-- img_0.png
            |   |-- img_1.png
            |
            --- class_1
                |-- img_0.png

        >>> from chainercv.datasets import DirectoryParsingLabelDataset
        >>> dataset = DirectoryParsingLabelDataset('root')
        >>> dataset.paths
        ['root/class_0/img_0.png', 'root/class_0/img_1.png',
        'root_class_1/img_0.png']
        >>> dataset.labels
        array([0, 0, 1])

    Args:
        root (str): The root directory.
        check_img_file (callable): A function to determine
            if a file should be included in the dataset.
        color (bool): If :obj:`True`, this dataset read images
            as color images. The default value is :obj:`True`.
        numerical_sort (bool): Label names are sorted numerically.
            This means that label :obj:`2` is before label :obj:`10`,
            which is not the case when string sort is used.
            Regardless of this option, string sort is used for the
            order of files with the same label.
            The default value is :obj:`False`.

    This dataset returns the following data.

    .. csv-table::
        :header: name, shape, dtype, format

        :obj:`img`, ":math:`(3, H, W)` [#directory_parsing_1]_", \
        :obj:`float32`, "RGB, :math:`[0, 255]`"
        :obj:`label`, scalar, :obj:`int32`, ":math:`[0, \#class - 1]`"

    .. [#directory_parsing_1] :math:`(1, H, W)` if :obj:`color = False`.
    """

    def __init__(self, root, check_img_file=None, color=True,
                 numerical_sort=False):
        super(DirectoryParsingLabelDataset, self).__init__()

        self.color = color

        label_names = directory_parsing_label_names(
            root, numerical_sort=numerical_sort)
        if check_img_file is None:
            check_img_file = _check_img_ext

        self.img_paths, self.labels = _parse_label_dataset(
            root, label_names, check_img_file)

        self.add_getter('img', lambda i:
                        read_image(self.img_paths[i], color=self.color))
        self.add_getter('label', lambda i: self.labels[i])

    def __len__(self):
        return len(self.img_paths)
<commit_msg>flake8<commit_after>import numpy as np
import os

from chainercv.chainer_experimental.datasets.sliceable import GetterDataset
from chainercv.utils import read_image

def directory_parsing_label_names(root, numerical_sort=False):
    """Get label names from the directories that are named by them.

    The label names are the names of the directories that locate a
    layer below the root directory.

    The label names can be used together with
    :class:`~chainercv.datasets.DirectoryParsingLabelDataset`.
    The index of a label name corresponds to the label id
    that is used by the dataset to refer the label.

    Args:
        root (str): The root directory.
        numerical_sort (bool): Label names are sorted numerically.
            This means that label :obj:`2` is before label :obj:`10`,
            which is not the case when string sort is used.
            The default value is :obj:`False`.

    Returns:
        list of strings:
        Sorted names of classes.

    """
    label_names = [d for d in os.listdir(root)
                   if os.path.isdir(os.path.join(root, d))]

    if not numerical_sort:
        label_names.sort()
    else:
        label_names = sorted(label_names, key=int)
    return label_names

def _check_img_ext(path):
    img_extensions = ['.jpg', '.jpeg', '.png', '.ppm', '.bmp']
    return any(os.path.splitext(path)[1].lower() == extension for
               extension in img_extensions)

def _parse_label_dataset(root, label_names,
                         check_img_file=_check_img_ext):
    img_paths = []
    labels = []
    for label, label_name in enumerate(label_names):
        label_dir = os.path.join(root, label_name)
        if not os.path.isdir(label_dir):
            continue

        walk_dir = sorted(os.walk(label_dir), key=lambda x: x[0])
        for cur_dir, _, names in walk_dir:
            names = sorted(names)
            for name in names:
                img_path = os.path.join(cur_dir, name)
                if check_img_file(img_path):
                    img_paths.append(img_path)
                    labels.append(label)

    return img_paths, np.array(labels, np.int32)

class DirectoryParsingLabelDataset(GetterDataset):
    """A label dataset whose label names are the names of the subdirectories.

    The label names are the names of the directories that locate a layer below
    the root directory.
    All images locating under the subdirectoies will be categorized to classes
    with subdirectory names.
    An image is parsed only when the function :obj:`check_img_file`
    returns :obj:`True` by taking the path to the image as an argument.
    If :obj:`check_img_file` is :obj:`None`,
    the path with any image extensions will be parsed.

    Example:

        A directory structure should be one like below.

        .. code::

            root
            |-- class_0
            |   |-- img_0.png
            |   |-- img_1.png
            |
            --- class_1
                |-- img_0.png

        >>> from chainercv.datasets import DirectoryParsingLabelDataset
        >>> dataset = DirectoryParsingLabelDataset('root')
        >>> dataset.paths
        ['root/class_0/img_0.png', 'root/class_0/img_1.png',
        'root_class_1/img_0.png']
        >>> dataset.labels
        array([0, 0, 1])

    Args:
        root (str): The root directory.
        check_img_file (callable): A function to determine
            if a file should be included in the dataset.
        color (bool): If :obj:`True`, this dataset read images
            as color images. The default value is :obj:`True`.
        numerical_sort (bool): Label names are sorted numerically.
            This means that label :obj:`2` is before label :obj:`10`,
            which is not the case when string sort is used.
            Regardless of this option, string sort is used for the
            order of files with the same label.
            The default value is :obj:`False`.

    This dataset returns the following data.

    .. csv-table::
        :header: name, shape, dtype, format

        :obj:`img`, ":math:`(3, H, W)` [#directory_parsing_1]_", \
        :obj:`float32`, "RGB, :math:`[0, 255]`"
        :obj:`label`, scalar, :obj:`int32`, ":math:`[0, \#class - 1]`"

    .. [#directory_parsing_1] :math:`(1, H, W)` if :obj:`color = False`.
    """

    def __init__(self, root, check_img_file=None, color=True,
                 numerical_sort=False):
        super(DirectoryParsingLabelDataset, self).__init__()

        self.color = color

        label_names = directory_parsing_label_names(
            root, numerical_sort=numerical_sort)
        if check_img_file is None:
            check_img_file = _check_img_ext

        self.img_paths, self.labels = _parse_label_dataset(
            root, label_names, check_img_file)

        self.add_getter('img', lambda i:
                        read_image(self.img_paths[i], color=self.color))
        self.add_getter('label', lambda i: self.labels[i])

    def __len__(self):
        return len(self.img_paths)
<|endoftext|>