<commit_before># encoding: utf-8
#!/usr/bin/env python
import logging
import datetime
import time
from sqlalchemy import Column, Table, MetaData, select, create_engine, \
    ForeignKey, bindparam, and_, or_, exc
import connectors.task_pb2
import connectors.realtime_pb2
import connectors.type_pb2
import google
import os

def int_to_bitset(s):
    return str(s) if s <= 1 else int_to_bitset(s >> 1) + str(s & 1)

def get_pos_time(sql_time):
    return int(time.mktime(sql_time.timetuple()))

def get_datetime_to_second(sql_time):
    tt = sql_time.timetuple()
    return (tt.tm_hour * 60 * 60) + (tt.tm_min * 60) + tt.tm_sec

def get_navitia_type(object_type):
    if object_type == 'StopPoint':
        return connectors.type_pb2.STOP_POINT
    elif object_type == 'VehicleJourney':
        return connectors.type_pb2.VEHICLE_JOURNEY
    elif object_type == 'Line':
        return connectors.type_pb2.LINE
    elif object_type == 'Network':
        return connectors.type_pb2.NETWORK
    elif object_type == 'Route':
        return connectors.type_pb2.JOURNEY_PATTERN
    elif object_type == 'StopArea':
        return connectors.type_pb2.STOP_AREA
    elif object_type == 'RoutePoint':
        return connectors.type_pb2.JOURNEY_PATTERN_POINT
    else:
        return -1

class AtRealtimeReader(object):

    """
    This class load messages and perturbation from the "alerte trafic" database
    """

    def __init__(self, config, redis_helper):
        self.message_list = []
        self.perturbation_list = []
        self.__engine = create_engine(
            config.at_connection_string + '?charset=utf8', echo=False)
        self._redis_helper = redis_helper
        self.meta = MetaData(self.__engine)
        self.event_table = Table('event', self.meta, autoload=True)

        self.impact_table = Table('impact', self.meta,
                                  Column('Event_ID', None,
                                         ForeignKey('event.Event_ID')),
                                  Column('TCObjectRef_ID', None, ForeignKey(
                                      'tcobjectref.TCObjectRef_ID')),
                                  autoload=True)

        self.tcobjectref_table = Table('tcobjectref', self.meta, autoload=True)

        self.impactbroadcast_table = \
            Table('impactbroadcast', self.meta,
                  Column('Impact_ID', None,
                         ForeignKey('impact.Impact_ID')),
                  Column('MsgMedia_ID', None,
                         ForeignKey('msgmedia.MsgMedia_ID')),
                  autoload=True)

        self.msgmedia_table = Table('msgmedia', self.meta, autoload=True)

        self.label_impact_id = 'impact_id'
        self.label_impact_state = 'impact_state'
        self.label_publication_start_date = 'publication_start_date'
        self.label_publication_end_date = 'publication_end_date'
        self.label_application_start_date = 'application_start_date'
        self.label_application_end_date = 'application_end_date'
        self.label_application_daily_start_hour =\
            'application_daily_start_hour'
        self.label_application_daily_end_hour =\
            'application_daily_end_hour'
        self.label_active_days = 'active_days'
        self.label_object_external_code = 'object_external_code'
        self.label_object_type = 'object_type'
        self.label_title = 'title'
        self.label_message = 'message'
        self.label_message_lang = 'lang'

        self.last_exec_file_name = './last_exec_time.txt'
        self.datetime_format = '%Y-%m-%d %H:%M:%S'
        self._collections = {"StopPoint": "stop_point",
                             "VehicleJourney": "vehicle_journey",
                             "Line": "line",
                             "Network": "network",
                             "StopArea": "stop_area"}

    def get_last_execution_time(self):
        if os.path.exists(self.last_exec_file_name):
            last_exec_file = open(self.last_exec_file_name, 'r')
            las_execution_time = last_exec_file.readline().rstrip('\n\r')
            return datetime.datetime.strptime(las_execution_time,
                                              self.datetime_format)
        else:
            return datetime.datetime.now()

    def set_last_execution_time(self, last_execution_time):
        last_exec_file = open(self.last_exec_file_name, 'w')
        last_exec_file.write(
            last_execution_time.strftime(self.datetime_format))

    def get_uri(self, externalcode, object_type):
        uri = None
        if object_type in self._collections.keys():
            self._redis_helper.set_prefix(self._collections[object_type])
            uri = self._redis_helper.get(externalcode)
        return uri

    def create_pertubation(self, message):
        pertubation = connectors.realtime_pb2.AtPerturbation()
        pertubation.uri = message.uri
        pertubation.object.object_uri = message.object.object_uri
        pertubation.object.object_type = message.object.object_type
        pertubation.start_application_date = \
            message.start_publication_date
        pertubation.end_application_date = \
            message.end_application_date
        pertubation.start_application_daily_hour = \
            message.start_application_daily_hour
        pertubation.end_application_daily_hour = \
            message.end_application_daily_hour
        pertubation.active_days = message.active_days
        return pertubation

    def get_status_message(self, status):
        if status.lower() == "information":
            return 0
        if status.lower() == "warning":
            return 1
        if status.lower() == "disrupt":
            return 2
        return None

    def set_message(self, result_proxy):
        """

        :param result_proxy:
        """
        last_impact_id = -1
        for row in result_proxy:
            try:
                current_uri = self.get_uri(
                    row[self.label_object_external_code],
                    row[self.label_object_type])
                if current_uri is None:
                    logging.getLogger('connector').warn(
                        "Object [%s] rejected : "
                        "No extcode and uri correspondance",
                        row[self.label_object_external_code])
                else:
                    if last_impact_id != row[self.label_impact_id]:
                        last_impact_id = row[self.label_impact_id]
                        message = connectors.realtime_pb2.Message()
                        self.message_list.append(message)

                        message.uri = str(row[self.label_impact_id]) + '-' + \
                            row[self.label_message_lang]

                        message.start_publication_date = get_pos_time(
                            row[self.label_publication_start_date])

                        message.end_publication_date = get_pos_time(
                            row[self.label_publication_end_date])
                        message.start_application_date = get_pos_time(
                            row[self.label_application_start_date])
                        message.end_application_date = get_pos_time(
                            row[self.label_application_end_date])
                        # number of seconds since midnigth
                        message.start_application_daily_hour = \
                            get_datetime_to_second(
                                row[self.label_application_daily_start_hour])
                        message.end_application_daily_hour = \
                            get_datetime_to_second(
                                row[self.label_application_daily_end_hour])
                        message.active_days = \
                            int_to_bitset(row[self.label_active_days]) + '1'

                        message.object.object_uri = current_uri

                        message.object.object_type = \
                            get_navitia_type(row[self.label_object_type])

                        message.message_status = self.get_status_message(
                            row[self.label_impact_state])

                    localized_message = message.localized_messages.add()
                    localized_message.language = row[self.label_message_lang]
                    localized_message.body = row[self.label_message]
                    localized_message.title = row[self.label_title]

                    if row[self.label_impact_state] == 'Disrupt':
                        self.perturbation_list.append(
                            self.create_pertubation(message))

                    print str(
                        row[self.label_active_days]) + ' - ' +\
                        message.active_days
            except google.protobuf.message.DecodeError as e:
                logging.getLogger('connector').warn(
                    "message is not a valid "
                    "protobuf task: %s", str(e))

    def set_request(self):
        return select([self.event_table.c.Event_ID,
                       self.impact_table.c.Impact_ID
                       .label(self.label_impact_id),
                       self.impact_table.c.Impact_State
                       .label(self.label_impact_state),
                       self.event_table.c.Event_PublicationStartDate
                       .label(self.label_publication_start_date),
                       self.event_table.c.Event_PublicationEndDate
                       .label(self.label_publication_end_date),
                       self.impact_table.c.Impact_EventStartDate
                       .label(self.label_application_start_date),
                       self.impact_table.c.Impact_EventEndDate
                       .label(self.label_application_end_date),
                       self.impact_table.c.Impact_DailyStartDate
                       .label(self.label_application_daily_start_hour),
                       self.impact_table.c.Impact_DailyEndDate
                       .label(self.label_application_daily_end_hour),
                       self.impact_table.c.Impact_ActiveDays
                       .label(self.label_active_days),
                       self.tcobjectref_table.c.TCObjectCodeExt
                       .label(self.label_object_external_code),
                       self.tcobjectref_table.c.TCObjectType
                       .label(self.label_object_type),
                       self.impactbroadcast_table.c.Impact_Title
                       .label(self.label_title),
                       self.impactbroadcast_table.c.Impact_Msg
                       .label(self.label_message),
                       self.msgmedia_table.c.MsgMedia_Lang
                       .label(self.label_message_lang)],
                      # clause where
                      and_(self.msgmedia_table.c.MsgMedia_Media == bindparam(
                          'media_media'),
                          self.event_table.c.Event_PublicationEndDate
                          >= bindparam('event_publicationenddate'),
                          or_(self.event_table.c.Event_CloseDate == None,
                              self.event_table.c.Event_CloseDate > bindparam(
                                  'event_closedate'),
                              self.impact_table.c
                              .Impact_SelfModificationDate == None,
                              self.impact_table.c
                              .Impact_SelfModificationDate > bindparam(
                                  'impact_modification_date'),
                              self.impact_table.c
                              .Impact_ChildrenModificationDate == None,
                              self.impact_table.c
                              .Impact_ChildrenModificationDate > bindparam(
                                  'impact_modification_date'))
                      ),
                      # jointure
                      from_obj=[self.event_table.join(self.impact_table).join(
                          self.tcobjectref_table).join(
                          self.impactbroadcast_table).join(
                          self.msgmedia_table)]
                      ).order_by(self.impact_table.c.Impact_ID)

    # impact_SelfModificationDate, impact_ChildrenModificationDate

    def execute(self):
        logger = logging.getLogger('connector')
        conn = None
        try:
            conn = self.__engine.connect()
        except exc.SQLAlchemyError as e:
            logger.exception('error durring connection')
        if conn is not None:
            result = None
            try:
                s = self.set_request()
                execution_time = datetime.datetime.now()
                # read last execution time
                last_execution_time = self.get_last_execution_time()
                result = \
                    conn.execute(s,
                                 media_media='Internet',
                                 event_publicationenddate=last_execution_time,
                                 #datetime.datetime(2013, 9, 01),
                                 event_closedate=datetime.datetime.now(),
                                 impact_modification_date=last_execution_time)
                # save execution time
                self.set_last_execution_time(execution_time)
            except exc.SQLAlchemyError as e:
                logger.exception('error during request')
            if result is not None:
                self.set_message(result)
                result.close()
<commit_msg>at : control of connection string<commit_after># encoding: utf-8
#!/usr/bin/env python
import logging
import datetime
import time
from sqlalchemy import Column, Table, MetaData, select, create_engine, \
    ForeignKey, bindparam, and_, or_, exc
from sqlalchemy.engine import url
import connectors.task_pb2
import connectors.realtime_pb2
import connectors.type_pb2
import google
import os

def int_to_bitset(s):
    return str(s) if s <= 1 else int_to_bitset(s >> 1) + str(s & 1)

def get_pos_time(sql_time):
    return int(time.mktime(sql_time.timetuple()))

def get_datetime_to_second(sql_time):
    tt = sql_time.timetuple()
    return (tt.tm_hour * 60 * 60) + (tt.tm_min * 60) + tt.tm_sec

def get_navitia_type(object_type):
    if object_type == 'StopPoint':
        return connectors.type_pb2.STOP_POINT
    elif object_type == 'VehicleJourney':
        return connectors.type_pb2.VEHICLE_JOURNEY
    elif object_type == 'Line':
        return connectors.type_pb2.LINE
    elif object_type == 'Network':
        return connectors.type_pb2.NETWORK
    elif object_type == 'Route':
        return connectors.type_pb2.JOURNEY_PATTERN
    elif object_type == 'StopArea':
        return connectors.type_pb2.STOP_AREA
    elif object_type == 'RoutePoint':
        return connectors.type_pb2.JOURNEY_PATTERN_POINT
    else:
        return -1

class AtRealtimeReader(object):

    """
    This class load messages and perturbation from the "alerte trafic" database
    """
    def url_valide(self, config):
        u = url.make_url(config.at_connection_string)
        if u.drivername is None:
            return False
        if u.username is None:
            return False
        if u.password is None:
            return False
        if u.host is None:
            return False
        if u.database is None:
            return False
        return True

    def __init__(self, config, redis_helper):
        self.message_list = []
        self.perturbation_list = []
        if not self.url_valide(config):
            raise ValueError("at-connection-string is not valid: " +
                             config.at_connection_string)
        try:
            self.__engine = create_engine(
                config.at_connection_string + '?charset=utf8')
        except:
            raise ValueError("AT : Connecting at server failed")

        self._redis_helper = redis_helper
        self.meta = MetaData(self.__engine)
        self.event_table = Table('event', self.meta, autoload=True)

        self.impact_table = Table('impact', self.meta,
                                  Column('Event_ID', None,
                                         ForeignKey('event.Event_ID')),
                                  Column('TCObjectRef_ID', None, ForeignKey(
                                      'tcobjectref.TCObjectRef_ID')),
                                  autoload=True)

        self.tcobjectref_table = Table('tcobjectref', self.meta, autoload=True)

        self.impactbroadcast_table = \
            Table('impactbroadcast', self.meta,
                  Column('Impact_ID', None,
                         ForeignKey('impact.Impact_ID')),
                  Column('MsgMedia_ID', None,
                         ForeignKey('msgmedia.MsgMedia_ID')),
                  autoload=True)

        self.msgmedia_table = Table('msgmedia', self.meta, autoload=True)

        self.label_impact_id = 'impact_id'
        self.label_impact_state = 'impact_state'
        self.label_publication_start_date = 'publication_start_date'
        self.label_publication_end_date = 'publication_end_date'
        self.label_application_start_date = 'application_start_date'
        self.label_application_end_date = 'application_end_date'
        self.label_application_daily_start_hour =\
            'application_daily_start_hour'
        self.label_application_daily_end_hour =\
            'application_daily_end_hour'
        self.label_active_days = 'active_days'
        self.label_object_external_code = 'object_external_code'
        self.label_object_type = 'object_type'
        self.label_title = 'title'
        self.label_message = 'message'
        self.label_message_lang = 'lang'

        self.last_exec_file_name = './last_exec_time.txt'
        self.datetime_format = '%Y-%m-%d %H:%M:%S'
        self._collections = {"StopPoint": "stop_point",
                             "VehicleJourney": "vehicle_journey",
                             "Line": "line",
                             "Network": "network",
                             "StopArea": "stop_area"}

    def get_last_execution_time(self):
        if os.path.exists(self.last_exec_file_name):
            last_exec_file = open(self.last_exec_file_name, 'r')
            las_execution_time = last_exec_file.readline().rstrip('\n\r')
            return datetime.datetime.strptime(las_execution_time,
                                              self.datetime_format)
        else:
            return datetime.datetime.now()

    def set_last_execution_time(self, last_execution_time):
        last_exec_file = open(self.last_exec_file_name, 'w')
        last_exec_file.write(
            last_execution_time.strftime(self.datetime_format))

    def get_uri(self, externalcode, object_type):
        uri = None
        if object_type in self._collections.keys():
            self._redis_helper.set_prefix(self._collections[object_type])
            uri = self._redis_helper.get(externalcode)
        return uri

    def create_pertubation(self, message):
        pertubation = connectors.realtime_pb2.AtPerturbation()
        pertubation.uri = message.uri
        pertubation.object.object_uri = message.object.object_uri
        pertubation.object.object_type = message.object.object_type
        pertubation.start_application_date = \
            message.start_publication_date
        pertubation.end_application_date = \
            message.end_application_date
        pertubation.start_application_daily_hour = \
            message.start_application_daily_hour
        pertubation.end_application_daily_hour = \
            message.end_application_daily_hour
        pertubation.active_days = message.active_days
        return pertubation

    def get_status_message(self, status):
        if status.lower() == "information":
            return 0
        if status.lower() == "warning":
            return 1
        if status.lower() == "disrupt":
            return 2
        return None

    def set_message(self, result_proxy):
        """

        :param result_proxy:
        """
        last_impact_id = -1
        for row in result_proxy:
            try:
                current_uri = self.get_uri(
                    row[self.label_object_external_code],
                    row[self.label_object_type])
                if current_uri is None:
                    logging.getLogger('connector').warn(
                        "Object [%s] rejected : "
                        "No extcode and uri correspondance",
                        row[self.label_object_external_code])
                else:
                    if last_impact_id != row[self.label_impact_id]:
                        last_impact_id = row[self.label_impact_id]
                        message = connectors.realtime_pb2.Message()
                        self.message_list.append(message)

                        message.uri = str(row[self.label_impact_id]) + '-' + \
                            row[self.label_message_lang]

                        message.start_publication_date = get_pos_time(
                            row[self.label_publication_start_date])

                        message.end_publication_date = get_pos_time(
                            row[self.label_publication_end_date])
                        message.start_application_date = get_pos_time(
                            row[self.label_application_start_date])
                        message.end_application_date = get_pos_time(
                            row[self.label_application_end_date])
                        # number of seconds since midnigth
                        message.start_application_daily_hour = \
                            get_datetime_to_second(
                                row[self.label_application_daily_start_hour])
                        message.end_application_daily_hour = \
                            get_datetime_to_second(
                                row[self.label_application_daily_end_hour])
                        message.active_days = \
                            int_to_bitset(row[self.label_active_days]) + '1'

                        message.object.object_uri = current_uri

                        message.object.object_type = \
                            get_navitia_type(row[self.label_object_type])

                        message.message_status = self.get_status_message(
                            row[self.label_impact_state])

                    localized_message = message.localized_messages.add()
                    localized_message.language = row[self.label_message_lang]
                    localized_message.body = row[self.label_message]
                    localized_message.title = row[self.label_title]

                    if row[self.label_impact_state] == 'Disrupt':
                        self.perturbation_list.append(
                            self.create_pertubation(message))

                    print str(
                        row[self.label_active_days]) + ' - ' +\
                        message.active_days
            except google.protobuf.message.DecodeError as e:
                logging.getLogger('connector').warn(
                    "message is not a valid "
                    "protobuf task: %s", str(e))

    def set_request(self):
        return select([self.event_table.c.Event_ID,
                       self.impact_table.c.Impact_ID
                       .label(self.label_impact_id),
                       self.impact_table.c.Impact_State
                       .label(self.label_impact_state),
                       self.event_table.c.Event_PublicationStartDate
                       .label(self.label_publication_start_date),
                       self.event_table.c.Event_PublicationEndDate
                       .label(self.label_publication_end_date),
                       self.impact_table.c.Impact_EventStartDate
                       .label(self.label_application_start_date),
                       self.impact_table.c.Impact_EventEndDate
                       .label(self.label_application_end_date),
                       self.impact_table.c.Impact_DailyStartDate
                       .label(self.label_application_daily_start_hour),
                       self.impact_table.c.Impact_DailyEndDate
                       .label(self.label_application_daily_end_hour),
                       self.impact_table.c.Impact_ActiveDays
                       .label(self.label_active_days),
                       self.tcobjectref_table.c.TCObjectCodeExt
                       .label(self.label_object_external_code),
                       self.tcobjectref_table.c.TCObjectType
                       .label(self.label_object_type),
                       self.impactbroadcast_table.c.Impact_Title
                       .label(self.label_title),
                       self.impactbroadcast_table.c.Impact_Msg
                       .label(self.label_message),
                       self.msgmedia_table.c.MsgMedia_Lang
                       .label(self.label_message_lang)],
                      # clause where
                      and_(self.msgmedia_table.c.MsgMedia_Media == bindparam(
                          'media_media'),
                          self.event_table.c.Event_PublicationEndDate
                          >= bindparam('event_publicationenddate'),
                          or_(self.event_table.c.Event_CloseDate == None,
                              self.event_table.c.Event_CloseDate > bindparam(
                                  'event_closedate'),
                              self.impact_table.c
                              .Impact_SelfModificationDate == None,
                              self.impact_table.c
                              .Impact_SelfModificationDate > bindparam(
                                  'impact_modification_date'),
                              self.impact_table.c
                              .Impact_ChildrenModificationDate == None,
                              self.impact_table.c
                              .Impact_ChildrenModificationDate > bindparam(
                                  'impact_modification_date'))
                      ),
                      # jointure
                      from_obj=[self.event_table.join(self.impact_table).join(
                          self.tcobjectref_table).join(
                          self.impactbroadcast_table).join(
                          self.msgmedia_table)]
                      ).order_by(self.impact_table.c.Impact_ID)

    # impact_SelfModificationDate, impact_ChildrenModificationDate

    def execute(self):
        logger = logging.getLogger('connector')
        conn = None
        try:
            conn = self.__engine.connect()
        except exc.SQLAlchemyError as e:
            logger.exception('error durring connection')
        if conn is not None:
            result = None
            try:
                s = self.set_request()
                execution_time = datetime.datetime.now()
                # read last execution time
                last_execution_time = self.get_last_execution_time()
                result = \
                    conn.execute(s,
                                 media_media='Internet',
                                 event_publicationenddate=last_execution_time,
                                 #datetime.datetime(2013, 9, 01),
                                 event_closedate=datetime.datetime.now(),
                                 impact_modification_date=last_execution_time)
                # save execution time
                self.set_last_execution_time(execution_time)
            except exc.SQLAlchemyError as e:
                logger.exception('error during request')
            if result is not None:
                self.set_message(result)
                result.close()
<|endoftext|>


<commit_before>import json
from django.test import SimpleTestCase
from corehq.util.test_utils import generate_cases

from corehq.apps.app_manager.models import AdvancedForm, Form, PreloadAction, CaseReferences
from corehq.apps.app_manager.views.forms import _get_case_references

def _assert_references_equal(testcase, wrapped_references, unwrapped_references):
    testcase.assertEqual(
        wrapped_references.to_json(),
        CaseReferences.wrap(unwrapped_references).to_json(),
    )

class FormCaseReferenceTest(SimpleTestCase):

    def setUp(self):
        self.form = Form()

    def test_one_reference_old_format(self):
        post_data = {
            "references": json.dumps({
                "preload": {
                    "/data/question": "name"
                },
                "condition": {
                    "answer": None,
                    "question": None,
                    "type": "always",
                    "operator": None,
                }
            })
        }
        self.form.case_references = _get_case_references(post_data)
        _assert_references_equal(
            self,
            self.form.case_references,
            {"load": {"/data/question": ["name"]}}
        )
        self.assertFalse(self.form.actions.load_from_form.preload)

    def test_two_references(self):
        refs = {
            "load": {
                "/data/question": ["name", "dob"],
                "/data/other_question": ["close_reason"],
            },
        }
        post_data = {"case_references": json.dumps(refs)}
        self.form.case_references = _get_case_references(post_data)
        _assert_references_equal(self, self.form.case_references, refs)
        self.assertFalse(self.form.actions.load_from_form.preload)

    def test_legacy_preload_action_case_references(self):
        self.form.actions.load_from_form = PreloadAction({
            "preload": {
                "/data/question": "name"
            },
            "condition": {
                "answer": None,
                "question": None,
                "type": "always",
                "operator": None,
            }
        })
        _assert_references_equal(
            self,
            self.form.case_references,
            {"load": {"/data/question": ["name"]}}
        )

class AdvancedFormCaseReferenceTest(SimpleTestCase):

    def setUp(self):
        self.form = AdvancedForm()

    def test_one_reference(self):
        post_data = {
            "references": json.dumps({
                "preload": {
                    "/data/question": "name"
                },
                "condition": {
                    "answer": None,
                    "question": None,
                    "type": "always",
                    "operator": None,
                }
            })
        }
        self.form.case_references = _get_case_references(post_data)
        _assert_references_equal(self, self.form.case_references, {"load": {"/data/question": ["name"]}})
        self.assertFalse(hasattr(self.form.actions, "load_from_form"))

    def test_two_references(self):
        refs = {
            "load": {
                "/data/question": ["name", "dob"],
                "/data/other_question": ["close_reason"],
            },
        }
        post_data = {"case_references": json.dumps(refs)}
        self.form.case_references = _get_case_references(post_data)
        _assert_references_equal(self, self.form.case_references, refs)
        self.assertFalse(hasattr(self.form.actions, "load_from_form"))

@generate_cases([
    ({},),
    ({'load': {}},),
    ({'load': {'data/empty': []}},),
    ({'load': {'data/properties': ['p1', 'p2']}},),
    ({'load': {'data/multiple1': ['p1', 'p2'], 'data/multiple2': ['p1', 'p2']}},),
    ({'load': {}, 'save': {}},),
    ({'load': {}, 'save': {'data/empty': {}}},),
    ({'load': {}, 'save': {'data/case_type': {'case_type': 'ct'}}},),
    ({'load': {}, 'save': {'data/props': {'properties': ['p1', 'p2']}}},),
    ({'load': {}, 'save': {'data/create': {'create': True}}},),
    ({'load': {}, 'save': {'data/close': {'close': True}}},),
    ({
         'load': {
             'data/load1': ['p1', 'p2'],
             'data/load2': ['p1', 'p2']
         },
         'save': {
             'data/save1': {
                 'case_type': 'foo_type',
                 'properties': ['p1', 'p2'],
                 'create': True,
                 'close': True,
             },
             'data/save2': {
                 'case_type': 'foo_type',
                 'properties': ['p1', 'p2'],
                 'create': True,
                 'close': True,
             }
         }
     },),
])
def test_valid_args(self, case_references):
    wrapped_references = {
        'case_references': json.dumps(case_references)
    }
    _assert_references_equal(self, _get_case_references(wrapped_references), case_references)

@generate_cases([
    ({'load': {'data/non-strings': [0, 'p2']}},),
    ({'load': {}, 'extra': 'stuff'},),
    ({'load': {}, 'save': {'data/extra': {'extra': 'stuff'}}},),
    ({'load': {}, 'save': {'data/non-strings': {'properties': [0, 'p2']}}},),
    ({'load': {}, 'save': {'data/create-non-bool': {'create': 'abc'}}},),
    ({'load': {}, 'save': {'data/close-non-bool': {'close': 0}}},),
])
def test_invalid_args(self, case_references):
    with self.assertRaises(ValueError):
        wrapped_references = {
            'case_references': json.dumps(case_references)
        }
        _get_case_references(wrapped_references)
<commit_msg>[ci skip] more whitespace<commit_after>import json
from django.test import SimpleTestCase
from corehq.util.test_utils import generate_cases

from corehq.apps.app_manager.models import AdvancedForm, Form, PreloadAction, CaseReferences
from corehq.apps.app_manager.views.forms import _get_case_references

def _assert_references_equal(testcase, wrapped_references, unwrapped_references):
    testcase.assertEqual(
        wrapped_references.to_json(),
        CaseReferences.wrap(unwrapped_references).to_json(),
    )

class FormCaseReferenceTest(SimpleTestCase):

    def setUp(self):
        self.form = Form()

    def test_one_reference_old_format(self):
        post_data = {
            "references": json.dumps({
                "preload": {
                    "/data/question": "name"
                },
                "condition": {
                    "answer": None,
                    "question": None,
                    "type": "always",
                    "operator": None,
                }
            })
        }
        self.form.case_references = _get_case_references(post_data)
        _assert_references_equal(
            self,
            self.form.case_references,
            {"load": {"/data/question": ["name"]}}
        )
        self.assertFalse(self.form.actions.load_from_form.preload)

    def test_two_references(self):
        refs = {
            "load": {
                "/data/question": ["name", "dob"],
                "/data/other_question": ["close_reason"],
            },
        }
        post_data = {"case_references": json.dumps(refs)}
        self.form.case_references = _get_case_references(post_data)
        _assert_references_equal(self, self.form.case_references, refs)
        self.assertFalse(self.form.actions.load_from_form.preload)

    def test_legacy_preload_action_case_references(self):
        self.form.actions.load_from_form = PreloadAction({
            "preload": {
                "/data/question": "name"
            },
            "condition": {
                "answer": None,
                "question": None,
                "type": "always",
                "operator": None,
            }
        })
        _assert_references_equal(
            self,
            self.form.case_references,
            {"load": {"/data/question": ["name"]}}
        )

class AdvancedFormCaseReferenceTest(SimpleTestCase):

    def setUp(self):
        self.form = AdvancedForm()

    def test_one_reference(self):
        post_data = {
            "references": json.dumps({
                "preload": {
                    "/data/question": "name"
                },
                "condition": {
                    "answer": None,
                    "question": None,
                    "type": "always",
                    "operator": None,
                }
            })
        }
        self.form.case_references = _get_case_references(post_data)
        _assert_references_equal(self, self.form.case_references, {"load": {"/data/question": ["name"]}})
        self.assertFalse(hasattr(self.form.actions, "load_from_form"))

    def test_two_references(self):
        refs = {
            "load": {
                "/data/question": ["name", "dob"],
                "/data/other_question": ["close_reason"],
            },
        }
        post_data = {"case_references": json.dumps(refs)}
        self.form.case_references = _get_case_references(post_data)
        _assert_references_equal(self, self.form.case_references, refs)
        self.assertFalse(hasattr(self.form.actions, "load_from_form"))

@generate_cases([
    ({},),
    ({'load': {}},),
    ({'load': {'data/empty': []}},),
    ({'load': {'data/properties': ['p1', 'p2']}},),
    ({'load': {'data/multiple1': ['p1', 'p2'], 'data/multiple2': ['p1', 'p2']}},),
    ({'load': {}, 'save': {}},),
    ({'load': {}, 'save': {'data/empty': {}}},),
    ({'load': {}, 'save': {'data/case_type': {'case_type': 'ct'}}},),
    ({'load': {}, 'save': {'data/props': {'properties': ['p1', 'p2']}}},),
    ({'load': {}, 'save': {'data/create': {'create': True}}},),
    ({'load': {}, 'save': {'data/close': {'close': True}}},),
    ({
        'load': {
            'data/load1': ['p1', 'p2'],
            'data/load2': ['p1', 'p2']
        },
        'save': {
            'data/save1': {
                'case_type': 'foo_type',
                'properties': ['p1', 'p2'],
                'create': True,
                'close': True,
            },
            'data/save2': {
                'case_type': 'foo_type',
                'properties': ['p1', 'p2'],
                'create': True,
                'close': True,
            }
        }
    },),
])
def test_valid_args(self, case_references):
    wrapped_references = {
        'case_references': json.dumps(case_references)
    }
    _assert_references_equal(self, _get_case_references(wrapped_references), case_references)

@generate_cases([
    ({'load': {'data/non-strings': [0, 'p2']}},),
    ({'load': {}, 'extra': 'stuff'},),
    ({'load': {}, 'save': {'data/extra': {'extra': 'stuff'}}},),
    ({'load': {}, 'save': {'data/non-strings': {'properties': [0, 'p2']}}},),
    ({'load': {}, 'save': {'data/create-non-bool': {'create': 'abc'}}},),
    ({'load': {}, 'save': {'data/close-non-bool': {'close': 0}}},),
])
def test_invalid_args(self, case_references):
    with self.assertRaises(ValueError):
        wrapped_references = {
            'case_references': json.dumps(case_references)
        }
        _get_case_references(wrapped_references)
<|endoftext|>


<commit_before>from optparse import make_option

from django.core.management.base import BaseCommand

from corehq.apps.userreports.dbaccessors import get_report_configs_for_domain, get_datasources_for_domain

class Command(BaseCommand):
    args = "<existing_domain> <new_domain>"
    help = """Clone a domain and it's data"""

    option_list = BaseCommand.option_list + (
        make_option('--no-flags', action='store_false', dest='flags', default=True,
                    help="Don't process domain flags"),
        make_option('--no-fixtures', action='store_false', dest='fixtures', default=True,
                    help="Don't process fixtures"),
        make_option('--no-locations', action='store_false', dest='locations', default=True,
                    help="Don't process locations"),
        make_option('--no-products', action='store_false', dest='products', default=True,
                    help="Don't process products"),
        make_option('--no-ucr-or-apps', action='store_false', dest='ucr_apps', default=True,
                    help="Don't process UCR"),
    )

    def handle(self, *args, **options):
        existing_domain, new_domain = args
        self.clone_domain_and_settings(existing_domain, new_domain)
        if options['flags']:
            self.set_flags(existing_domain, new_domain)

        if options['fixtures']:
            self.copy_fixtures(existing_domain, new_domain)

        if options['locations']:
            self.copy_locations(existing_domain, new_domain)

        if options['products']:
            self.copy_products(existing_domain, new_domain)

        if options['ucr_apps']:
            report_map = self.copy_ucr_data(existing_domain, new_domain)
            self.copy_applications(existing_domain, new_domain, report_map)

    def clone_domain_and_settings(self, old_domain, new_domain):
        from corehq.apps.domain.models import Domain
        new_domain_obj = Domain.get_by_name(new_domain)
        if not new_domain_obj or raw_input(
                '{} domain already exists. Do you still want to continue? [y/n]'.format(new_domain)
            ).lower() == 'y':
            domain = Domain.get_by_name(old_domain)
            domain.name = new_domain
            self.save_couch_copy(domain)

            from corehq.apps.commtrack.models import CommtrackConfig
            commtrack_config = CommtrackConfig.for_domain(old_domain)
            self.save_couch_copy(commtrack_config, new_domain)

    def set_flags(self, old_domain, new_domain):
        from corehq.toggles import all_toggles, NAMESPACE_DOMAIN
        from corehq.feature_previews import all_previews
        from corehq.apps.hqwebapp.templatetags.hq_shared_tags import toggle_js_domain_cachebuster

        for toggle in all_toggles():
            if toggle.enabled(old_domain):
                toggle.set(new_domain, True, NAMESPACE_DOMAIN)

        for preview in all_previews():
            if preview.enabled(old_domain):
                preview.set(new_domain, True, NAMESPACE_DOMAIN)
                if preview.save_fn is not None:
                    preview.save_fn(new_domain, True)

        toggle_js_domain_cachebuster.clear(new_domain)

    def copy_fixtures(self, old_domain, new_domain):
        from corehq.apps.fixtures.models import FixtureDataItem
        from corehq.apps.fixtures.dbaccessors import get_fixture_data_types_in_domain

        fixture_types = get_fixture_data_types_in_domain(old_domain)
        for fixture_type in fixture_types:
            old_id, new_id = self.save_couch_copy(fixture_type, new_domain)
            for item in FixtureDataItem.by_data_type(old_domain, old_id):
                item.data_type_id = new_id
                self.save_couch_copy(item, new_domain)

        # TODO: FixtureOwnership - requires copying users & groups

    def copy_locations(self, old_domain, new_domain):
        from corehq.apps.locations.models import LocationType
        from corehq.apps.locations.models import Location

        location_types = LocationType.objects.filter(domain=old_domain)
        for location_type in location_types:
            self.save_sql_copy(location_type, new_domain)

        def copy_location_hierarchy(location, id_map):
            new_lineage = []
            for ancestor in location.lineage:
                try:
                    new_lineage.appen(id_map[ancestor])
                except KeyError:
                    self.stderr.write("Ancestor {} for location {} missing".format(location._id, ancestor))
            location.lineage = new_lineage

            old_id, new_id = self.save_couch_copy(location, new_domain)
            id_map[old_id] = new_id
            for child in location.children:
                copy_location_hierarchy(child, id_map)

        locations = Location.root_locations(old_domain)
        for location in locations:
            copy_location_hierarchy(location, {})

    def copy_products(self, old_domain, new_domain):
        from corehq.apps.products.models import Product
        from corehq.apps.programs.models import Program

        program_map = {}
        programs = Program.by_domain(old_domain)
        for program in programs:
            old_id, new_id = self.save_couch_copy(program)
            program_map[old_id] = new_id

        products = Product.by_domain(old_domain)
        for product in products:
            if product.program_id:
                try:
                    product.program_id = program_map[product.program_id]
                except:
                    self.stderr('Missing program {} for product {}'.format(product.program_id, product._id))
            self.save_couch_copy(product)

    def copy_applications(self, old_domain, new_domain, report_map):
        from corehq.apps.app_manager.dbaccessors import get_apps_in_domain
        from corehq.apps.app_manager.models import ReportModule
        apps = get_apps_in_domain(old_domain)
        for app in apps:
            for module in app.modules:
                if module.module_type == ReportModule.module_type:
                    for config in module.report_configs:
                        config.report_id = report_map[config.report_id]

            self.save_couch_copy(app, new_domain)

    def copy_ucr_data(self, old_domain, new_domain):
        datasource_map = self.copy_ucr_datasources(new_domain, old_domain)
        report_map = self.copy_ucr_reports(datasource_map, new_domain)
        return report_map

    def copy_ucr_reports(self, datasource_map, new_domain):
        report_map = {}
        reports = get_report_configs_for_domain('icds-sql')
        for report in reports:
            old_datasource_id = report.config_id
            try:
                report.config_id = datasource_map[old_datasource_id]
            except KeyError:
                pass  # datasource not found

            old_id, new_id = self.save_couch_copy(report, new_domain)
            report_map[old_id] = new_id
        return report_map

    def copy_ucr_datasources(self, new_domain, old_domain):
        datasource_map = {}
        datasources = get_datasources_for_domain(old_domain)
        for datasource in datasources:
            datasource.meta.build.finished = False
            datasource.meta.build.initiated = None

            old_id, new_id = self.save_couch_copy(datasource, new_domain)
            datasource_map[old_id] = new_id
        return datasource_map

    def save_couch_copy(self, doc, new_domain=None):
        old_id = doc._id
        del doc._id
        del doc['_rev']
        if new_domain:
            doc.domain = new_domain
        doc.save()
        new_id = doc._id
        self.log_copy(doc.doc_type, old_id, new_id)
        return old_id, new_id

    def save_sql_copy(self, model, new_domain):
        old_id = model.id
        model.domain = new_domain
        model.id = None
        model.save()
        new_id = model.id
        self.log_copy(model.__class__.__name__, old_id, new_id)
        return old_id, new_id

    def log_copy(self, name, old_id, new_id):
        self.stdout.write("{name}(id={old_id}) -> {name}(id={new_id})".format(
            name=name, old_id=old_id, new_id=new_id
        ))
<commit_msg>sql location type<commit_after>from optparse import make_option

from django.core.management.base import BaseCommand

from corehq.apps.userreports.dbaccessors import get_report_configs_for_domain, get_datasources_for_domain

class Command(BaseCommand):
    args = "<existing_domain> <new_domain>"
    help = """Clone a domain and it's data"""

    option_list = BaseCommand.option_list + (
        make_option('--no-flags', action='store_false', dest='flags', default=True,
                    help="Don't process domain flags"),
        make_option('--no-fixtures', action='store_false', dest='fixtures', default=True,
                    help="Don't process fixtures"),
        make_option('--no-locations', action='store_false', dest='locations', default=True,
                    help="Don't process locations"),
        make_option('--no-products', action='store_false', dest='products', default=True,
                    help="Don't process products"),
        make_option('--no-ucr-or-apps', action='store_false', dest='ucr_apps', default=True,
                    help="Don't process UCR"),
    )

    def handle(self, *args, **options):
        existing_domain, new_domain = args
        self.clone_domain_and_settings(existing_domain, new_domain)
        if options['flags']:
            self.set_flags(existing_domain, new_domain)

        if options['fixtures']:
            self.copy_fixtures(existing_domain, new_domain)

        if options['locations']:
            self.copy_locations(existing_domain, new_domain)

        if options['products']:
            self.copy_products(existing_domain, new_domain)

        if options['ucr_apps']:
            report_map = self.copy_ucr_data(existing_domain, new_domain)
            self.copy_applications(existing_domain, new_domain, report_map)

    def clone_domain_and_settings(self, old_domain, new_domain):
        from corehq.apps.domain.models import Domain
        new_domain_obj = Domain.get_by_name(new_domain)
        if not new_domain_obj or raw_input(
                '{} domain already exists. Do you still want to continue? [y/n]'.format(new_domain)
            ).lower() == 'y':
            domain = Domain.get_by_name(old_domain)
            domain.name = new_domain
            self.save_couch_copy(domain)

            from corehq.apps.commtrack.models import CommtrackConfig
            commtrack_config = CommtrackConfig.for_domain(old_domain)
            self.save_couch_copy(commtrack_config, new_domain)

    def set_flags(self, old_domain, new_domain):
        from corehq.toggles import all_toggles, NAMESPACE_DOMAIN
        from corehq.feature_previews import all_previews
        from corehq.apps.hqwebapp.templatetags.hq_shared_tags import toggle_js_domain_cachebuster

        for toggle in all_toggles():
            if toggle.enabled(old_domain):
                toggle.set(new_domain, True, NAMESPACE_DOMAIN)

        for preview in all_previews():
            if preview.enabled(old_domain):
                preview.set(new_domain, True, NAMESPACE_DOMAIN)
                if preview.save_fn is not None:
                    preview.save_fn(new_domain, True)

        toggle_js_domain_cachebuster.clear(new_domain)

    def copy_fixtures(self, old_domain, new_domain):
        from corehq.apps.fixtures.models import FixtureDataItem
        from corehq.apps.fixtures.dbaccessors import get_fixture_data_types_in_domain

        fixture_types = get_fixture_data_types_in_domain(old_domain)
        for fixture_type in fixture_types:
            old_id, new_id = self.save_couch_copy(fixture_type, new_domain)
            for item in FixtureDataItem.by_data_type(old_domain, old_id):
                item.data_type_id = new_id
                self.save_couch_copy(item, new_domain)

        # TODO: FixtureOwnership - requires copying users & groups

    def copy_locations(self, old_domain, new_domain):
        from corehq.apps.locations.models import LocationType
        from corehq.apps.locations.models import Location

        location_types = LocationType.objects.filter(domain=old_domain)
        for location_type in location_types:
            self.save_sql_copy(location_type, new_domain)

        def copy_location_hierarchy(location, id_map):
            new_lineage = []
            for ancestor in location.lineage:
                try:
                    new_lineage.appen(id_map[ancestor])
                except KeyError:
                    self.stderr.write("Ancestor {} for location {} missing".format(location._id, ancestor))
            location.lineage = new_lineage

            old_type_name = location.location_type_name
            location._sql_location_type = LocationType.objects.get(
                domain=new_domain,
                name=old_type_name,
            )
            old_id, new_id = self.save_couch_copy(location, new_domain)
            id_map[old_id] = new_id
            for child in location.children:
                copy_location_hierarchy(child, id_map)

        locations = Location.root_locations(old_domain)
        for location in locations:
            copy_location_hierarchy(location, {})

    def copy_products(self, old_domain, new_domain):
        from corehq.apps.products.models import Product
        from corehq.apps.programs.models import Program

        program_map = {}
        programs = Program.by_domain(old_domain)
        for program in programs:
            old_id, new_id = self.save_couch_copy(program)
            program_map[old_id] = new_id

        products = Product.by_domain(old_domain)
        for product in products:
            if product.program_id:
                try:
                    product.program_id = program_map[product.program_id]
                except:
                    self.stderr('Missing program {} for product {}'.format(product.program_id, product._id))
            self.save_couch_copy(product)

    def copy_applications(self, old_domain, new_domain, report_map):
        from corehq.apps.app_manager.dbaccessors import get_apps_in_domain
        from corehq.apps.app_manager.models import ReportModule
        apps = get_apps_in_domain(old_domain)
        for app in apps:
            for module in app.modules:
                if module.module_type == ReportModule.module_type:
                    for config in module.report_configs:
                        config.report_id = report_map[config.report_id]

            self.save_couch_copy(app, new_domain)

    def copy_ucr_data(self, old_domain, new_domain):
        datasource_map = self.copy_ucr_datasources(new_domain, old_domain)
        report_map = self.copy_ucr_reports(datasource_map, new_domain)
        return report_map

    def copy_ucr_reports(self, datasource_map, new_domain):
        report_map = {}
        reports = get_report_configs_for_domain('icds-sql')
        for report in reports:
            old_datasource_id = report.config_id
            try:
                report.config_id = datasource_map[old_datasource_id]
            except KeyError:
                pass  # datasource not found

            old_id, new_id = self.save_couch_copy(report, new_domain)
            report_map[old_id] = new_id
        return report_map

    def copy_ucr_datasources(self, new_domain, old_domain):
        datasource_map = {}
        datasources = get_datasources_for_domain(old_domain)
        for datasource in datasources:
            datasource.meta.build.finished = False
            datasource.meta.build.initiated = None

            old_id, new_id = self.save_couch_copy(datasource, new_domain)
            datasource_map[old_id] = new_id
        return datasource_map

    def save_couch_copy(self, doc, new_domain=None):
        old_id = doc._id
        del doc._id
        del doc['_rev']
        if new_domain:
            doc.domain = new_domain
        doc.save()
        new_id = doc._id
        self.log_copy(doc.doc_type, old_id, new_id)
        return old_id, new_id

    def save_sql_copy(self, model, new_domain):
        old_id = model.id
        model.domain = new_domain
        model.id = None
        model.save()
        new_id = model.id
        self.log_copy(model.__class__.__name__, old_id, new_id)
        return old_id, new_id

    def log_copy(self, name, old_id, new_id):
        self.stdout.write("{name}(id={old_id}) -> {name}(id={new_id})".format(
            name=name, old_id=old_id, new_id=new_id
        ))
<|endoftext|>


<commit_before>from django.core.management import BaseCommand, CommandError

from corehq.apps.app_manager.dbaccessors import get_build_doc_by_version, wrap_app
from corehq.apps.app_manager.models import import_app
from corehq.util.view_utils import absolute_reverse

class Command(BaseCommand):
    help = """
        For use when creating a new master app for ICDS.

        Makes a copy of a specific version of an application using the normal
        copy app workflow and then does a find and replace on report UUIDs in
        all form XML.

        This command is necessary until the ICDS apps are migrated to UCR v2.
    """

    def add_arguments(self, parser):
        parser.add_argument('app_id')
        parser.add_argument('version')
        parser.add_argument('source_domain')
        parser.add_argument('target_domain')
        parser.add_argument('new_name')
        parser.add_argument('--skip-dynamic-report-check', action='store_true', default=False)

    def handle(self, app_id, version, source_domain, target_domain, new_name, **options):
        if options['skip_dynamic_report_check']:
            message = """
                WARNING: You are skipping report-related safety checks.
                If your app uses mobile UCR and contains references to dynamic reports, it wwill break
                if those references do not use aliases to reference the correct report ids.
                Do you wish to proceed?
            """
            response = input('{} [y/N]'.format(message)).lower()
            if response != 'y':
                raise CommandError('abort')

        old_app = get_app_by_version(source_domain, app_id, version)
        new_app = import_app(old_app.to_json(), target_domain, source_properties={'name': new_name},
                             check_all_reports=not(options['skip_dynamic_report_check']))

        find_and_replace_report_ids(old_app, new_app)

        new_app.save()

        print("App succesfully copied, you can view it at\n{}".format(
            absolute_reverse('view_app', args=[target_domain, new_app.get_id])
        ))

        print("""
            Next, make a build of the new app and then run copy_icds_app_2_of_2
            for all linked apps that use this app as a master app.
        """)

def get_app_by_version(domain, app_id, version):
    app = get_build_doc_by_version(domain, app_id, version)
    if not app:
        raise Exception("No app found with id '{}' and version '{}', on '{}'"
                        .format(app_id, version, domain))
    return wrap_app(app)

def find_and_replace_report_ids(old_app, new_app):
    old_to_new = get_old_to_new_config_ids(old_app, new_app)
    for form in new_app.get_forms():
        if form.form_type != 'shadow_form':
            for old_id, new_id in old_to_new:
                form.source = form.source.replace(old_id, new_id)

def get_old_to_new_config_ids(old_app, new_app):
    return [
        (old_config.uuid, new_config.uuid)
        for old_module, new_module in zip(old_app.get_report_modules(), new_app.get_report_modules())
        for old_config, new_config in zip(old_module.report_configs, new_module.report_configs)
    ]
<commit_msg>Fixed typo [ci skip]<commit_after>from django.core.management import BaseCommand, CommandError

from corehq.apps.app_manager.dbaccessors import get_build_doc_by_version, wrap_app
from corehq.apps.app_manager.models import import_app
from corehq.util.view_utils import absolute_reverse

class Command(BaseCommand):
    help = """
        For use when creating a new master app for ICDS.

        Makes a copy of a specific version of an application using the normal
        copy app workflow and then does a find and replace on report UUIDs in
        all form XML.

        This command is necessary until the ICDS apps are migrated to UCR v2.
    """

    def add_arguments(self, parser):
        parser.add_argument('app_id')
        parser.add_argument('version')
        parser.add_argument('source_domain')
        parser.add_argument('target_domain')
        parser.add_argument('new_name')
        parser.add_argument('--skip-dynamic-report-check', action='store_true', default=False)

    def handle(self, app_id, version, source_domain, target_domain, new_name, **options):
        if options['skip_dynamic_report_check']:
            message = """
                WARNING: You are skipping report-related safety checks.
                If your app uses mobile UCR and contains references to dynamic reports, it will break
                if those references do not use aliases to reference the correct report ids.
                Do you wish to proceed?
            """
            response = input('{} [y/N]'.format(message)).lower()
            if response != 'y':
                raise CommandError('abort')

        old_app = get_app_by_version(source_domain, app_id, version)
        new_app = import_app(old_app.to_json(), target_domain, source_properties={'name': new_name},
                             check_all_reports=not(options['skip_dynamic_report_check']))

        find_and_replace_report_ids(old_app, new_app)

        new_app.save()

        print("App succesfully copied, you can view it at\n{}".format(
            absolute_reverse('view_app', args=[target_domain, new_app.get_id])
        ))

        print("""
            Next, make a build of the new app and then run copy_icds_app_2_of_2
            for all linked apps that use this app as a master app.
        """)

def get_app_by_version(domain, app_id, version):
    app = get_build_doc_by_version(domain, app_id, version)
    if not app:
        raise Exception("No app found with id '{}' and version '{}', on '{}'"
                        .format(app_id, version, domain))
    return wrap_app(app)

def find_and_replace_report_ids(old_app, new_app):
    old_to_new = get_old_to_new_config_ids(old_app, new_app)
    for form in new_app.get_forms():
        if form.form_type != 'shadow_form':
            for old_id, new_id in old_to_new:
                form.source = form.source.replace(old_id, new_id)

def get_old_to_new_config_ids(old_app, new_app):
    return [
        (old_config.uuid, new_config.uuid)
        for old_module, new_module in zip(old_app.get_report_modules(), new_app.get_report_modules())
        for old_config, new_config in zip(old_module.report_configs, new_module.report_configs)
    ]
<|endoftext|>


<commit_before>from collections import OrderedDict, defaultdict
from datetime import datetime

from dateutil.relativedelta import relativedelta
from dateutil.rrule import rrule, MONTHLY
from django.db.models.aggregates import Sum
from django.utils.translation import ugettext as _

from custom.icds_reports.cache import icds_quickcache
from custom.icds_reports.const import LocationTypes, ChartColors, MapColors
from custom.icds_reports.messages import percent_lactating_women_enrolled_help_text
from custom.icds_reports.models import AggCcsRecordMonthly
from custom.icds_reports.utils import apply_exclude, indian_formatted_number, get_child_locations

@icds_quickcache(['domain', 'config', 'loc_level', 'show_test'], timeout=30 * 60)
def get_lactating_enrolled_women_data_map(domain, config, loc_level, show_test=False):

    def get_data_for(filters):
        filters['month'] = datetime(*filters['month'])
        queryset = AggCcsRecordMonthly.objects.filter(
            **filters
        ).values(
            '%s_name' % loc_level, '%s_map_location_name' % loc_level
        ).annotate(
            valid=Sum('lactating'),
            all=Sum('lactating_all'),
        ).order_by('%s_name' % loc_level, '%s_map_location_name' % loc_level)
        if not show_test:
            queryset = apply_exclude(domain, queryset)
        return queryset

    data_for_map = defaultdict(lambda: {
        'valid': 0,
        'all': 0,
        'original_name': [],
        'fillKey': 'Women'
    })
    average = []
    total_valid = 0
    total = 0
    for row in get_data_for(config):
        valid = row['valid'] or 0
        all_lactating = row['all'] or 0
        name = row['%s_name' % loc_level]
        on_map_name = row['%s_map_location_name' % loc_level] or name

        average.append(valid)

        total_valid += valid
        total += all_lactating

        data_for_map[on_map_name]['valid'] += valid
        data_for_map[on_map_name]['all'] += all_lactating
        data_for_map[on_map_name]['original_name'].append(name)

    fills = OrderedDict()
    fills.update({'Women': MapColors.BLUE})
    fills.update({'defaultFill': MapColors.GREY})

    return {
        "slug": "lactating_enrolled_women",
        "label": "",
        "fills": fills,
        "rightLegend": {
            "average": '%.2f' % (total_valid * 100 / float(total or 1)),
            "info": percent_lactating_women_enrolled_help_text(),
            "extended_info": [
                {
                    'indicator': 'Number of lactating women who are enrolled for Anganwadi Services:',
                    'value': indian_formatted_number(total_valid)
                },
                {
                    'indicator': (
                        'Total number of lactating women who are registered:'
                    ),
                    'value': indian_formatted_number(total)
                },
                {
                    'indicator': (
                        'Percentage of registered lactating women who are enrolled for Anganwadi Services:'
                    ),
                    'value': '%.2f%%' % (total_valid * 100 / float(total or 1))
                }
            ]
        },
        "data": dict(data_for_map),
    }

@icds_quickcache(['domain', 'config', 'loc_level', 'location_id', 'show_test'], timeout=30 * 60)
def get_lactating_enrolled_women_sector_data(domain, config, loc_level, location_id, show_test=False):
    group_by = ['%s_name' % loc_level]

    config['month'] = datetime(*config['month'])
    data = AggCcsRecordMonthly.objects.filter(
        **config
    ).values(
        *group_by
    ).annotate(
        valid=Sum('lactating'),
        all=Sum('lactating_all')
    ).order_by('%s_name' % loc_level)

    if not show_test:
        data = apply_exclude(domain, data)

    chart_data = {
        'blue': []
    }

    tooltips_data = defaultdict(lambda: {
        'valid': 0,
        'all': 0
    })

    loc_children = get_child_locations(domain, location_id, show_test)
    result_set = set()

    for row in data:
        valid = row['valid'] or 0
        all_lactating = row['all'] or 0
        name = row['%s_name' % loc_level]
        result_set.add(name)

        row_values = {
            'valid': valid,
            'all': all_lactating
        }
        for prop, value in row_values).items(:
            tooltips_data[name][prop] += value

        chart_data['blue'].append([
            name, valid
        ])

    for sql_location in loc_children:
        if sql_location.name not in result_set:
            chart_data['blue'].append([sql_location.name, 0])

    chart_data['blue'] = sorted(chart_data['blue'])

    return {
        "tooltips_data": dict(tooltips_data),
        "format": "number",
        "info": percent_lactating_women_enrolled_help_text(),
        "chart_data": [
            {
                "values": chart_data['blue'],
                "key": "",
                "strokeWidth": 2,
                "classed": "dashed",
                "color": MapColors.BLUE
            }
        ]
    }

# @icds_quickcache(['domain', 'config', 'loc_level', 'show_test'], timeout=30 * 60)
def get_lactating_enrolled_data_chart(domain, config, loc_level, show_test=False):
    month = datetime(*config['month'])
    three_before = datetime(*config['month']) - relativedelta(months=3)

    config['month__range'] = (three_before, month)
    del config['month']

    chart_data = AggCcsRecordMonthly.objects.filter(
        **config
    ).values(
        'month', '%s_name' % loc_level
    ).annotate(
        valid=Sum('lactating'),
        all=Sum('lactating_all')
    ).order_by('month')

    if not show_test:
        chart_data = apply_exclude(domain, chart_data)

    data = {
        'blue': OrderedDict()
    }

    dates = [dt for dt in rrule(MONTHLY, dtstart=three_before, until=month)]

    for date in dates:
        miliseconds = int(date.strftime("%s")) * 1000
        data['blue'][miliseconds] = {'y': 0, 'all': 0}

    best_worst = {}
    for row in chart_data:
        date = row['month']
        valid = row['valid'] or 0
        all_lactating = row['all'] or 0
        location = row['%s_name' % loc_level]

        if date.month == month.month:
            if location in best_worst:
                best_worst[location].append(valid)
            else:
                best_worst[location] = [valid]

        date_in_miliseconds = int(date.strftime("%s")) * 1000

        data['blue'][date_in_miliseconds]['y'] += valid
        data['blue'][date_in_miliseconds]['all'] += all_lactating

    all_locations = [
        {
            'loc_name': key,
            'value': sum(value) / len(value)
        }
        for key, value in best_worst.items()
    ]
    all_locations_sorted_by_name = sorted(all_locations, key=lambda x: x['loc_name'])
    all_locations_sorted_by_value_and_name = sorted(
        all_locations_sorted_by_name, key=lambda x: x['value'], reverse=True)

    return {
        "chart_data": [
            {
                "values": [
                    {
                        'x': key,
                        'y': value['y'],
                        'all': value['all']
                    } for key, value in data['blue'].items()
                ],
                "key": "Total number of lactating women who are enrolled for Anganwadi Services",
                "strokeWidth": 2,
                "classed": "dashed",
                "color": ChartColors.BLUE
            }
        ],
        "all_locations": all_locations_sorted_by_value_and_name,
        "top_five": all_locations_sorted_by_value_and_name[:5],
        "bottom_five": all_locations_sorted_by_value_and_name[-5:],
        "location_type": loc_level.title() if loc_level != LocationTypes.SUPERVISOR else 'Sector'
    }
<commit_msg>Fixed syntax error<commit_after>from collections import OrderedDict, defaultdict
from datetime import datetime

from dateutil.relativedelta import relativedelta
from dateutil.rrule import rrule, MONTHLY
from django.db.models.aggregates import Sum
from django.utils.translation import ugettext as _

from custom.icds_reports.cache import icds_quickcache
from custom.icds_reports.const import LocationTypes, ChartColors, MapColors
from custom.icds_reports.messages import percent_lactating_women_enrolled_help_text
from custom.icds_reports.models import AggCcsRecordMonthly
from custom.icds_reports.utils import apply_exclude, indian_formatted_number, get_child_locations

@icds_quickcache(['domain', 'config', 'loc_level', 'show_test'], timeout=30 * 60)
def get_lactating_enrolled_women_data_map(domain, config, loc_level, show_test=False):

    def get_data_for(filters):
        filters['month'] = datetime(*filters['month'])
        queryset = AggCcsRecordMonthly.objects.filter(
            **filters
        ).values(
            '%s_name' % loc_level, '%s_map_location_name' % loc_level
        ).annotate(
            valid=Sum('lactating'),
            all=Sum('lactating_all'),
        ).order_by('%s_name' % loc_level, '%s_map_location_name' % loc_level)
        if not show_test:
            queryset = apply_exclude(domain, queryset)
        return queryset

    data_for_map = defaultdict(lambda: {
        'valid': 0,
        'all': 0,
        'original_name': [],
        'fillKey': 'Women'
    })
    average = []
    total_valid = 0
    total = 0
    for row in get_data_for(config):
        valid = row['valid'] or 0
        all_lactating = row['all'] or 0
        name = row['%s_name' % loc_level]
        on_map_name = row['%s_map_location_name' % loc_level] or name

        average.append(valid)

        total_valid += valid
        total += all_lactating

        data_for_map[on_map_name]['valid'] += valid
        data_for_map[on_map_name]['all'] += all_lactating
        data_for_map[on_map_name]['original_name'].append(name)

    fills = OrderedDict()
    fills.update({'Women': MapColors.BLUE})
    fills.update({'defaultFill': MapColors.GREY})

    return {
        "slug": "lactating_enrolled_women",
        "label": "",
        "fills": fills,
        "rightLegend": {
            "average": '%.2f' % (total_valid * 100 / float(total or 1)),
            "info": percent_lactating_women_enrolled_help_text(),
            "extended_info": [
                {
                    'indicator': 'Number of lactating women who are enrolled for Anganwadi Services:',
                    'value': indian_formatted_number(total_valid)
                },
                {
                    'indicator': (
                        'Total number of lactating women who are registered:'
                    ),
                    'value': indian_formatted_number(total)
                },
                {
                    'indicator': (
                        'Percentage of registered lactating women who are enrolled for Anganwadi Services:'
                    ),
                    'value': '%.2f%%' % (total_valid * 100 / float(total or 1))
                }
            ]
        },
        "data": dict(data_for_map),
    }

@icds_quickcache(['domain', 'config', 'loc_level', 'location_id', 'show_test'], timeout=30 * 60)
def get_lactating_enrolled_women_sector_data(domain, config, loc_level, location_id, show_test=False):
    group_by = ['%s_name' % loc_level]

    config['month'] = datetime(*config['month'])
    data = AggCcsRecordMonthly.objects.filter(
        **config
    ).values(
        *group_by
    ).annotate(
        valid=Sum('lactating'),
        all=Sum('lactating_all')
    ).order_by('%s_name' % loc_level)

    if not show_test:
        data = apply_exclude(domain, data)

    chart_data = {
        'blue': []
    }

    tooltips_data = defaultdict(lambda: {
        'valid': 0,
        'all': 0
    })

    loc_children = get_child_locations(domain, location_id, show_test)
    result_set = set()

    for row in data:
        valid = row['valid'] or 0
        all_lactating = row['all'] or 0
        name = row['%s_name' % loc_level]
        result_set.add(name)

        row_values = {
            'valid': valid,
            'all': all_lactating
        }
        for prop, value in row_values.items():
            tooltips_data[name][prop] += value

        chart_data['blue'].append([
            name, valid
        ])

    for sql_location in loc_children:
        if sql_location.name not in result_set:
            chart_data['blue'].append([sql_location.name, 0])

    chart_data['blue'] = sorted(chart_data['blue'])

    return {
        "tooltips_data": dict(tooltips_data),
        "format": "number",
        "info": percent_lactating_women_enrolled_help_text(),
        "chart_data": [
            {
                "values": chart_data['blue'],
                "key": "",
                "strokeWidth": 2,
                "classed": "dashed",
                "color": MapColors.BLUE
            }
        ]
    }

# @icds_quickcache(['domain', 'config', 'loc_level', 'show_test'], timeout=30 * 60)
def get_lactating_enrolled_data_chart(domain, config, loc_level, show_test=False):
    month = datetime(*config['month'])
    three_before = datetime(*config['month']) - relativedelta(months=3)

    config['month__range'] = (three_before, month)
    del config['month']

    chart_data = AggCcsRecordMonthly.objects.filter(
        **config
    ).values(
        'month', '%s_name' % loc_level
    ).annotate(
        valid=Sum('lactating'),
        all=Sum('lactating_all')
    ).order_by('month')

    if not show_test:
        chart_data = apply_exclude(domain, chart_data)

    data = {
        'blue': OrderedDict()
    }

    dates = [dt for dt in rrule(MONTHLY, dtstart=three_before, until=month)]

    for date in dates:
        miliseconds = int(date.strftime("%s")) * 1000
        data['blue'][miliseconds] = {'y': 0, 'all': 0}

    best_worst = {}
    for row in chart_data:
        date = row['month']
        valid = row['valid'] or 0
        all_lactating = row['all'] or 0
        location = row['%s_name' % loc_level]

        if date.month == month.month:
            if location in best_worst:
                best_worst[location].append(valid)
            else:
                best_worst[location] = [valid]

        date_in_miliseconds = int(date.strftime("%s")) * 1000

        data['blue'][date_in_miliseconds]['y'] += valid
        data['blue'][date_in_miliseconds]['all'] += all_lactating

    all_locations = [
        {
            'loc_name': key,
            'value': sum(value) / len(value)
        }
        for key, value in best_worst.items()
    ]
    all_locations_sorted_by_name = sorted(all_locations, key=lambda x: x['loc_name'])
    all_locations_sorted_by_value_and_name = sorted(
        all_locations_sorted_by_name, key=lambda x: x['value'], reverse=True)

    return {
        "chart_data": [
            {
                "values": [
                    {
                        'x': key,
                        'y': value['y'],
                        'all': value['all']
                    } for key, value in data['blue'].items()
                ],
                "key": "Total number of lactating women who are enrolled for Anganwadi Services",
                "strokeWidth": 2,
                "classed": "dashed",
                "color": ChartColors.BLUE
            }
        ],
        "all_locations": all_locations_sorted_by_value_and_name,
        "top_five": all_locations_sorted_by_value_and_name[:5],
        "bottom_five": all_locations_sorted_by_value_and_name[-5:],
        "location_type": loc_level.title() if loc_level != LocationTypes.SUPERVISOR else 'Sector'
    }
<|endoftext|>


<commit_before>"""
Copyright 2013 Steven Diamond

This file is part of CVXPY.

CVXPY is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

CVXPY is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with CVXPY.  If not, see <http://www.gnu.org/licenses/>.
"""

from .conic_solver import ConicSolver
import cvxpy.settings as s
from cvxpy.reductions.solvers.conic_solvers.ecos_conif import (
                                                    dims_to_solver_dict, ECOS)

class ECOS_BB(ECOS):
    """An interface for the ECOS BB solver.
    """

    # Solver capabilities.
    MIP_CAPABLE = True

    # Exit flags from ECOS_BB
    # ECOS_BB found optimal solution.
    # MI_OPTIMAL_SOLN (ECOS_OPTIMAL)
    # ECOS_BB proved problem is infeasible.
    # MI_INFEASIBLE (ECOS_PINF)
    # ECOS_BB proved problem is unbounded.
    # MI_UNBOUNDED (ECOS_DINF)
    # ECOS_BB hit maximum iterations but a feasible solution was found and
    # the best seen feasible solution was returned.
    # MI_MAXITER_FEASIBLE_SOLN (ECOS_OPTIMAL + ECOS_INACC_OFFSET)
    # ECOS_BB hit maximum iterations without finding a feasible solution.
    # MI_MAXITER_NO_SOLN (ECOS_PINF + ECOS_INACC_OFFSET)
    # ECOS_BB hit maximum iterations without finding a feasible solution
    #   that was unbounded.
    # MI_MAXITER_UNBOUNDED (ECOS_DINF + ECOS_INACC_OFFSET)

    def name(self):
        """The name of the solver.
        """
        return s.ECOS_BB

    def apply(self, problem):
        data, inv_data = super(ECOS_BB, self).apply(problem)
        # Because the problem variable is single dimensional, every
        # boolean/integer index has length one.
        var = problem.variables()[0]
        data[s.BOOL_IDX] = [t[0] for t in var.boolean_idx]
        data[s.INT_IDX] = [t[0] for t in var.integer_idx]
        return data, inv_data

    def solve_via_data(self, data, warm_start, verbose, solver_opts):
        print data
        import ecos
        cones = dims_to_solver_dict(data[ConicSolver.DIMS])
        # Default verbose to false for BB wrapper.
        mi_verbose = solver_opts.get('mi_verbose', False)
        solution = ecos.solve(data[s.C], data[s.G], data[s.H],
                              cones, data[s.A], data[s.B],
                              verbose=verbose,
                              mi_verbose=mi_verbose,
                              bool_vars_idx=data[s.BOOL_IDX],
                              int_vars_idx=data[s.INT_IDX],
                              **solver_opts)
        return solution
<commit_msg>remove print statement<commit_after>"""
Copyright 2013 Steven Diamond

This file is part of CVXPY.

CVXPY is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

CVXPY is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with CVXPY.  If not, see <http://www.gnu.org/licenses/>.
"""

from .conic_solver import ConicSolver
import cvxpy.settings as s
from cvxpy.reductions.solvers.conic_solvers.ecos_conif import (
                                                    dims_to_solver_dict, ECOS)

class ECOS_BB(ECOS):
    """An interface for the ECOS BB solver.
    """

    # Solver capabilities.
    MIP_CAPABLE = True

    # Exit flags from ECOS_BB
    # ECOS_BB found optimal solution.
    # MI_OPTIMAL_SOLN (ECOS_OPTIMAL)
    # ECOS_BB proved problem is infeasible.
    # MI_INFEASIBLE (ECOS_PINF)
    # ECOS_BB proved problem is unbounded.
    # MI_UNBOUNDED (ECOS_DINF)
    # ECOS_BB hit maximum iterations but a feasible solution was found and
    # the best seen feasible solution was returned.
    # MI_MAXITER_FEASIBLE_SOLN (ECOS_OPTIMAL + ECOS_INACC_OFFSET)
    # ECOS_BB hit maximum iterations without finding a feasible solution.
    # MI_MAXITER_NO_SOLN (ECOS_PINF + ECOS_INACC_OFFSET)
    # ECOS_BB hit maximum iterations without finding a feasible solution
    #   that was unbounded.
    # MI_MAXITER_UNBOUNDED (ECOS_DINF + ECOS_INACC_OFFSET)

    def name(self):
        """The name of the solver.
        """
        return s.ECOS_BB

    def apply(self, problem):
        data, inv_data = super(ECOS_BB, self).apply(problem)
        # Because the problem variable is single dimensional, every
        # boolean/integer index has length one.
        var = problem.variables()[0]
        data[s.BOOL_IDX] = [t[0] for t in var.boolean_idx]
        data[s.INT_IDX] = [t[0] for t in var.integer_idx]
        return data, inv_data

    def solve_via_data(self, data, warm_start, verbose, solver_opts):
        import ecos
        cones = dims_to_solver_dict(data[ConicSolver.DIMS])
        # Default verbose to false for BB wrapper.
        mi_verbose = solver_opts.get('mi_verbose', False)
        solution = ecos.solve(data[s.C], data[s.G], data[s.H],
                              cones, data[s.A], data[s.B],
                              verbose=verbose,
                              mi_verbose=mi_verbose,
                              bool_vars_idx=data[s.BOOL_IDX],
                              int_vars_idx=data[s.INT_IDX],
                              **solver_opts)
        return solution
<|endoftext|>