<commit_before>from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals
import os
from datetime import datetime

import mock
import postgres_copy
import six
import sqlalchemy
import csv342 as csv

from django.conf import settings
from django.test.utils import override_settings
from django.test.testcases import TestCase

from corehq.apps.domain.models import Domain
from corehq.apps.domain.shortcuts import create_domain
from corehq.apps.locations.models import SQLLocation, LocationType
from corehq.apps.userreports.models import StaticDataSourceConfiguration
from corehq.apps.userreports.util import get_indicator_adapter, get_table_name
from corehq.sql_db.connections import connection_manager, ICDS_UCR_ENGINE_ID
from corehq.sql_db.routers import db_for_read_write
from custom.icds_reports.const import AGG_CCS_RECORD_DELIVERY_TABLE, AGG_COMP_FEEDING_TABLE, \
    AGG_GROWTH_MONITORING_TABLE, AGG_CCS_RECORD_BP_TABLE, AGG_CHILD_HEALTH_PNC_TABLE, AGG_CCS_RECORD_PNC_TABLE, \
    AGG_CCS_RECORD_THR_TABLE, AGG_DAILY_FEEDING_TABLE, AGG_CCS_RECORD_CF_TABLE, AGG_CHILD_HEALTH_THR_TABLE

from custom.icds_reports.tasks import (
    move_ucr_data_into_aggregation_tables,
    build_incentive_report,
    _aggregate_child_health_pnc_forms,
    _aggregate_bp_forms,
    _aggregate_gm_forms)
from custom.icds_reports.models.aggregate import AggregateCcsRecordDeliveryForms
from io import open
from six.moves import range
from six.moves import zip

FILE_NAME_TO_TABLE_MAPPING = {
    'awc_mgmt': get_table_name('icds-cas', 'static-awc_mgt_forms'),
    'ccs_monthly': get_table_name('icds-cas', 'static-ccs_record_cases_monthly_tableau_v2'),
    "ccs_cases": get_table_name('icds-cas', 'static-ccs_record_cases'),
    'child_cases': get_table_name('icds-cas', 'static-child_health_cases'),
    'daily_feeding': get_table_name('icds-cas', 'static-daily_feeding_forms'),
    'household_cases': get_table_name('icds-cas', 'static-household_cases'),
    'infrastructure': get_table_name('icds-cas', 'static-infrastructure_form'),
    'infrastructure_v2': get_table_name('icds-cas', 'static-infrastructure_form_v2'),
    'location_ucr': get_table_name('icds-cas', 'static-awc_location'),
    'person_cases': get_table_name('icds-cas', 'static-person_cases_v3'),
    'usage': get_table_name('icds-cas', 'static-usage_forms'),
    'vhnd': get_table_name('icds-cas', 'static-vhnd_form'),
    'complementary_feeding': get_table_name('icds-cas', 'static-complementary_feeding_forms'),
    'aww_user': get_table_name('icds-cas', 'static-commcare_user_cases'),
    'child_tasks': get_table_name('icds-cas', 'static-child_tasks_cases'),
    'pregnant_tasks': get_table_name('icds-cas', 'static-pregnant-tasks_cases'),
    'thr_form': get_table_name('icds-cas', 'static-dashboard_thr_forms'),
    'gm_form': get_table_name('icds-cas', 'static-dashboard_growth_monitoring_forms'),
    'pnc_forms': get_table_name('icds-cas', 'static-postnatal_care_forms'),
    'dashboard_daily_feeding': get_table_name('icds-cas', 'dashboard_child_health_daily_feeding_forms'),
    'ls_awc_mgt': get_table_name('icds-cas', 'static-awc_mgt_forms'),
    'ls_home_vists': get_table_name('icds-cas', 'static-ls_home_visit_forms_filled'),
    'ls_vhnd': get_table_name('icds-cas', 'static-ls_vhnd_form'),
    'cbe_form': get_table_name('icds-cas', 'static-cbe_form'),
    'agg_awc': 'agg_awc',
    'birth_preparedness': get_table_name('icds-cas', 'static-dashboard_birth_preparedness_forms'),
    'delivery_form': get_table_name('icds-cas', 'static-dashboard_delivery_forms'),
}

OUTPUT_PATH = os.path.join(os.path.dirname(__file__), 'outputs')

_use_citus = override_settings(ICDS_USE_CITS=True)

def setUpModule():
    if settings.USE_PARTITIONED_DATABASE:
        print('============= WARNING: not running test setup because settings.USE_PARTITIONED_DATABASE is True.')
        return

    _call_center_domain_mock = mock.patch(
        'corehq.apps.callcenter.data_source.call_center_data_source_configuration_provider'
    )
    _call_center_domain_mock.start()
    _use_citus.enable()

    domain = create_domain('icds-cas')
    SQLLocation.objects.all().delete()
    LocationType.objects.all().delete()
    location_type = LocationType.objects.create(
        domain=domain.name,
        name='block',
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='b1',
        location_id='b1',
        location_type=location_type
    )

    state_location_type = LocationType.objects.create(
        domain=domain.name,
        name='state',
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='st1',
        location_id='st1',
        location_type=state_location_type
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='st2',
        location_id='st2',
        location_type=state_location_type
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='st3',
        location_id='st3',
        location_type=state_location_type
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='st4',
        location_id='st4',
        location_type=state_location_type
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='st5',
        location_id='st5',
        location_type=state_location_type
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='st6',
        location_id='st6',
        location_type=state_location_type
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='st7',
        location_id='st7',
        location_type=state_location_type
    )

    awc_location_type = LocationType.objects.create(
        domain=domain.name,
        name='awc',
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='a7',
        location_id='a7',
        location_type=awc_location_type
    )

    with override_settings(SERVER_ENVIRONMENT='icds'):
        configs = StaticDataSourceConfiguration.by_domain('icds-cas')
        adapters = [get_indicator_adapter(config) for config in configs]

        for adapter in adapters:
            try:
                adapter.drop_table()
            except Exception:
                pass
            adapter.build_table()

        engine = connection_manager.get_engine(ICDS_UCR_ENGINE_ID)
        metadata = sqlalchemy.MetaData(bind=engine)
        metadata.reflect(bind=engine, extend_existing=True)
        path = os.path.join(os.path.dirname(__file__), 'fixtures')
        for file_name in os.listdir(path):
            with open(os.path.join(path, file_name), encoding='utf-8') as f:
                table_name = FILE_NAME_TO_TABLE_MAPPING[file_name[:-4]]
                table = metadata.tables[table_name]
                if not table_name.startswith('icds_dashboard_'):
                    columns = [
                        '"{}"'.format(c.strip())  # quote to preserve case
                        for c in f.readline().split(',')
                    ]
                    postgres_copy.copy_from(
                        f, table, engine, format='csv' if six.PY3 else b'csv',
                        null='' if six.PY3 else b'', columns=columns
                    )

        # move to migrations
        distribute = [
            (AGG_CCS_RECORD_DELIVERY_TABLE, 'supervisor_id'),
            (AGG_COMP_FEEDING_TABLE, 'supervisor_id'),
            (AGG_CCS_RECORD_CF_TABLE, 'supervisor_id'),
            (AGG_CHILD_HEALTH_THR_TABLE, 'supervisor_id'),
            (AGG_GROWTH_MONITORING_TABLE, 'supervisor_id'),
            (AGG_CHILD_HEALTH_PNC_TABLE, 'supervisor_id'),
            (AGG_CCS_RECORD_PNC_TABLE, 'supervisor_id'),
            (AGG_CCS_RECORD_BP_TABLE, 'supervisor_id'),
            (AGG_CCS_RECORD_THR_TABLE, 'supervisor_id'),
            (AGG_DAILY_FEEDING_TABLE, 'supervisor_id'),
            ('child_health_monthly', 'supervisor_id'),
            ('ccs_record_monthly', 'supervisor_id'),
            ('daily_attendance', 'supervisor_id'),
        ]

        reference = [
            'awc_location',
            'icds_months'
        ]

        for table, col in distribute:
            with engine.begin() as conn:
                res = conn.execute(
                    """
                    SELECT c.relname AS child
                    FROM
                        pg_inherits JOIN pg_class AS c ON (inhrelid=c.oid)
                        JOIN pg_class as p ON (inhparent=p.oid)
                        where p.relname = %s;
                    """,
                    table
                )
                for child in [row.child for row in res]:
                    # only need this because of reusedb if testing on master and this branch
                    conn.execute('drop table if exists "{}"'.format(child))
                res = conn.execute("""
                    select 1 from pg_dist_partition
                    where partmethod = 'h' and logicalrelid = %s::regclass
                """, table)
                if not list(res):
                    conn.execute("select create_distributed_table(%s, %s)", [table, col])

        for table in reference:
            with engine.begin() as conn:
                res = conn.execute("""
                    select 1 from pg_dist_partition
                    where partmethod = 'n' and logicalrelid = %s::regclass
                """, table)
                if not list(res):
                    conn.execute("select create_reference_table(%s)", [table])

        for state_id in ('st1', 'st2'):
            _aggregate_child_health_pnc_forms(state_id, datetime(2017, 3, 31))
            _aggregate_gm_forms(state_id, datetime(2017, 3, 31))
            _aggregate_bp_forms(state_id, datetime(2017, 3, 31))

        try:
            move_ucr_data_into_aggregation_tables(datetime(2017, 5, 28), intervals=2)
            build_incentive_report(agg_date=datetime(2017, 5, 28))
        except AssertionError as e:
            # we always use soft assert to email when the aggregation has completed
            if "Aggregation completed" not in str(e):
                print(e)
                tearDownModule()
                raise
        except Exception as e:
            print(e)
            tearDownModule()
            raise
        finally:
            _call_center_domain_mock.stop()

def tearDownModule():
    if settings.USE_PARTITIONED_DATABASE:
        return

    _call_center_domain_mock = mock.patch(
        'corehq.apps.callcenter.data_source.call_center_data_source_configuration_provider'
    )
    _call_center_domain_mock.start()
    with override_settings(SERVER_ENVIRONMENT='icds'):
        configs = StaticDataSourceConfiguration.by_domain('icds-cas')
        adapters = [get_indicator_adapter(config) for config in configs]
        for adapter in adapters:
            if adapter.config.table_id == 'static-child_health_cases':
                # hack because this is in a migration
                adapter.clear_table()
                continue
            adapter.drop_table()

        engine = connection_manager.get_engine(ICDS_UCR_ENGINE_ID)
        with engine.begin() as connection:
            metadata = sqlalchemy.MetaData(bind=engine)
            metadata.reflect(bind=engine, extend_existing=True)
            table = metadata.tables['ucr_table_name_mapping']
            delete = table.delete()
            connection.execute(delete)
    LocationType.objects.filter(domain='icds-cas').delete()
    SQLLocation.objects.filter(domain='icds-cas').delete()

    Domain.get_by_name('icds-cas').delete()
    _use_citus.disable()
    _call_center_domain_mock.stop()

class CSVTestCase(TestCase):

    def _load_csv(self, path):
        with open(path, encoding='utf-8') as f:
            csv_data = list(csv.reader(f))
            headers = csv_data[0]
            for row_count, row in enumerate(csv_data):
                csv_data[row_count] = dict(zip(headers, row))
        return csv_data[1:]

    def _fasterAssertListEqual(self, list1, list2):
        if len(list1) != len(list2):
            self.fail('Lists are not equal')

        messages = []

        for idx in range(len(list1)):
            dict1 = list1[idx]
            dict2 = list2[idx]

            differences = set()

            for key in dict1.keys():
                if key != 'id':
                    if isinstance(dict1[key], six.text_type):
                        value1 = dict1[key]
                    elif isinstance(dict1[key], list):
                        value1 = str(dict1[key])
                    else:
                        value1 = dict1[key].decode('utf-8')
                    value1 = value1.replace('\r\n', '\n')
                    value2 = dict2.get(key, '').replace('\r\n', '\n')
                    if value1 != value2:
                        differences.add(key)

            if differences:
                if self.always_include_columns:
                    differences |= self.always_include_columns
                messages.append("""
                    Actual and expected row {} are not the same
                    Actual:   {}
                    Expected: {}
                """.format(
                    idx + 1,
                    ', '.join(['{}: {}'.format(
                        difference, str(dict1[difference])) for difference in differences]
                    ),
                    ', '.join(['{}: {}'.format(
                        difference, dict2.get(difference, '')) for difference in differences]
                    )
                ))

        if messages:
            self.fail('\n'.join(messages))
<commit_msg>distribute tables<commit_after>from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals
import os
from datetime import datetime

import mock
import postgres_copy
import six
import sqlalchemy
import csv342 as csv

from django.conf import settings
from django.test.utils import override_settings
from django.test.testcases import TestCase

from corehq.apps.domain.models import Domain
from corehq.apps.domain.shortcuts import create_domain
from corehq.apps.locations.models import SQLLocation, LocationType
from corehq.apps.userreports.models import StaticDataSourceConfiguration
from corehq.apps.userreports.util import get_indicator_adapter, get_table_name
from corehq.sql_db.connections import connection_manager, ICDS_UCR_ENGINE_ID
from corehq.sql_db.routers import db_for_read_write
from custom.icds_reports.const import AGG_CCS_RECORD_DELIVERY_TABLE, AGG_COMP_FEEDING_TABLE, \
    AGG_GROWTH_MONITORING_TABLE, AGG_CCS_RECORD_BP_TABLE, AGG_CHILD_HEALTH_PNC_TABLE, AGG_CCS_RECORD_PNC_TABLE, \
    AGG_CCS_RECORD_THR_TABLE, AGG_DAILY_FEEDING_TABLE, AGG_CCS_RECORD_CF_TABLE, AGG_CHILD_HEALTH_THR_TABLE, \
    AGG_INFRASTRUCTURE_TABLE, AWW_INCENTIVE_TABLE

from custom.icds_reports.tasks import (
    move_ucr_data_into_aggregation_tables,
    build_incentive_report,
    _aggregate_child_health_pnc_forms,
    _aggregate_bp_forms,
    _aggregate_gm_forms)
from custom.icds_reports.models.aggregate import AggregateCcsRecordDeliveryForms
from io import open
from six.moves import range
from six.moves import zip

FILE_NAME_TO_TABLE_MAPPING = {
    'awc_mgmt': get_table_name('icds-cas', 'static-awc_mgt_forms'),
    'ccs_monthly': get_table_name('icds-cas', 'static-ccs_record_cases_monthly_tableau_v2'),
    "ccs_cases": get_table_name('icds-cas', 'static-ccs_record_cases'),
    'child_cases': get_table_name('icds-cas', 'static-child_health_cases'),
    'daily_feeding': get_table_name('icds-cas', 'static-daily_feeding_forms'),
    'household_cases': get_table_name('icds-cas', 'static-household_cases'),
    'infrastructure': get_table_name('icds-cas', 'static-infrastructure_form'),
    'infrastructure_v2': get_table_name('icds-cas', 'static-infrastructure_form_v2'),
    'location_ucr': get_table_name('icds-cas', 'static-awc_location'),
    'person_cases': get_table_name('icds-cas', 'static-person_cases_v3'),
    'usage': get_table_name('icds-cas', 'static-usage_forms'),
    'vhnd': get_table_name('icds-cas', 'static-vhnd_form'),
    'complementary_feeding': get_table_name('icds-cas', 'static-complementary_feeding_forms'),
    'aww_user': get_table_name('icds-cas', 'static-commcare_user_cases'),
    'child_tasks': get_table_name('icds-cas', 'static-child_tasks_cases'),
    'pregnant_tasks': get_table_name('icds-cas', 'static-pregnant-tasks_cases'),
    'thr_form': get_table_name('icds-cas', 'static-dashboard_thr_forms'),
    'gm_form': get_table_name('icds-cas', 'static-dashboard_growth_monitoring_forms'),
    'pnc_forms': get_table_name('icds-cas', 'static-postnatal_care_forms'),
    'dashboard_daily_feeding': get_table_name('icds-cas', 'dashboard_child_health_daily_feeding_forms'),
    'ls_awc_mgt': get_table_name('icds-cas', 'static-awc_mgt_forms'),
    'ls_home_vists': get_table_name('icds-cas', 'static-ls_home_visit_forms_filled'),
    'ls_vhnd': get_table_name('icds-cas', 'static-ls_vhnd_form'),
    'cbe_form': get_table_name('icds-cas', 'static-cbe_form'),
    'agg_awc': 'agg_awc',
    'birth_preparedness': get_table_name('icds-cas', 'static-dashboard_birth_preparedness_forms'),
    'delivery_form': get_table_name('icds-cas', 'static-dashboard_delivery_forms'),
}

OUTPUT_PATH = os.path.join(os.path.dirname(__file__), 'outputs')

_use_citus = override_settings(ICDS_USE_CITS=True)

def setUpModule():
    if settings.USE_PARTITIONED_DATABASE:
        print('============= WARNING: not running test setup because settings.USE_PARTITIONED_DATABASE is True.')
        return

    _call_center_domain_mock = mock.patch(
        'corehq.apps.callcenter.data_source.call_center_data_source_configuration_provider'
    )
    _call_center_domain_mock.start()
    _use_citus.enable()

    domain = create_domain('icds-cas')
    SQLLocation.objects.all().delete()
    LocationType.objects.all().delete()
    location_type = LocationType.objects.create(
        domain=domain.name,
        name='block',
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='b1',
        location_id='b1',
        location_type=location_type
    )

    state_location_type = LocationType.objects.create(
        domain=domain.name,
        name='state',
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='st1',
        location_id='st1',
        location_type=state_location_type
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='st2',
        location_id='st2',
        location_type=state_location_type
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='st3',
        location_id='st3',
        location_type=state_location_type
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='st4',
        location_id='st4',
        location_type=state_location_type
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='st5',
        location_id='st5',
        location_type=state_location_type
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='st6',
        location_id='st6',
        location_type=state_location_type
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='st7',
        location_id='st7',
        location_type=state_location_type
    )

    awc_location_type = LocationType.objects.create(
        domain=domain.name,
        name='awc',
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='a7',
        location_id='a7',
        location_type=awc_location_type
    )

    with override_settings(SERVER_ENVIRONMENT='icds'):
        configs = StaticDataSourceConfiguration.by_domain('icds-cas')
        adapters = [get_indicator_adapter(config) for config in configs]

        for adapter in adapters:
            try:
                adapter.drop_table()
            except Exception:
                pass
            adapter.build_table()

        engine = connection_manager.get_engine(ICDS_UCR_ENGINE_ID)
        metadata = sqlalchemy.MetaData(bind=engine)
        metadata.reflect(bind=engine, extend_existing=True)
        path = os.path.join(os.path.dirname(__file__), 'fixtures')
        for file_name in os.listdir(path):
            with open(os.path.join(path, file_name), encoding='utf-8') as f:
                table_name = FILE_NAME_TO_TABLE_MAPPING[file_name[:-4]]
                table = metadata.tables[table_name]
                if not table_name.startswith('icds_dashboard_'):
                    columns = [
                        '"{}"'.format(c.strip())  # quote to preserve case
                        for c in f.readline().split(',')
                    ]
                    postgres_copy.copy_from(
                        f, table, engine, format='csv' if six.PY3 else b'csv',
                        null='' if six.PY3 else b'', columns=columns
                    )

        # move to migrations
        distribute = [
            (AGG_CCS_RECORD_DELIVERY_TABLE, 'supervisor_id'),
            (AGG_COMP_FEEDING_TABLE, 'supervisor_id'),
            (AGG_CCS_RECORD_CF_TABLE, 'supervisor_id'),
            (AGG_CHILD_HEALTH_THR_TABLE, 'supervisor_id'),
            (AGG_GROWTH_MONITORING_TABLE, 'supervisor_id'),
            (AGG_CHILD_HEALTH_PNC_TABLE, 'supervisor_id'),
            (AGG_CCS_RECORD_PNC_TABLE, 'supervisor_id'),
            (AGG_CCS_RECORD_BP_TABLE, 'supervisor_id'),
            (AGG_CCS_RECORD_THR_TABLE, 'supervisor_id'),
            (AGG_DAILY_FEEDING_TABLE, 'supervisor_id'),
            (AGG_INFRASTRUCTURE_TABLE, 'supervisor_id'),
            (AWW_INCENTIVE_TABLE, 'supervisor_id'),
            ('child_health_monthly', 'supervisor_id'),
            ('ccs_record_monthly', 'supervisor_id'),
            ('daily_attendance', 'supervisor_id'),
        ]

        reference = [
            'awc_location',
            'icds_months'
        ]

        for table, col in distribute:
            with engine.begin() as conn:
                res = conn.execute(
                    """
                    SELECT c.relname AS child
                    FROM
                        pg_inherits JOIN pg_class AS c ON (inhrelid=c.oid)
                        JOIN pg_class as p ON (inhparent=p.oid)
                        where p.relname = %s;
                    """,
                    table
                )
                for child in [row.child for row in res]:
                    # only need this because of reusedb if testing on master and this branch
                    conn.execute('drop table if exists "{}"'.format(child))
                res = conn.execute("""
                    select 1 from pg_dist_partition
                    where partmethod = 'h' and logicalrelid = %s::regclass
                """, table)
                if not list(res):
                    conn.execute("select create_distributed_table(%s, %s)", [table, col])

        for table in reference:
            with engine.begin() as conn:
                res = conn.execute("""
                    select 1 from pg_dist_partition
                    where partmethod = 'n' and logicalrelid = %s::regclass
                """, table)
                if not list(res):
                    conn.execute("select create_reference_table(%s)", [table])

        for state_id in ('st1', 'st2'):
            _aggregate_child_health_pnc_forms(state_id, datetime(2017, 3, 31))
            _aggregate_gm_forms(state_id, datetime(2017, 3, 31))
            _aggregate_bp_forms(state_id, datetime(2017, 3, 31))

        try:
            move_ucr_data_into_aggregation_tables(datetime(2017, 5, 28), intervals=2)
            build_incentive_report(agg_date=datetime(2017, 5, 28))
        except AssertionError as e:
            # we always use soft assert to email when the aggregation has completed
            if "Aggregation completed" not in str(e):
                print(e)
                tearDownModule()
                raise
        except Exception as e:
            print(e)
            tearDownModule()
            raise
        finally:
            _call_center_domain_mock.stop()

def tearDownModule():
    if settings.USE_PARTITIONED_DATABASE:
        return

    _call_center_domain_mock = mock.patch(
        'corehq.apps.callcenter.data_source.call_center_data_source_configuration_provider'
    )
    _call_center_domain_mock.start()
    with override_settings(SERVER_ENVIRONMENT='icds'):
        configs = StaticDataSourceConfiguration.by_domain('icds-cas')
        adapters = [get_indicator_adapter(config) for config in configs]
        for adapter in adapters:
            if adapter.config.table_id == 'static-child_health_cases':
                # hack because this is in a migration
                adapter.clear_table()
                continue
            adapter.drop_table()

        engine = connection_manager.get_engine(ICDS_UCR_ENGINE_ID)
        with engine.begin() as connection:
            metadata = sqlalchemy.MetaData(bind=engine)
            metadata.reflect(bind=engine, extend_existing=True)
            table = metadata.tables['ucr_table_name_mapping']
            delete = table.delete()
            connection.execute(delete)
    LocationType.objects.filter(domain='icds-cas').delete()
    SQLLocation.objects.filter(domain='icds-cas').delete()

    Domain.get_by_name('icds-cas').delete()
    _use_citus.disable()
    _call_center_domain_mock.stop()

class CSVTestCase(TestCase):

    def _load_csv(self, path):
        with open(path, encoding='utf-8') as f:
            csv_data = list(csv.reader(f))
            headers = csv_data[0]
            for row_count, row in enumerate(csv_data):
                csv_data[row_count] = dict(zip(headers, row))
        return csv_data[1:]

    def _fasterAssertListEqual(self, list1, list2):
        if len(list1) != len(list2):
            self.fail('Lists are not equal')

        messages = []

        for idx in range(len(list1)):
            dict1 = list1[idx]
            dict2 = list2[idx]

            differences = set()

            for key in dict1.keys():
                if key != 'id':
                    if isinstance(dict1[key], six.text_type):
                        value1 = dict1[key]
                    elif isinstance(dict1[key], list):
                        value1 = str(dict1[key])
                    else:
                        value1 = dict1[key].decode('utf-8')
                    value1 = value1.replace('\r\n', '\n')
                    value2 = dict2.get(key, '').replace('\r\n', '\n')
                    if value1 != value2:
                        differences.add(key)

            if differences:
                if self.always_include_columns:
                    differences |= self.always_include_columns
                messages.append("""
                    Actual and expected row {} are not the same
                    Actual:   {}
                    Expected: {}
                """.format(
                    idx + 1,
                    ', '.join(['{}: {}'.format(
                        difference, str(dict1[difference])) for difference in differences]
                    ),
                    ', '.join(['{}: {}'.format(
                        difference, dict2.get(difference, '')) for difference in differences]
                    )
                ))

        if messages:
            self.fail('\n'.join(messages))
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals
import os
from datetime import datetime

import mock
import postgres_copy
import six
import sqlalchemy
import csv342 as csv

from django.conf import settings
from django.test.utils import override_settings
from django.test.testcases import TestCase

from corehq.apps.domain.models import Domain
from corehq.apps.domain.shortcuts import create_domain
from corehq.apps.locations.models import SQLLocation, LocationType
from corehq.apps.userreports.models import StaticDataSourceConfiguration
from corehq.apps.userreports.util import get_indicator_adapter, get_table_name
from corehq.sql_db.connections import connection_manager, ICDS_UCR_ENGINE_ID
from corehq.sql_db.routers import db_for_read_write
from custom.icds_reports.const import AGG_CCS_RECORD_DELIVERY_TABLE, AGG_COMP_FEEDING_TABLE, \
    AGG_GROWTH_MONITORING_TABLE, AGG_CCS_RECORD_BP_TABLE, AGG_CHILD_HEALTH_PNC_TABLE, AGG_CCS_RECORD_PNC_TABLE, \
    AGG_CCS_RECORD_THR_TABLE, AGG_DAILY_FEEDING_TABLE, AGG_CCS_RECORD_CF_TABLE, AGG_CHILD_HEALTH_THR_TABLE, \
    AGG_INFRASTRUCTURE_TABLE, AWW_INCENTIVE_TABLE

from custom.icds_reports.tasks import (
    move_ucr_data_into_aggregation_tables,
    build_incentive_report,
    _aggregate_child_health_pnc_forms,
    _aggregate_bp_forms,
    _aggregate_gm_forms)
from custom.icds_reports.models.aggregate import AggregateCcsRecordDeliveryForms
from io import open
from six.moves import range
from six.moves import zip

FILE_NAME_TO_TABLE_MAPPING = {
    'awc_mgmt': get_table_name('icds-cas', 'static-awc_mgt_forms'),
    'ccs_monthly': get_table_name('icds-cas', 'static-ccs_record_cases_monthly_tableau_v2'),
    "ccs_cases": get_table_name('icds-cas', 'static-ccs_record_cases'),
    'child_cases': get_table_name('icds-cas', 'static-child_health_cases'),
    'daily_feeding': get_table_name('icds-cas', 'static-daily_feeding_forms'),
    'household_cases': get_table_name('icds-cas', 'static-household_cases'),
    'infrastructure': get_table_name('icds-cas', 'static-infrastructure_form'),
    'infrastructure_v2': get_table_name('icds-cas', 'static-infrastructure_form_v2'),
    'location_ucr': get_table_name('icds-cas', 'static-awc_location'),
    'person_cases': get_table_name('icds-cas', 'static-person_cases_v3'),
    'usage': get_table_name('icds-cas', 'static-usage_forms'),
    'vhnd': get_table_name('icds-cas', 'static-vhnd_form'),
    'complementary_feeding': get_table_name('icds-cas', 'static-complementary_feeding_forms'),
    'aww_user': get_table_name('icds-cas', 'static-commcare_user_cases'),
    'child_tasks': get_table_name('icds-cas', 'static-child_tasks_cases'),
    'pregnant_tasks': get_table_name('icds-cas', 'static-pregnant-tasks_cases'),
    'thr_form': get_table_name('icds-cas', 'static-dashboard_thr_forms'),
    'gm_form': get_table_name('icds-cas', 'static-dashboard_growth_monitoring_forms'),
    'pnc_forms': get_table_name('icds-cas', 'static-postnatal_care_forms'),
    'dashboard_daily_feeding': get_table_name('icds-cas', 'dashboard_child_health_daily_feeding_forms'),
    'ls_awc_mgt': get_table_name('icds-cas', 'static-awc_mgt_forms'),
    'ls_home_vists': get_table_name('icds-cas', 'static-ls_home_visit_forms_filled'),
    'ls_vhnd': get_table_name('icds-cas', 'static-ls_vhnd_form'),
    'cbe_form': get_table_name('icds-cas', 'static-cbe_form'),
    'agg_awc': 'agg_awc',
    'birth_preparedness': get_table_name('icds-cas', 'static-dashboard_birth_preparedness_forms'),
    'delivery_form': get_table_name('icds-cas', 'static-dashboard_delivery_forms'),
}

OUTPUT_PATH = os.path.join(os.path.dirname(__file__), 'outputs')

_use_citus = override_settings(ICDS_USE_CITS=True)

def setUpModule():
    if settings.USE_PARTITIONED_DATABASE:
        print('============= WARNING: not running test setup because settings.USE_PARTITIONED_DATABASE is True.')
        return

    _call_center_domain_mock = mock.patch(
        'corehq.apps.callcenter.data_source.call_center_data_source_configuration_provider'
    )
    _call_center_domain_mock.start()
    _use_citus.enable()

    domain = create_domain('icds-cas')
    SQLLocation.objects.all().delete()
    LocationType.objects.all().delete()
    location_type = LocationType.objects.create(
        domain=domain.name,
        name='block',
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='b1',
        location_id='b1',
        location_type=location_type
    )

    state_location_type = LocationType.objects.create(
        domain=domain.name,
        name='state',
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='st1',
        location_id='st1',
        location_type=state_location_type
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='st2',
        location_id='st2',
        location_type=state_location_type
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='st3',
        location_id='st3',
        location_type=state_location_type
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='st4',
        location_id='st4',
        location_type=state_location_type
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='st5',
        location_id='st5',
        location_type=state_location_type
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='st6',
        location_id='st6',
        location_type=state_location_type
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='st7',
        location_id='st7',
        location_type=state_location_type
    )

    awc_location_type = LocationType.objects.create(
        domain=domain.name,
        name='awc',
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='a7',
        location_id='a7',
        location_type=awc_location_type
    )

    with override_settings(SERVER_ENVIRONMENT='icds'):
        configs = StaticDataSourceConfiguration.by_domain('icds-cas')
        adapters = [get_indicator_adapter(config) for config in configs]

        for adapter in adapters:
            try:
                adapter.drop_table()
            except Exception:
                pass
            adapter.build_table()

        engine = connection_manager.get_engine(ICDS_UCR_ENGINE_ID)
        metadata = sqlalchemy.MetaData(bind=engine)
        metadata.reflect(bind=engine, extend_existing=True)
        path = os.path.join(os.path.dirname(__file__), 'fixtures')
        for file_name in os.listdir(path):
            with open(os.path.join(path, file_name), encoding='utf-8') as f:
                table_name = FILE_NAME_TO_TABLE_MAPPING[file_name[:-4]]
                table = metadata.tables[table_name]
                if not table_name.startswith('icds_dashboard_'):
                    columns = [
                        '"{}"'.format(c.strip())  # quote to preserve case
                        for c in f.readline().split(',')
                    ]
                    postgres_copy.copy_from(
                        f, table, engine, format='csv' if six.PY3 else b'csv',
                        null='' if six.PY3 else b'', columns=columns
                    )

        # move to migrations
        distribute = [
            (AGG_CCS_RECORD_DELIVERY_TABLE, 'supervisor_id'),
            (AGG_COMP_FEEDING_TABLE, 'supervisor_id'),
            (AGG_CCS_RECORD_CF_TABLE, 'supervisor_id'),
            (AGG_CHILD_HEALTH_THR_TABLE, 'supervisor_id'),
            (AGG_GROWTH_MONITORING_TABLE, 'supervisor_id'),
            (AGG_CHILD_HEALTH_PNC_TABLE, 'supervisor_id'),
            (AGG_CCS_RECORD_PNC_TABLE, 'supervisor_id'),
            (AGG_CCS_RECORD_BP_TABLE, 'supervisor_id'),
            (AGG_CCS_RECORD_THR_TABLE, 'supervisor_id'),
            (AGG_DAILY_FEEDING_TABLE, 'supervisor_id'),
            (AGG_INFRASTRUCTURE_TABLE, 'supervisor_id'),
            (AWW_INCENTIVE_TABLE, 'supervisor_id'),
            ('child_health_monthly', 'supervisor_id'),
            ('ccs_record_monthly', 'supervisor_id'),
            ('daily_attendance', 'supervisor_id'),
        ]

        reference = [
            'awc_location',
            'icds_months'
        ]

        for table, col in distribute:
            with engine.begin() as conn:
                res = conn.execute(
                    """
                    SELECT c.relname AS child
                    FROM
                        pg_inherits JOIN pg_class AS c ON (inhrelid=c.oid)
                        JOIN pg_class as p ON (inhparent=p.oid)
                        where p.relname = %s;
                    """,
                    table
                )
                for child in [row.child for row in res]:
                    # only need this because of reusedb if testing on master and this branch
                    conn.execute('drop table if exists "{}"'.format(child))
                res = conn.execute("""
                    select 1 from pg_dist_partition
                    where partmethod = 'h' and logicalrelid = %s::regclass
                """, table)
                if not list(res):
                    conn.execute("select create_distributed_table(%s, %s)", [table, col])

        for table in reference:
            with engine.begin() as conn:
                res = conn.execute("""
                    select 1 from pg_dist_partition
                    where partmethod = 'n' and logicalrelid = %s::regclass
                """, table)
                if not list(res):
                    conn.execute("select create_reference_table(%s)", [table])

        for state_id in ('st1', 'st2'):
            _aggregate_child_health_pnc_forms(state_id, datetime(2017, 3, 31))
            _aggregate_gm_forms(state_id, datetime(2017, 3, 31))
            _aggregate_bp_forms(state_id, datetime(2017, 3, 31))

        try:
            move_ucr_data_into_aggregation_tables(datetime(2017, 5, 28), intervals=2)
            build_incentive_report(agg_date=datetime(2017, 5, 28))
        except AssertionError as e:
            # we always use soft assert to email when the aggregation has completed
            if "Aggregation completed" not in str(e):
                print(e)
                tearDownModule()
                raise
        except Exception as e:
            print(e)
            tearDownModule()
            raise
        finally:
            _call_center_domain_mock.stop()

def tearDownModule():
    if settings.USE_PARTITIONED_DATABASE:
        return

    _call_center_domain_mock = mock.patch(
        'corehq.apps.callcenter.data_source.call_center_data_source_configuration_provider'
    )
    _call_center_domain_mock.start()
    with override_settings(SERVER_ENVIRONMENT='icds'):
        configs = StaticDataSourceConfiguration.by_domain('icds-cas')
        adapters = [get_indicator_adapter(config) for config in configs]
        for adapter in adapters:
            if adapter.config.table_id == 'static-child_health_cases':
                # hack because this is in a migration
                adapter.clear_table()
                continue
            adapter.drop_table()

        engine = connection_manager.get_engine(ICDS_UCR_ENGINE_ID)
        with engine.begin() as connection:
            metadata = sqlalchemy.MetaData(bind=engine)
            metadata.reflect(bind=engine, extend_existing=True)
            table = metadata.tables['ucr_table_name_mapping']
            delete = table.delete()
            connection.execute(delete)
    LocationType.objects.filter(domain='icds-cas').delete()
    SQLLocation.objects.filter(domain='icds-cas').delete()

    Domain.get_by_name('icds-cas').delete()
    _use_citus.disable()
    _call_center_domain_mock.stop()

class CSVTestCase(TestCase):

    def _load_csv(self, path):
        with open(path, encoding='utf-8') as f:
            csv_data = list(csv.reader(f))
            headers = csv_data[0]
            for row_count, row in enumerate(csv_data):
                csv_data[row_count] = dict(zip(headers, row))
        return csv_data[1:]

    def _fasterAssertListEqual(self, list1, list2):
        if len(list1) != len(list2):
            self.fail('Lists are not equal')

        messages = []

        for idx in range(len(list1)):
            dict1 = list1[idx]
            dict2 = list2[idx]

            differences = set()

            for key in dict1.keys():
                if key != 'id':
                    if isinstance(dict1[key], six.text_type):
                        value1 = dict1[key]
                    elif isinstance(dict1[key], list):
                        value1 = str(dict1[key])
                    else:
                        value1 = dict1[key].decode('utf-8')
                    value1 = value1.replace('\r\n', '\n')
                    value2 = dict2.get(key, '').replace('\r\n', '\n')
                    if value1 != value2:
                        differences.add(key)

            if differences:
                if self.always_include_columns:
                    differences |= self.always_include_columns
                messages.append("""
                    Actual and expected row {} are not the same
                    Actual:   {}
                    Expected: {}
                """.format(
                    idx + 1,
                    ', '.join(['{}: {}'.format(
                        difference, str(dict1[difference])) for difference in differences]
                    ),
                    ', '.join(['{}: {}'.format(
                        difference, dict2.get(difference, '')) for difference in differences]
                    )
                ))

        if messages:
            self.fail('\n'.join(messages))
<commit_msg>run tests without citus<commit_after>from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals
import os
from datetime import datetime

import mock
import postgres_copy
import six
import sqlalchemy
import csv342 as csv

from django.conf import settings
from django.test.utils import override_settings
from django.test.testcases import TestCase

from corehq.apps.domain.models import Domain
from corehq.apps.domain.shortcuts import create_domain
from corehq.apps.locations.models import SQLLocation, LocationType
from corehq.apps.userreports.models import StaticDataSourceConfiguration
from corehq.apps.userreports.util import get_indicator_adapter, get_table_name
from corehq.sql_db.connections import connection_manager, ICDS_UCR_ENGINE_ID
from corehq.sql_db.routers import db_for_read_write
from custom.icds_reports.const import AGG_CCS_RECORD_DELIVERY_TABLE, AGG_COMP_FEEDING_TABLE, \
    AGG_GROWTH_MONITORING_TABLE, AGG_CCS_RECORD_BP_TABLE, AGG_CHILD_HEALTH_PNC_TABLE, AGG_CCS_RECORD_PNC_TABLE, \
    AGG_CCS_RECORD_THR_TABLE, AGG_DAILY_FEEDING_TABLE, AGG_CCS_RECORD_CF_TABLE, AGG_CHILD_HEALTH_THR_TABLE, \
    AGG_INFRASTRUCTURE_TABLE, AWW_INCENTIVE_TABLE

from custom.icds_reports.tasks import (
    move_ucr_data_into_aggregation_tables,
    build_incentive_report,
    _aggregate_child_health_pnc_forms,
    _aggregate_bp_forms,
    _aggregate_gm_forms)
from custom.icds_reports.models.aggregate import AggregateCcsRecordDeliveryForms
from io import open
from six.moves import range
from six.moves import zip

FILE_NAME_TO_TABLE_MAPPING = {
    'awc_mgmt': get_table_name('icds-cas', 'static-awc_mgt_forms'),
    'ccs_monthly': get_table_name('icds-cas', 'static-ccs_record_cases_monthly_tableau_v2'),
    "ccs_cases": get_table_name('icds-cas', 'static-ccs_record_cases'),
    'child_cases': get_table_name('icds-cas', 'static-child_health_cases'),
    'daily_feeding': get_table_name('icds-cas', 'static-daily_feeding_forms'),
    'household_cases': get_table_name('icds-cas', 'static-household_cases'),
    'infrastructure': get_table_name('icds-cas', 'static-infrastructure_form'),
    'infrastructure_v2': get_table_name('icds-cas', 'static-infrastructure_form_v2'),
    'location_ucr': get_table_name('icds-cas', 'static-awc_location'),
    'person_cases': get_table_name('icds-cas', 'static-person_cases_v3'),
    'usage': get_table_name('icds-cas', 'static-usage_forms'),
    'vhnd': get_table_name('icds-cas', 'static-vhnd_form'),
    'complementary_feeding': get_table_name('icds-cas', 'static-complementary_feeding_forms'),
    'aww_user': get_table_name('icds-cas', 'static-commcare_user_cases'),
    'child_tasks': get_table_name('icds-cas', 'static-child_tasks_cases'),
    'pregnant_tasks': get_table_name('icds-cas', 'static-pregnant-tasks_cases'),
    'thr_form': get_table_name('icds-cas', 'static-dashboard_thr_forms'),
    'gm_form': get_table_name('icds-cas', 'static-dashboard_growth_monitoring_forms'),
    'pnc_forms': get_table_name('icds-cas', 'static-postnatal_care_forms'),
    'dashboard_daily_feeding': get_table_name('icds-cas', 'dashboard_child_health_daily_feeding_forms'),
    'ls_awc_mgt': get_table_name('icds-cas', 'static-awc_mgt_forms'),
    'ls_home_vists': get_table_name('icds-cas', 'static-ls_home_visit_forms_filled'),
    'ls_vhnd': get_table_name('icds-cas', 'static-ls_vhnd_form'),
    'cbe_form': get_table_name('icds-cas', 'static-cbe_form'),
    'agg_awc': 'agg_awc',
    'birth_preparedness': get_table_name('icds-cas', 'static-dashboard_birth_preparedness_forms'),
    'delivery_form': get_table_name('icds-cas', 'static-dashboard_delivery_forms'),
}

OUTPUT_PATH = os.path.join(os.path.dirname(__file__), 'outputs')

_use_citus = override_settings(ICDS_USE_CITS=True)

def setUpModule():
    if settings.USE_PARTITIONED_DATABASE:
        print('============= WARNING: not running test setup because settings.USE_PARTITIONED_DATABASE is True.')
        return

    _call_center_domain_mock = mock.patch(
        'corehq.apps.callcenter.data_source.call_center_data_source_configuration_provider'
    )
    _call_center_domain_mock.start()
    # _use_citus.enable()

    domain = create_domain('icds-cas')
    SQLLocation.objects.all().delete()
    LocationType.objects.all().delete()
    location_type = LocationType.objects.create(
        domain=domain.name,
        name='block',
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='b1',
        location_id='b1',
        location_type=location_type
    )

    state_location_type = LocationType.objects.create(
        domain=domain.name,
        name='state',
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='st1',
        location_id='st1',
        location_type=state_location_type
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='st2',
        location_id='st2',
        location_type=state_location_type
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='st3',
        location_id='st3',
        location_type=state_location_type
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='st4',
        location_id='st4',
        location_type=state_location_type
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='st5',
        location_id='st5',
        location_type=state_location_type
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='st6',
        location_id='st6',
        location_type=state_location_type
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='st7',
        location_id='st7',
        location_type=state_location_type
    )

    awc_location_type = LocationType.objects.create(
        domain=domain.name,
        name='awc',
    )
    SQLLocation.objects.create(
        domain=domain.name,
        name='a7',
        location_id='a7',
        location_type=awc_location_type
    )

    with override_settings(SERVER_ENVIRONMENT='icds'):
        configs = StaticDataSourceConfiguration.by_domain('icds-cas')
        adapters = [get_indicator_adapter(config) for config in configs]

        for adapter in adapters:
            try:
                adapter.drop_table()
            except Exception:
                pass
            adapter.build_table()

        engine = connection_manager.get_engine(ICDS_UCR_ENGINE_ID)
        metadata = sqlalchemy.MetaData(bind=engine)
        metadata.reflect(bind=engine, extend_existing=True)
        path = os.path.join(os.path.dirname(__file__), 'fixtures')
        for file_name in os.listdir(path):
            with open(os.path.join(path, file_name), encoding='utf-8') as f:
                table_name = FILE_NAME_TO_TABLE_MAPPING[file_name[:-4]]
                table = metadata.tables[table_name]
                if not table_name.startswith('icds_dashboard_'):
                    columns = [
                        '"{}"'.format(c.strip())  # quote to preserve case
                        for c in f.readline().split(',')
                    ]
                    postgres_copy.copy_from(
                        f, table, engine, format='csv' if six.PY3 else b'csv',
                        null='' if six.PY3 else b'', columns=columns
                    )

        _distribute_tables_for_citus(engine)

        for state_id in ('st1', 'st2'):
            _aggregate_child_health_pnc_forms(state_id, datetime(2017, 3, 31))
            _aggregate_gm_forms(state_id, datetime(2017, 3, 31))
            _aggregate_bp_forms(state_id, datetime(2017, 3, 31))

        try:
            move_ucr_data_into_aggregation_tables(datetime(2017, 5, 28), intervals=2)
            build_incentive_report(agg_date=datetime(2017, 5, 28))
        except AssertionError as e:
            # we always use soft assert to email when the aggregation has completed
            if "Aggregation completed" not in str(e):
                print(e)
                tearDownModule()
                raise
        except Exception as e:
            print(e)
            tearDownModule()
            raise
        finally:
            _call_center_domain_mock.stop()

def _distribute_tables_for_citus(engine):
    if not getattr(settings, 'ICDS_USE_CITS', False):
        return

    # move to migrations
    distribute = [
        (AGG_CCS_RECORD_DELIVERY_TABLE, 'supervisor_id'),
        (AGG_COMP_FEEDING_TABLE, 'supervisor_id'),
        (AGG_CCS_RECORD_CF_TABLE, 'supervisor_id'),
        (AGG_CHILD_HEALTH_THR_TABLE, 'supervisor_id'),
        (AGG_GROWTH_MONITORING_TABLE, 'supervisor_id'),
        (AGG_CHILD_HEALTH_PNC_TABLE, 'supervisor_id'),
        (AGG_CCS_RECORD_PNC_TABLE, 'supervisor_id'),
        (AGG_CCS_RECORD_BP_TABLE, 'supervisor_id'),
        (AGG_CCS_RECORD_THR_TABLE, 'supervisor_id'),
        (AGG_DAILY_FEEDING_TABLE, 'supervisor_id'),
        (AGG_INFRASTRUCTURE_TABLE, 'supervisor_id'),
        (AWW_INCENTIVE_TABLE, 'supervisor_id'),
        ('child_health_monthly', 'supervisor_id'),
        ('ccs_record_monthly', 'supervisor_id'),
        ('daily_attendance', 'supervisor_id'),
    ]
    reference = [
        'awc_location',
        'icds_months'
    ]
    for table, col in distribute:
        with engine.begin() as conn:
            res = conn.execute(
                """
                SELECT c.relname AS child
                FROM
                    pg_inherits JOIN pg_class AS c ON (inhrelid=c.oid)
                    JOIN pg_class as p ON (inhparent=p.oid)
                    where p.relname = %s;
                """,
                table
            )
            for child in [row.child for row in res]:
                # only need this because of reusedb if testing on master and this branch
                conn.execute('drop table if exists "{}"'.format(child))
            res = conn.execute("""
                    select 1 from pg_dist_partition
                    where partmethod = 'h' and logicalrelid = %s::regclass
                """, table)
            if not list(res):
                conn.execute("select create_distributed_table(%s, %s)", [table, col])
    for table in reference:
        with engine.begin() as conn:
            res = conn.execute("""
                    select 1 from pg_dist_partition
                    where partmethod = 'n' and logicalrelid = %s::regclass
                """, table)
            if not list(res):
                conn.execute("select create_reference_table(%s)", [table])

def tearDownModule():
    if settings.USE_PARTITIONED_DATABASE:
        return

    _call_center_domain_mock = mock.patch(
        'corehq.apps.callcenter.data_source.call_center_data_source_configuration_provider'
    )
    _call_center_domain_mock.start()
    with override_settings(SERVER_ENVIRONMENT='icds'):
        configs = StaticDataSourceConfiguration.by_domain('icds-cas')
        adapters = [get_indicator_adapter(config) for config in configs]
        for adapter in adapters:
            if adapter.config.table_id == 'static-child_health_cases':
                # hack because this is in a migration
                adapter.clear_table()
                continue
            adapter.drop_table()

        engine = connection_manager.get_engine(ICDS_UCR_ENGINE_ID)
        with engine.begin() as connection:
            metadata = sqlalchemy.MetaData(bind=engine)
            metadata.reflect(bind=engine, extend_existing=True)
            table = metadata.tables['ucr_table_name_mapping']
            delete = table.delete()
            connection.execute(delete)
    LocationType.objects.filter(domain='icds-cas').delete()
    SQLLocation.objects.filter(domain='icds-cas').delete()

    Domain.get_by_name('icds-cas').delete()
    # _use_citus.disable()
    _call_center_domain_mock.stop()

class CSVTestCase(TestCase):

    def _load_csv(self, path):
        with open(path, encoding='utf-8') as f:
            csv_data = list(csv.reader(f))
            headers = csv_data[0]
            for row_count, row in enumerate(csv_data):
                csv_data[row_count] = dict(zip(headers, row))
        return csv_data[1:]

    def _fasterAssertListEqual(self, list1, list2):
        if len(list1) != len(list2):
            self.fail('Lists are not equal')

        messages = []

        for idx in range(len(list1)):
            dict1 = list1[idx]
            dict2 = list2[idx]

            differences = set()

            for key in dict1.keys():
                if key != 'id':
                    if isinstance(dict1[key], six.text_type):
                        value1 = dict1[key]
                    elif isinstance(dict1[key], list):
                        value1 = str(dict1[key])
                    else:
                        value1 = dict1[key].decode('utf-8')
                    value1 = value1.replace('\r\n', '\n')
                    value2 = dict2.get(key, '').replace('\r\n', '\n')
                    if value1 != value2:
                        differences.add(key)

            if differences:
                if self.always_include_columns:
                    differences |= self.always_include_columns
                messages.append("""
                    Actual and expected row {} are not the same
                    Actual:   {}
                    Expected: {}
                """.format(
                    idx + 1,
                    ', '.join(['{}: {}'.format(
                        difference, str(dict1[difference])) for difference in differences]
                    ),
                    ', '.join(['{}: {}'.format(
                        difference, dict2.get(difference, '')) for difference in differences]
                    )
                ))

        if messages:
            self.fail('\n'.join(messages))
<|endoftext|>


<commit_before>"""

Creates a timeseries plot

This module wraps the plot_timeseries.sh script found in git repository cwsl-ctools.

Part of the CWSLab VisTrails plugin.

Authors: Tim Bedin, Tim.Bedin@csiro.au
         Tim Erwin, Tim.Erwin@csiro.au

Copyright 2014 CSIRO

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

"""

__version__ = '$Id$'

import os

from vistrails.core.modules import vistrails_module
from vistrails.core.modules.basic_modules import String, List

from cwsl.configuration import configuration
from cwsl.core.process_unit import ProcessUnit
from cwsl.core.constraint import Constraint
from cwsl.core.pattern_generator import PatternGenerator

class PlotTimeSeries(vistrails_module.Module):
    """
    Plots a timeseries.

    Required inputs: variable_name (varible name of data from input file)

    Other inputs:    infile (from dataset connector)
                     title (model name from dataset connector)

    Requires: python, cdat

    """

    # Define the module ports.
    _input_ports = [('in_dataset', 'csiro.au.cwsl:VtDataSet',
                     {'labels': str(['Input Dataset'])}),
                    ('variable_name', String),
                   ]

    _output_ports = [('out_dataset', 'csiro.au.cwsl:VtDataSet')]

    _execution_options = {'required_modules': ['python/2.7.5','python/2.7.5-matplotlib', 'python-cdat-lite/6.0rc2-py2.7.5']
                         }

    def __init__(self):

        super(PlotTimeSeries, self).__init__()

        self.command = '${CWSL_CTOOLS}/visualisation/plot_timeseries.py'

        # Get the output pattern using the PatternGenerator object.
        # Gets the user infomation / authoritative path etc from the
        # user configuration.
        self.out_pattern = PatternGenerator('user', 'indicies').pattern

    def compute(self):

        # Required input
        in_dataset = self.getInputFromPort("in_dataset")
        variable_name = self.getInputFromPort("variable_name")
        self.positional_args=[(variable_name,0,'raw'),('model',1)]

        new_cons = set([Constraint('variable', [variable_name]),
                        Constraint('suffix', ['png']),
                        ])

        # Optional added constraints.
        try:
            # Add extra constraints if necessary.
            added_constraints = self.getInputFromPort('added_constraints')
            cons_for_output = new_cons.intersection(added_constraints)
        except vistrails_module.ModuleError:
            cons_for_output = new_cons

        # Execute the seas_vars process.
        this_process = ProcessUnit([in_dataset],
                                   self.out_pattern,
                                   self.command,
                                   cons_for_output,
                                   execution_options=self._execution_options,
                                   positional_args=self.positional_args)

        this_process.execute(simulate=configuration.simulate_execution)
        process_output = this_process.file_creator

        self.setResult('out_dataset', process_output)
<commit_msg>Removed svn ID tag<commit_after>"""

Creates a timeseries plot

This module wraps the plot_timeseries.sh script found in git repository cwsl-ctools.

Part of the CWSLab VisTrails plugin.

Authors: Tim Bedin, Tim.Bedin@csiro.au
         Tim Erwin, Tim.Erwin@csiro.au

Copyright 2014 CSIRO

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

"""

import os

from vistrails.core.modules import vistrails_module
from vistrails.core.modules.basic_modules import String, List

from cwsl.configuration import configuration
from cwsl.core.process_unit import ProcessUnit
from cwsl.core.constraint import Constraint
from cwsl.core.pattern_generator import PatternGenerator

class PlotTimeSeries(vistrails_module.Module):
    """
    Plots a timeseries.

    Required inputs: variable_name (varible name of data from input file)

    Other inputs:    infile (from dataset connector)
                     title (model name from dataset connector)

    Requires: python, cdat

    """

    # Define the module ports.
    _input_ports = [('in_dataset', 'csiro.au.cwsl:VtDataSet',
                     {'labels': str(['Input Dataset'])}),
                    ('variable_name', String),
                   ]

    _output_ports = [('out_dataset', 'csiro.au.cwsl:VtDataSet')]

    _execution_options = {'required_modules': ['python/2.7.5','python/2.7.5-matplotlib', 'python-cdat-lite/6.0rc2-py2.7.5']
                         }

    def __init__(self):

        super(PlotTimeSeries, self).__init__()

        self.command = '${CWSL_CTOOLS}/visualisation/plot_timeseries.py'

        # Get the output pattern using the PatternGenerator object.
        # Gets the user infomation / authoritative path etc from the
        # user configuration.
        self.out_pattern = PatternGenerator('user', 'indicies').pattern

    def compute(self):

        # Required input
        in_dataset = self.getInputFromPort("in_dataset")
        variable_name = self.getInputFromPort("variable_name")
        self.positional_args=[(variable_name,0,'raw'),('model',1)]

        new_cons = set([Constraint('variable', [variable_name]),
                        Constraint('suffix', ['png']),
                        ])

        # Optional added constraints.
        try:
            # Add extra constraints if necessary.
            added_constraints = self.getInputFromPort('added_constraints')
            cons_for_output = new_cons.intersection(added_constraints)
        except vistrails_module.ModuleError:
            cons_for_output = new_cons

        # Execute the seas_vars process.
        this_process = ProcessUnit([in_dataset],
                                   self.out_pattern,
                                   self.command,
                                   cons_for_output,
                                   execution_options=self._execution_options,
                                   positional_args=self.positional_args)

        this_process.execute(simulate=configuration.simulate_execution)
        process_output = this_process.file_creator

        self.setResult('out_dataset', process_output)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import re

from datetime import datetime, timedelta
from subprocess import Popen, PIPE

from moulinette.utils.filesystem import read_file

from yunohost.utils.network import dig
from yunohost.diagnosis import Diagnoser
from yunohost.domain import domain_list, _build_dns_conf, _get_maindomain

SMALL_SUFFIX_LIST = ['noho.st', 'nohost.me', 'ynh.fr', 'netlib.re']

class DNSRecordsDiagnoser(Diagnoser):

    id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
    cache_duration = 600
    dependencies = ["ip"]

    def run(self):

        resolvers = read_file("/etc/resolv.dnsmasq.conf").split("\n")
        ipv4_resolvers = [r.split(" ")[1] for r in resolvers if r.startswith("nameserver") and ":" not in r]
        # FIXME some day ... handle ipv4-only and ipv6-only servers. For now we assume we have at least ipv4
        assert ipv4_resolvers != [], "Uhoh, need at least one IPv4 DNS resolver ..."

        self.resolver = ipv4_resolvers[0]
        main_domain = _get_maindomain()

        all_domains = domain_list()["domains"]
        for domain in all_domains:
            self.logger_debug("Diagnosing DNS conf for %s" % domain)
            is_subdomain = domain.split(".",1)[1] in all_domains
            for report in self.check_domain(domain, domain == main_domain, is_subdomain=is_subdomain):
                yield report

        # Check if a domain buy by the user will expire soon
        domains_from_registrar = ['.'.join(domain.split('.')[-2:]) for domain in all_domains]
        domains_from_registrar = set(domains_from_registrar) - set(SMALL_SUFFIX_LIST)
        for report in self.check_expiration_date(domains_from_registrar):
            yield report

    def check_domain(self, domain, is_main_domain, is_subdomain):

        expected_configuration = _build_dns_conf(domain, include_empty_AAAA_if_no_ipv6=True)

        categories = ["basic", "mail", "xmpp", "extra"]
        # For subdomains, we only diagnosis A and AAAA records
        if is_subdomain:
            categories = ["basic"]

        for category in categories:

            records = expected_configuration[category]
            discrepancies = []
            results = {}

            for r in records:
                id_ = r["type"] + ":" + r["name"]
                r["current"] = self.get_current_record(domain, r["name"], r["type"])
                if r["value"] == "@":
                    r["value"] = domain + "."

                if self.current_record_match_expected(r):
                    results[id_] = "OK"
                else:
                    if r["current"] is None:
                        results[id_] = "MISSING"
                        discrepancies.append(("diagnosis_dns_missing_record", r))
                    else:
                        results[id_] = "WRONG"
                        discrepancies.append(("diagnosis_dns_discrepancy", r))

            def its_important():
                # Every mail DNS records are important for main domain
                # For other domain, we only report it as a warning for now...
                if is_main_domain and category == "mail":
                    return True
                elif category == "basic":
                    # A bad or missing A record is critical ...
                    # And so is a wrong AAAA record
                    # (However, a missing AAAA record is acceptable)
                    if results["A:@"] != "OK" or results["AAAA:@"] == "WRONG":
                        return True

                return False

            if discrepancies:
                status = "ERROR" if its_important() else "WARNING"
                summary = "diagnosis_dns_bad_conf"
            else:
                status = "SUCCESS"
                summary = "diagnosis_dns_good_conf"

            output = dict(meta={"domain": domain, "category": category},
                          data=results,
                          status=status,
                          summary=summary)

            if discrepancies:
                output["details"] = ["diagnosis_dns_point_to_doc"] + discrepancies

            yield output

    def get_current_record(self, domain, name, type_):

        query = "%s.%s" % (name, domain) if name != "@" else domain
        success, answers = dig(query, type_, resolvers="force_external")

        if success != "ok":
            return None
        else:
            return answers[0] if len(answers) == 1 else answers

    def current_record_match_expected(self, r):
        if r["value"] is not None and r["current"] is None:
            return False
        if r["value"] is None and r["current"] is not None:
            return False
        elif isinstance(r["current"], list):
            return False

        if r["type"] == "TXT":
            # Split expected/current
            #  from  "v=DKIM1; k=rsa; p=hugekey;"
            #  to a set like {'v=DKIM1', 'k=rsa', 'p=...'}
            expected = set(r["value"].strip(';" ').replace(";", " ").split())
            current = set(r["current"].strip(';" ').replace(";", " ").split())

            # For SPF, ignore parts starting by ip4: or ip6:
            if r["name"] == "@":
                current = {part for part in current if not part.startswith("ip4:") and not part.startswith("ip6:")}
            return expected == current
        elif r["type"] ==  "MX":
            # For MX, we want to ignore the priority
            expected = r["value"].split()[-1]
            current = r["current"].split()[-1]
            return expected == current
        else:
            return r["current"] == r["value"]

    def check_expiration_date(self, domains):
        """
        Alert if expiration date of a domain is soon
        """

        # FIXME find a way to ignore a specific domain without
        # create a report by domain each time. We need something small
        details = {
            "not_found": [],
            "error": [],
            "warning": [],
            "info": []
        }

        for domain in domains:
            expire_date = self.get_domain_expiration(domain)

            if not expire_date:
                details["not_found"].append((
                    "diagnosis_domain_expiration_date_not_found",
                    {"domain": domain}))
                continue

            expire_in = expire_date - datetime.now()

            alert_type = "info"
            if expire_in <= timedelta(7):
                alert_type = "error"
            elif expire_in <= timedelta(30):
                alert_type = "warning"

            args = {
                "domain": domain,
                "days": expire_in.days - 1,
                "expire_date": str(expire_date)
            }
            details[alert_type].append(("diagnosis_domain_expires_in", args))

        for alert_type in ["error", "warning", "not_found", "info"]:
            if details[alert_type]:
                yield dict(meta={"category": "expiration"},
                           data={},
                           status=alert_type.upper() if alert_type != "not_found" else "INFO",
                           summary="diagnosis_domain_expiration_" + alert_type,
                           details=details[alert_type])

    def get_domain_expiration(self, domain):
        """
        Return the expiration datetime of a domain or None
        """

        p1 = Popen(['whois', domain], stdout=PIPE)
        p2 = Popen(['grep', 'Expir'], stdin=p1.stdout, stdout=PIPE)
        out, err = p2.communicate()
        out = out.decode("utf-8").split('\n')
        p1.terminate()
        #p2.terminate()

        for line in out:
            match = re.search(r'\d{4}-\d{2}-\d{2}', line)
            if match is not None:
                return datetime.strptime(match.group(), '%Y-%m-%d')
        return None

def main(args, env, loggers):
    return DNSRecordsDiagnoser(args, env, loggers).diagnose()
<commit_msg>[fix] Bad i18n key<commit_after>#!/usr/bin/env python

import os
import re

from datetime import datetime, timedelta
from subprocess import Popen, PIPE

from moulinette.utils.filesystem import read_file

from yunohost.utils.network import dig
from yunohost.diagnosis import Diagnoser
from yunohost.domain import domain_list, _build_dns_conf, _get_maindomain

SMALL_SUFFIX_LIST = ['noho.st', 'nohost.me', 'ynh.fr', 'netlib.re']

class DNSRecordsDiagnoser(Diagnoser):

    id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
    cache_duration = 600
    dependencies = ["ip"]

    def run(self):

        resolvers = read_file("/etc/resolv.dnsmasq.conf").split("\n")
        ipv4_resolvers = [r.split(" ")[1] for r in resolvers if r.startswith("nameserver") and ":" not in r]
        # FIXME some day ... handle ipv4-only and ipv6-only servers. For now we assume we have at least ipv4
        assert ipv4_resolvers != [], "Uhoh, need at least one IPv4 DNS resolver ..."

        self.resolver = ipv4_resolvers[0]
        main_domain = _get_maindomain()

        all_domains = domain_list()["domains"]
        for domain in all_domains:
            self.logger_debug("Diagnosing DNS conf for %s" % domain)
            is_subdomain = domain.split(".",1)[1] in all_domains
            for report in self.check_domain(domain, domain == main_domain, is_subdomain=is_subdomain):
                yield report

        # Check if a domain buy by the user will expire soon
        domains_from_registrar = ['.'.join(domain.split('.')[-2:]) for domain in all_domains]
        domains_from_registrar = set(domains_from_registrar) - set(SMALL_SUFFIX_LIST)
        for report in self.check_expiration_date(domains_from_registrar):
            yield report

    def check_domain(self, domain, is_main_domain, is_subdomain):

        expected_configuration = _build_dns_conf(domain, include_empty_AAAA_if_no_ipv6=True)

        categories = ["basic", "mail", "xmpp", "extra"]
        # For subdomains, we only diagnosis A and AAAA records
        if is_subdomain:
            categories = ["basic"]

        for category in categories:

            records = expected_configuration[category]
            discrepancies = []
            results = {}

            for r in records:
                id_ = r["type"] + ":" + r["name"]
                r["current"] = self.get_current_record(domain, r["name"], r["type"])
                if r["value"] == "@":
                    r["value"] = domain + "."

                if self.current_record_match_expected(r):
                    results[id_] = "OK"
                else:
                    if r["current"] is None:
                        results[id_] = "MISSING"
                        discrepancies.append(("diagnosis_dns_missing_record", r))
                    else:
                        results[id_] = "WRONG"
                        discrepancies.append(("diagnosis_dns_discrepancy", r))

            def its_important():
                # Every mail DNS records are important for main domain
                # For other domain, we only report it as a warning for now...
                if is_main_domain and category == "mail":
                    return True
                elif category == "basic":
                    # A bad or missing A record is critical ...
                    # And so is a wrong AAAA record
                    # (However, a missing AAAA record is acceptable)
                    if results["A:@"] != "OK" or results["AAAA:@"] == "WRONG":
                        return True

                return False

            if discrepancies:
                status = "ERROR" if its_important() else "WARNING"
                summary = "diagnosis_dns_bad_conf"
            else:
                status = "SUCCESS"
                summary = "diagnosis_dns_good_conf"

            output = dict(meta={"domain": domain, "category": category},
                          data=results,
                          status=status,
                          summary=summary)

            if discrepancies:
                output["details"] = ["diagnosis_dns_point_to_doc"] + discrepancies

            yield output

    def get_current_record(self, domain, name, type_):

        query = "%s.%s" % (name, domain) if name != "@" else domain
        success, answers = dig(query, type_, resolvers="force_external")

        if success != "ok":
            return None
        else:
            return answers[0] if len(answers) == 1 else answers

    def current_record_match_expected(self, r):
        if r["value"] is not None and r["current"] is None:
            return False
        if r["value"] is None and r["current"] is not None:
            return False
        elif isinstance(r["current"], list):
            return False

        if r["type"] == "TXT":
            # Split expected/current
            #  from  "v=DKIM1; k=rsa; p=hugekey;"
            #  to a set like {'v=DKIM1', 'k=rsa', 'p=...'}
            expected = set(r["value"].strip(';" ').replace(";", " ").split())
            current = set(r["current"].strip(';" ').replace(";", " ").split())

            # For SPF, ignore parts starting by ip4: or ip6:
            if r["name"] == "@":
                current = {part for part in current if not part.startswith("ip4:") and not part.startswith("ip6:")}
            return expected == current
        elif r["type"] ==  "MX":
            # For MX, we want to ignore the priority
            expected = r["value"].split()[-1]
            current = r["current"].split()[-1]
            return expected == current
        else:
            return r["current"] == r["value"]

    def check_expiration_date(self, domains):
        """
        Alert if expiration date of a domain is soon
        """

        # FIXME find a way to ignore a specific domain without
        # create a report by domain each time. We need something small
        details = {
            "not_found": [],
            "error": [],
            "warning": [],
            "info": []
        }

        for domain in domains:
            expire_date = self.get_domain_expiration(domain)

            if not expire_date:
                details["not_found"].append((
                    "diagnosis_domain_expiration_not_found",
                    {"domain": domain}))
                continue

            expire_in = expire_date - datetime.now()

            alert_type = "info"
            if expire_in <= timedelta(7):
                alert_type = "error"
            elif expire_in <= timedelta(30):
                alert_type = "warning"

            args = {
                "domain": domain,
                "days": expire_in.days - 1,
                "expire_date": str(expire_date)
            }
            details[alert_type].append(("diagnosis_domain_expires_in", args))

        for alert_type in ["error", "warning", "not_found", "info"]:
            if details[alert_type]:
                yield dict(meta={"category": "expiration"},
                           data={},
                           status=alert_type.upper() if alert_type != "not_found" else "INFO",
                           summary="diagnosis_domain_expiration_" + alert_type,
                           details=details[alert_type])

    def get_domain_expiration(self, domain):
        """
        Return the expiration datetime of a domain or None
        """

        p1 = Popen(['whois', domain], stdout=PIPE)
        p2 = Popen(['grep', 'Expir'], stdin=p1.stdout, stdout=PIPE)
        out, err = p2.communicate()
        out = out.decode("utf-8").split('\n')
        p1.terminate()
        #p2.terminate()

        for line in out:
            match = re.search(r'\d{4}-\d{2}-\d{2}', line)
            if match is not None:
                return datetime.strptime(match.group(), '%Y-%m-%d')
        return None

def main(args, env, loggers):
    return DNSRecordsDiagnoser(args, env, loggers).diagnose()
<|endoftext|>


<commit_before>
# This file has been generated using `bazel run scripts:pinned_browsers`

load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
load("//common/private:dmg_archive.bzl", "dmg_archive")
load("//common/private:drivers.bzl", "local_drivers")
load("//common/private:pkg_archive.bzl", "pkg_archive")

def pin_browsers():
    local_drivers()

    http_archive(
        name = "linux_firefox",
        url = "https://ftp.mozilla.org/pub/firefox/releases/91.0/linux-x86_64/en-US/firefox-91.0.tar.bz2",
        sha256 = "bced054543003caf29d0c93aa63359809bdf1f1fcbca92b82c57167fe94ca1c9",
        build_file_content = "exports_files([\"firefox\"])",
    )

    dmg_archive(
        name = "mac_firefox",
        url = "https://ftp.mozilla.org/pub/firefox/releases/91.0/mac/en-US/Firefox%2091.0.dmg",
        sha256 = "d26c43f490320b3f301578fa653a2235562dfed31ed12b9b1f7c4515da21a6fb",
        build_file_content = "exports_files([\"Firefox.app\"])",
    )

    http_archive(
        name = "linux_geckodriver",
        url = "https://github.com/mozilla/geckodriver/releases/download/v0.29.1/geckodriver-v0.29.1-linux64.tar.gz",
        sha256 = "ec164910a3de7eec71e596bd2a1814ae27ba4c9d112b611680a6470dbe2ce27b",
        build_file_content = "exports_files([\"geckodriver\"])",
    )

    http_archive(
        name = "mac_geckodriver",
        url = "https://github.com/mozilla/geckodriver/releases/download/v0.29.1/geckodriver-v0.29.1-macos.tar.gz",
        sha256 = "9929c804ad0157ca13fdafca808866c88815b658e7059280a9f08f7e70364963",
        build_file_content = "exports_files([\"geckodriver\"])",
    )

    pkg_archive(
        name = "mac_edge",
        url = "https://officecdn-microsoft-com.akamaized.net/pr/C1297A47-86C4-4C1F-97FA-950631F94777/MacAutoupdate/MicrosoftEdge-92.0.902.67.pkg?platform=Mac&Consent=0&channel=Stable",
        sha256 = "afb14750572b49c2ad5f25b637ffb3d637107e3db094b97b14a386a9f85a9af7",
        move = {
            "MicrosoftEdge-92.0.902.67.pkg/Payload/Microsoft Edge.app": "Edge.app",
        },
        build_file_content = "exports_files([\"Edge.app\"])",
    )

    http_archive(
        name = "mac_edgedriver",
        url = "https://msedgedriver.azureedge.net/92.0.902.67/edgedriver_mac64.zip",
        sha256 = "a5afad438001fe92ee71e539fdb1dd20c72a9cec7d08c01ec6516215e4abf181",
        build_file_content = "exports_files([\"msedgedriver\"])",
    )

    http_archive(
        name = "linux_chrome",
        url = "https://storage.googleapis.com/chromium-browser-snapshots/Linux_x64/885174/chrome-linux.zip",
        sha256 = "7a7781b13a9296f67aefda073b056446ef850c01c5b56e90dde1f3cfb1bde806",
        build_file_content = "exports_files([\"chrome-linux\"])",
    )

    http_archive(
        name = "mac_chrome",
        url = "https://storage.googleapis.com/chromium-browser-snapshots/Mac/885174/chrome-mac.zip",
        sha256 = "8aab4d6052d42c660afd18b100fe7c0b4df6d1d523be953dce6601b873b6f205",
        strip_prefix = "chrome-mac",
        build_file_content = "exports_files([\"Chromium.app\"])",
    )

    http_archive(
        name = "linux_chromedriver",
        url = "https://chromedriver.storage.googleapis.com/92.0.4515.107/chromedriver_linux64.zip",
        sha256 = "c3cfba55df31a0a0d62d901049e91ee1ec4e38dd165e752409b7430ae59fcc8b",
        build_file_content = "exports_files([\"chromedriver\"])",
    )

    http_archive(
        name = "mac_chromedriver",
        url = "https://chromedriver.storage.googleapis.com/92.0.4515.107/chromedriver_mac64.zip",
        sha256 = "97f9a5733ef83a001922fd14ff2f5ac5b03c8a16aa02986ecca300cfbffaa03e",
        build_file_content = "exports_files([\"chromedriver\"])",
    )

<commit_msg>Bump browsers: Firefox to 91.0.2, Edge to 92.0.902.84<commit_after>
# This file has been generated using `bazel run scripts:pinned_browsers`

load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
load("//common/private:dmg_archive.bzl", "dmg_archive")
load("//common/private:drivers.bzl", "local_drivers")
load("//common/private:pkg_archive.bzl", "pkg_archive")

def pin_browsers():
    local_drivers()

    http_archive(
        name = "linux_firefox",
        url = "https://ftp.mozilla.org/pub/firefox/releases/91.0.2/linux-x86_64/en-US/firefox-91.0.2.tar.bz2",
        sha256 = "9eaac9c88ff4696228292590b65ab2fd1b0d98b7a1edf5a21abc11b7803a046d",
        build_file_content = "exports_files([\"firefox\"])",
    )

    dmg_archive(
        name = "mac_firefox",
        url = "https://ftp.mozilla.org/pub/firefox/releases/91.0.2/mac/en-US/Firefox%2091.0.2.dmg",
        sha256 = "4147fff1b176a659d6d4d928efe6ec98af313533508809264d78cc83c93cab30",
        build_file_content = "exports_files([\"Firefox.app\"])",
    )

    http_archive(
        name = "linux_geckodriver",
        url = "https://github.com/mozilla/geckodriver/releases/download/v0.29.1/geckodriver-v0.29.1-linux64.tar.gz",
        sha256 = "ec164910a3de7eec71e596bd2a1814ae27ba4c9d112b611680a6470dbe2ce27b",
        build_file_content = "exports_files([\"geckodriver\"])",
    )

    http_archive(
        name = "mac_geckodriver",
        url = "https://github.com/mozilla/geckodriver/releases/download/v0.29.1/geckodriver-v0.29.1-macos.tar.gz",
        sha256 = "9929c804ad0157ca13fdafca808866c88815b658e7059280a9f08f7e70364963",
        build_file_content = "exports_files([\"geckodriver\"])",
    )

    pkg_archive(
        name = "mac_edge",
        url = "https://officecdn-microsoft-com.akamaized.net/pr/C1297A47-86C4-4C1F-97FA-950631F94777/MacAutoupdate/MicrosoftEdge-92.0.902.84.pkg?platform=Mac&Consent=0&channel=Stable",
        sha256 = "1cb38e5d07392a3e4598e8644842e97e5583324c1c53df77f4c3c60fde726c2b",
        move = {
            "MicrosoftEdge-92.0.902.84.pkg/Payload/Microsoft Edge.app": "Edge.app",
        },
        build_file_content = "exports_files([\"Edge.app\"])",
    )

    http_archive(
        name = "mac_edgedriver",
        url = "https://msedgedriver.azureedge.net/92.0.902.84/edgedriver_mac64.zip",
        sha256 = "c4bd74dc6e1026f6cb6beaf8fa0ecc2e6208ac1e604460432d14e6de90b0dabe",
        build_file_content = "exports_files([\"msedgedriver\"])",
    )

    http_archive(
        name = "linux_chrome",
        url = "https://storage.googleapis.com/chromium-browser-snapshots/Linux_x64/885174/chrome-linux.zip",
        sha256 = "7a7781b13a9296f67aefda073b056446ef850c01c5b56e90dde1f3cfb1bde806",
        build_file_content = "exports_files([\"chrome-linux\"])",
    )

    http_archive(
        name = "mac_chrome",
        url = "https://storage.googleapis.com/chromium-browser-snapshots/Mac/885174/chrome-mac.zip",
        sha256 = "8aab4d6052d42c660afd18b100fe7c0b4df6d1d523be953dce6601b873b6f205",
        strip_prefix = "chrome-mac",
        build_file_content = "exports_files([\"Chromium.app\"])",
    )

    http_archive(
        name = "linux_chromedriver",
        url = "https://chromedriver.storage.googleapis.com/92.0.4515.107/chromedriver_linux64.zip",
        sha256 = "c3cfba55df31a0a0d62d901049e91ee1ec4e38dd165e752409b7430ae59fcc8b",
        build_file_content = "exports_files([\"chromedriver\"])",
    )

    http_archive(
        name = "mac_chromedriver",
        url = "https://chromedriver.storage.googleapis.com/92.0.4515.107/chromedriver_mac64.zip",
        sha256 = "97f9a5733ef83a001922fd14ff2f5ac5b03c8a16aa02986ecca300cfbffaa03e",
        build_file_content = "exports_files([\"chromedriver\"])",
    )

<|endoftext|>