<commit_before>from couchdbkit import ResourceNotFound
from corehq.apps.fixtures.exceptions import FixtureUploadError, ExcelMalformatException, \
    DuplicateFixtureTagException, FixtureAPIException
from django.core.validators import ValidationError
from django.utils.translation import ugettext as _, ugettext_noop
from corehq.apps.fixtures.models import FixtureTypeField, FixtureDataType, FixtureDataItem, FixtureItemField, \
    FieldList
from corehq.apps.users.bulkupload import GroupMemoizer
from corehq.apps.users.models import CommCareUser
from corehq.apps.users.util import normalize_username
from dimagi.utils.couch.bulk import CouchTransaction
from dimagi.utils.excel import WorkbookJSONReader, WorksheetNotFound
from soil import DownloadBase

DELETE_HEADER = "Delete(Y/N)"
FAILURE_MESSAGES = {
    "has_no_column": ugettext_noop(
        "Workbook 'types' has no column '{column_name}'."
    ),
    "duplicate_tag": ugettext_noop(
        "Lookup-tables should have unique 'table_id'. There are two rows with table_id '{tag}' in 'types' sheet."
    ),
    "has_no_field_column": ugettext_noop(
        "Excel-sheet '{tag}' does not contain the column '{field}' "
        "as specified in its 'types' definition"
    ),
    "has_extra_column": ugettext_noop(
        "Excel-sheet '{tag}' has an extra column"
        "'{field}' that's not defined in its 'types' definition"
    ),
    "wrong_property_syntax": ugettext_noop(
        "Properties should be specified as 'field 1: property 1'. In 'types' sheet, "
        "'{prop_key}' for field '{field}' is not correctly formatted"
    ),
    "sheet_has_no_property": ugettext_noop(
        "Excel-sheet '{tag}' does not contain property "
        "'{property}' of the field '{field}' as specified in its 'types' definition"
    ),
    "sheet_has_extra_property": ugettext_noop(
        "Excel-sheet '{tag}'' has an extra property "
        "'{property}' for the field '{field}' that's not defined in its 'types' definition. "
        "Re-check the formatting"
    ),
    "invalid_field_with_property": ugettext_noop(
        "Fields with attributes should be numbered as 'field: {field} integer"
    ),
    "invalid_property": ugettext_noop(
        "Attribute should be written as '{field}: {prop} interger'"
    ),
    "wrong_field_property_combos": ugettext_noop(
        "Number of values for field '{field}' and attribute '{prop}' should be same"
    ),
    "replace_with_UID": ugettext_noop(
        "Rows shouldn't contain UIDs while using replace option. Excel sheet '{tag}' contains UID in a row."
    ),
}

class FixtureUploadResult(object):
    """
    Helper structure for handling the results of a fixture upload.
    """
    def __init__(self):
        self.success = True
        self.unknown_groups = []
        self.unknown_users = []
        self.messages = []
        self.errors = []
        self.number_of_fixtures = 0

class FixtureTableDefinitition(object):

    def __init__(self, table_id, fields, is_global, uid, delete):
        self.table_id = table_id
        self.fields = fields
        self.is_global = is_global
        self.uid = uid
        self.delete = delete

    @classmethod
    def from_row(cls, row_dict):
        tag = row_dict.get('table_id') or row_dict.get('tag')
        if tag is None:
            raise ExcelMalformatException(_(FAILURE_MESSAGES['has_no_column']).format(column_name='table_id'))

        field_names = row_dict.get('field')
        if field_names is None:
            raise ExcelMalformatException(_(FAILURE_MESSAGES['has_no_column']).format(column_name='table_id'))

        def _get_field_properties(field, prop_key):
            if row_dict.has_key(prop_key):
                try:
                    return row_dict[prop_key]["property"]
                except KeyError:
                    error_message = _(FAILURE_MESSAGES["wrong_property_syntax"]).format(
                        prop_key=prop_key,
                        field=field
                    )
                    raise ExcelMalformatException(error_message)
            else:
                return []

        fields = [
            FixtureTypeField(
                field_name=field,
                properties=_get_field_properties(field, 'field {count}'.format(count=i + 1))
            ) for i, field in enumerate(field_names)
        ]
        return FixtureTableDefinitition(
            table_id=tag,
            fields=fields,
            is_global=row_dict.get('is_global', False),
            uid=row_dict.get('UID'),
            delete=(row_dict.get(DELETE_HEADER) or '').lower() == 'y',
        )

class FixtureWorkbook(object):
    """
    Helper class for working with the fixture workbook
    """

    def __init__(self, file_or_filename):
        try:
            self.workbook = WorkbookJSONReader(file_or_filename)
        except AttributeError:
            raise FixtureUploadError(_("Error processing your Excel (.xlsx) file"))
        except Exception:
            raise FixtureUploadError(_("Invalid file-format. Please upload a valid xlsx file."))

    def get_types_sheet(self):
        try:
            return self.workbook.get_worksheet(title='types')
        except WorksheetNotFound as e:
            raise FixtureUploadError(_("Workbook does not contain a sheet called '%(title)s'") % {'title': e.title})

    def get_data_sheet(self, data_type):
        return self.workbook.get_worksheet(data_type.tag)

    def get_all_type_sheets(self):
        type_sheets = []
        seen_tags = set()
        for number_of_fixtures, dt in enumerate(self.get_types_sheet()):
            table_definition = FixtureTableDefinitition.from_row(dt)
            if table_definition.table_id in seen_tags:
                raise DuplicateFixtureTagException(
                    _(FAILURE_MESSAGES['duplicate_tag']).format(tag=table_definition.table_id))

            seen_tags.add(table_definition.table_id)
            type_sheets.append(table_definition)
        return type_sheets

    def validate(self):
        self.get_types_sheet()
        self.get_all_type_sheets()

def safe_fixture_upload(domain, file_ref, replace, task=None):
    try:
        return do_fixture_upload(domain, file_ref, replace, task)
    except FixtureUploadError as e:
        result = FixtureUploadResult()
        result.success = False
        result.errors.append(unicode(e))
        return result

def do_fixture_upload(domain, file_ref, replace, task=None):
    workbook = get_workbook(file_ref.get_filename())
    try:
        return run_upload(domain, workbook, replace=replace, task=task)
    except WorksheetNotFound as e:
        raise FixtureUploadError(_("Workbook does not contain a sheet called '%(title)s'") % {'title': e.title})
    except ExcelMalformatException as e:
        raise FixtureUploadError(_("Uploaded excel file has following formatting-problems: '%(e)s'") % {'e': e})
    except FixtureAPIException as e:
        raise FixtureUploadError(unicode(e))
    except Exception:
        raise FixtureUploadError(_("Fixture upload failed for some reason and we have noted this failure. "
                                   "Please make sure the excel file is correctly formatted and try again."))

def validate_file_format(file_or_filename):
    """
    Does basic validation on the uploaded file. Raises a FixtureUploadError if
    something goes wrong.
    """
    workbook = FixtureWorkbook(file_or_filename)
    workbook.validate()

def get_workbook(file_or_filename):
    return FixtureWorkbook(file_or_filename)

def run_upload(domain, workbook, replace=False, task=None):
    return_val = FixtureUploadResult()
    group_memoizer = GroupMemoizer(domain)

    def diff_lists(list_a, list_b):
        set_a = set(list_a)
        set_b = set(list_b)
        not_in_b = set_a.difference(set_b)
        not_in_a = set_a.difference(set_a)
        return list(not_in_a), list(not_in_b)

    with CouchTransaction() as transaction:
        type_sheets = workbook.get_all_type_sheets()
        total_tables = len(type_sheets)
        return_val.number_of_fixtures = total_tables

        def _update_progress(table_count, item_count, items_in_table):
            if task:
                processed = table_count * 10 + (10. * item_count / items_in_table)
                DownloadBase.set_progress(task, processed, 10 * total_tables)

        for table_number, table_def in enumerate(type_sheets):
            tag = table_def.table_id
            new_data_type = FixtureDataType(
                domain=domain,
                is_global=table_def.is_global,
                tag=table_def.table_id,
                fields=table_def.fields,
            )
            try:
                tagged_fdt = FixtureDataType.fixture_tag_exists(domain, tag)
                if tagged_fdt:
                    data_type = tagged_fdt
                # support old usage with 'UID'
                elif table_def.uid:
                    data_type = FixtureDataType.get(table_def.uid)
                else:
                    data_type = new_data_type

                if replace and data_type != new_data_type:
                    data_type.recursive_delete(transaction)
                    data_type = new_data_type

                data_type.fields = table_def.fields
                data_type.is_global = table_def.is_global
                assert data_type.doc_type == FixtureDataType._doc_type
                if data_type.domain != domain:
                    data_type = new_data_type
                    return_val.errors.append(_("'%(UID)s' is not a valid UID. But the new type is created.") % {'UID': table_def.uid})
                if table_def.delete:
                    data_type.recursive_delete(transaction)
                    continue
            except (ResourceNotFound, KeyError) as e:
                data_type = new_data_type
            transaction.save(data_type)

            data_items = list(workbook.get_data_sheet(data_type))
            items_in_table = len(data_items)
            for sort_key, di in enumerate(data_items):
                _update_progress(table_number, sort_key, items_in_table)
                # Check that type definitions in 'types' sheet vs corresponding columns in the item-sheet MATCH
                item_fields_list = di['field'].keys()
                not_in_sheet, not_in_types = diff_lists(item_fields_list, data_type.fields_without_attributes)
                if len(not_in_sheet) > 0:
                    error_message = _(FAILURE_MESSAGES["has_no_field_column"]).format(tag=tag, field=not_in_sheet[0])
                    raise ExcelMalformatException(error_message)
                if len(not_in_types) > 0:
                    error_message = _(FAILURE_MESSAGES["has_extra_column"]).format(tag=tag, field=not_in_types[0])
                    raise ExcelMalformatException(error_message)

                # check that properties in 'types' sheet vs item-sheet MATCH
                for field in data_type.fields:
                    if len(field.properties) > 0:
                        sheet_props = di.get(field.field_name, {})
                        sheet_props_list = sheet_props.keys()
                        type_props = field.properties
                        not_in_sheet, not_in_types = diff_lists(sheet_props_list, type_props)
                        if len(not_in_sheet) > 0:
                            error_message = _(FAILURE_MESSAGES["sheet_has_no_property"]).format(
                                tag=tag,
                                property=not_in_sheet[0],
                                field=field.field_name
                            )
                            raise ExcelMalformatException(error_message)
                        if len(not_in_types) > 0:
                            error_message = _(FAILURE_MESSAGES["sheet_has_extra_property"]).format(
                                tag=tag,
                                property=not_in_types[0],
                                field=field.field_name
                            )
                            raise ExcelMalformatException(error_message)
                        # check that fields with properties are numbered
                        if type(di['field'][field.field_name]) != list:
                            error_message = _(FAILURE_MESSAGES["invalid_field_with_property"]).format(field=field.field_name)
                            raise ExcelMalformatException(error_message)
                        field_prop_len = len(di['field'][field.field_name])
                        for prop in sheet_props:
                            if type(sheet_props[prop]) != list:
                                error_message = _(FAILURE_MESSAGES["invalid_property"]).format(
                                    field=field.field_name,
                                    prop=prop
                                )
                                raise ExcelMalformatException(error_message)
                            if len(sheet_props[prop]) != field_prop_len:
                                error_message = _(FAILURE_MESSAGES["wrong_field_property_combos"]).format(
                                    field=field.field_name,
                                    prop=prop
                                )
                                raise ExcelMalformatException(error_message)

                # excel format check should have been covered by this line. Can make assumptions about data now
                type_fields = data_type.fields
                item_fields = {}
                for field in type_fields:
                    # if field doesn't have properties
                    if len(field.properties) == 0:
                        item_fields[field.field_name] = FieldList(
                            field_list=[FixtureItemField(
                                # using unicode here, to cast ints, and multi-language strings
                                field_value=unicode(di['field'][field.field_name]),
                                properties={}
                            )]
                        )
                    else:
                        field_list = []
                        field_prop_combos = di['field'][field.field_name]
                        prop_combo_len = len(field_prop_combos)
                        prop_dict = di[field.field_name]
                        for x in range(0, prop_combo_len):
                            fix_item_field = FixtureItemField(
                                field_value=unicode(field_prop_combos[x]),
                                properties={prop: unicode(prop_dict[prop][x]) for prop in prop_dict}
                            )
                            field_list.append(fix_item_field)
                        item_fields[field.field_name] = FieldList(
                            field_list=field_list
                        )

                new_data_item = FixtureDataItem(
                    domain=domain,
                    data_type_id=data_type.get_id,
                    fields=item_fields,
                    sort_key=sort_key
                )
                try:
                    if di['UID'] and not replace:
                        old_data_item = FixtureDataItem.get(di['UID'])
                    else:
                        old_data_item = new_data_item
                        pass
                    old_data_item.fields = item_fields
                    if old_data_item.domain != domain or not old_data_item.data_type_id == data_type.get_id:
                        old_data_item = new_data_item
                        return_val.errors.append(_("'%(UID)s' is not a valid UID. But the new item is created.") % {'UID': di['UID']})
                    assert old_data_item.doc_type == FixtureDataItem._doc_type
                    if di[DELETE_HEADER] == "Y" or di[DELETE_HEADER] == "y":
                        old_data_item.recursive_delete(transaction)
                        continue
                except (ResourceNotFound, KeyError):
                    old_data_item = new_data_item
                transaction.save(old_data_item)

                old_groups = old_data_item.groups
                for group in old_groups:
                    old_data_item.remove_group(group)
                old_users = old_data_item.users
                for user in old_users:
                    old_data_item.remove_user(user)

                for group_name in di.get('group', []):
                    group = group_memoizer.by_name(group_name)
                    if group:
                        old_data_item.add_group(group, transaction=transaction)
                    else:
                        return_val.errors.append(_("Unknown group: '%(name)s'. But the row is successfully added") % {'name': group_name})

                for raw_username in di.get('user', []):
                    try:
                        username = normalize_username(str(raw_username), domain)
                    except ValidationError:
                        return_val.errors.append(_("Invalid username: '%(name)s'. Row is not added") % {'name': raw_username})
                        continue
                    user = CommCareUser.get_by_username(username)
                    if user:
                        old_data_item.add_user(user)
                    else:
                        return_val.errors.append(_("Unknown user: '%(name)s'. But the row is successfully added") % {'name': raw_username})

    return return_val
<commit_msg>Catch improperly formatted column headers<commit_after>from couchdbkit import ResourceNotFound
from corehq.apps.fixtures.exceptions import FixtureUploadError, ExcelMalformatException, \
    DuplicateFixtureTagException, FixtureAPIException
from django.core.validators import ValidationError
from django.utils.translation import ugettext as _, ugettext_noop
from corehq.apps.fixtures.models import FixtureTypeField, FixtureDataType, FixtureDataItem, FixtureItemField, \
    FieldList
from corehq.apps.users.bulkupload import GroupMemoizer
from corehq.apps.users.models import CommCareUser
from corehq.apps.users.util import normalize_username
from dimagi.utils.couch.bulk import CouchTransaction
from dimagi.utils.excel import WorkbookJSONReader, WorksheetNotFound
from soil import DownloadBase

DELETE_HEADER = "Delete(Y/N)"
FAILURE_MESSAGES = {
    "has_no_column": ugettext_noop(
        "Workbook 'types' has no column '{column_name}'."
    ),
    "duplicate_tag": ugettext_noop(
        "Lookup-tables should have unique 'table_id'. There are two rows with table_id '{tag}' in 'types' sheet."
    ),
    "has_no_field_column": ugettext_noop(
        "Excel-sheet '{tag}' does not contain the column '{field}' "
        "as specified in its 'types' definition"
    ),
    "has_extra_column": ugettext_noop(
        "Excel-sheet '{tag}' has an extra column"
        "'{field}' that's not defined in its 'types' definition"
    ),
    "wrong_property_syntax": ugettext_noop(
        "Properties should be specified as 'field 1: property 1'. In 'types' sheet, "
        "'{prop_key}' is not correctly formatted"
    ),
    "sheet_has_no_property": ugettext_noop(
        "Excel-sheet '{tag}' does not contain property "
        "'{property}' of the field '{field}' as specified in its 'types' definition"
    ),
    "sheet_has_extra_property": ugettext_noop(
        "Excel-sheet '{tag}'' has an extra property "
        "'{property}' for the field '{field}' that's not defined in its 'types' definition. "
        "Re-check the formatting"
    ),
    "invalid_field_with_property": ugettext_noop(
        "Fields with attributes should be numbered as 'field: {field} integer"
    ),
    "invalid_property": ugettext_noop(
        "Attribute should be written as '{field}: {prop} interger'"
    ),
    "wrong_field_property_combos": ugettext_noop(
        "Number of values for field '{field}' and attribute '{prop}' should be same"
    ),
    "replace_with_UID": ugettext_noop(
        "Rows shouldn't contain UIDs while using replace option. Excel sheet '{tag}' contains UID in a row."
    ),
}

class FixtureUploadResult(object):
    """
    Helper structure for handling the results of a fixture upload.
    """
    def __init__(self):
        self.success = True
        self.unknown_groups = []
        self.unknown_users = []
        self.messages = []
        self.errors = []
        self.number_of_fixtures = 0

class FixtureTableDefinitition(object):

    def __init__(self, table_id, fields, is_global, uid, delete):
        self.table_id = table_id
        self.fields = fields
        self.is_global = is_global
        self.uid = uid
        self.delete = delete

    @classmethod
    def from_row(cls, row_dict):
        tag = row_dict.get('table_id') or row_dict.get('tag')
        if tag is None:
            raise ExcelMalformatException(_(FAILURE_MESSAGES['has_no_column']).format(column_name='table_id'))

        field_names = row_dict.get('field')
        if field_names is None:
            raise ExcelMalformatException(_(FAILURE_MESSAGES['has_no_column']).format(column_name='table_id'))

        def _get_field_properties(prop_key):
            if row_dict.has_key(prop_key):
                try:
                    properties = row_dict[prop_key]["property"]
                    assert isinstance(properties, list)
                except (KeyError, AssertionError):
                    error_message = _(FAILURE_MESSAGES["wrong_property_syntax"]).format(
                        prop_key=prop_key,
                    )
                    raise ExcelMalformatException(error_message)
                else:
                    return properties
            else:
                return []

        fields = [
            FixtureTypeField(
                field_name=field,
                properties=_get_field_properties('field {count}'.format(count=i + 1))
            ) for i, field in enumerate(field_names)
        ]
        return FixtureTableDefinitition(
            table_id=tag,
            fields=fields,
            is_global=row_dict.get('is_global', False),
            uid=row_dict.get('UID'),
            delete=(row_dict.get(DELETE_HEADER) or '').lower() == 'y',
        )

class FixtureWorkbook(object):
    """
    Helper class for working with the fixture workbook
    """

    def __init__(self, file_or_filename):
        try:
            self.workbook = WorkbookJSONReader(file_or_filename)
        except AttributeError:
            raise FixtureUploadError(_("Error processing your Excel (.xlsx) file"))
        except Exception:
            raise FixtureUploadError(_("Invalid file-format. Please upload a valid xlsx file."))

    def get_types_sheet(self):
        try:
            return self.workbook.get_worksheet(title='types')
        except WorksheetNotFound as e:
            raise FixtureUploadError(_("Workbook does not contain a sheet called '%(title)s'") % {'title': e.title})

    def get_data_sheet(self, data_type):
        return self.workbook.get_worksheet(data_type.tag)

    def get_all_type_sheets(self):
        type_sheets = []
        seen_tags = set()
        for number_of_fixtures, dt in enumerate(self.get_types_sheet()):
            table_definition = FixtureTableDefinitition.from_row(dt)
            if table_definition.table_id in seen_tags:
                raise DuplicateFixtureTagException(
                    _(FAILURE_MESSAGES['duplicate_tag']).format(tag=table_definition.table_id))

            seen_tags.add(table_definition.table_id)
            type_sheets.append(table_definition)
        return type_sheets

    def validate(self):
        self.get_types_sheet()
        self.get_all_type_sheets()

def safe_fixture_upload(domain, file_ref, replace, task=None):
    try:
        return do_fixture_upload(domain, file_ref, replace, task)
    except FixtureUploadError as e:
        result = FixtureUploadResult()
        result.success = False
        result.errors.append(unicode(e))
        return result

def do_fixture_upload(domain, file_ref, replace, task=None):
    workbook = get_workbook(file_ref.get_filename())
    try:
        return run_upload(domain, workbook, replace=replace, task=task)
    except WorksheetNotFound as e:
        raise FixtureUploadError(_("Workbook does not contain a sheet called '%(title)s'") % {'title': e.title})
    except ExcelMalformatException as e:
        raise FixtureUploadError(_("Uploaded excel file has following formatting-problems: '%(e)s'") % {'e': e})
    except FixtureAPIException as e:
        raise FixtureUploadError(unicode(e))
    except Exception:
        raise FixtureUploadError(_("Fixture upload failed for some reason and we have noted this failure. "
                                   "Please make sure the excel file is correctly formatted and try again."))

def validate_file_format(file_or_filename):
    """
    Does basic validation on the uploaded file. Raises a FixtureUploadError if
    something goes wrong.
    """
    workbook = FixtureWorkbook(file_or_filename)
    workbook.validate()

def get_workbook(file_or_filename):
    return FixtureWorkbook(file_or_filename)

def run_upload(domain, workbook, replace=False, task=None):
    return_val = FixtureUploadResult()
    group_memoizer = GroupMemoizer(domain)

    def diff_lists(list_a, list_b):
        set_a = set(list_a)
        set_b = set(list_b)
        not_in_b = set_a.difference(set_b)
        not_in_a = set_a.difference(set_a)
        return list(not_in_a), list(not_in_b)

    with CouchTransaction() as transaction:
        type_sheets = workbook.get_all_type_sheets()
        total_tables = len(type_sheets)
        return_val.number_of_fixtures = total_tables

        def _update_progress(table_count, item_count, items_in_table):
            if task:
                processed = table_count * 10 + (10. * item_count / items_in_table)
                DownloadBase.set_progress(task, processed, 10 * total_tables)

        for table_number, table_def in enumerate(type_sheets):
            tag = table_def.table_id
            new_data_type = FixtureDataType(
                domain=domain,
                is_global=table_def.is_global,
                tag=table_def.table_id,
                fields=table_def.fields,
            )
            try:
                tagged_fdt = FixtureDataType.fixture_tag_exists(domain, tag)
                if tagged_fdt:
                    data_type = tagged_fdt
                # support old usage with 'UID'
                elif table_def.uid:
                    data_type = FixtureDataType.get(table_def.uid)
                else:
                    data_type = new_data_type

                if replace and data_type != new_data_type:
                    data_type.recursive_delete(transaction)
                    data_type = new_data_type

                data_type.fields = table_def.fields
                data_type.is_global = table_def.is_global
                assert data_type.doc_type == FixtureDataType._doc_type
                if data_type.domain != domain:
                    data_type = new_data_type
                    return_val.errors.append(_("'%(UID)s' is not a valid UID. But the new type is created.") % {'UID': table_def.uid})
                if table_def.delete:
                    data_type.recursive_delete(transaction)
                    continue
            except (ResourceNotFound, KeyError) as e:
                data_type = new_data_type
            transaction.save(data_type)

            data_items = list(workbook.get_data_sheet(data_type))
            items_in_table = len(data_items)
            for sort_key, di in enumerate(data_items):
                _update_progress(table_number, sort_key, items_in_table)
                # Check that type definitions in 'types' sheet vs corresponding columns in the item-sheet MATCH
                item_fields_list = di['field'].keys()
                not_in_sheet, not_in_types = diff_lists(item_fields_list, data_type.fields_without_attributes)
                if len(not_in_sheet) > 0:
                    error_message = _(FAILURE_MESSAGES["has_no_field_column"]).format(tag=tag, field=not_in_sheet[0])
                    raise ExcelMalformatException(error_message)
                if len(not_in_types) > 0:
                    error_message = _(FAILURE_MESSAGES["has_extra_column"]).format(tag=tag, field=not_in_types[0])
                    raise ExcelMalformatException(error_message)

                # check that properties in 'types' sheet vs item-sheet MATCH
                for field in data_type.fields:
                    if len(field.properties) > 0:
                        sheet_props = di.get(field.field_name, {})
                        sheet_props_list = sheet_props.keys()
                        type_props = field.properties
                        not_in_sheet, not_in_types = diff_lists(sheet_props_list, type_props)
                        if len(not_in_sheet) > 0:
                            error_message = _(FAILURE_MESSAGES["sheet_has_no_property"]).format(
                                tag=tag,
                                property=not_in_sheet[0],
                                field=field.field_name
                            )
                            raise ExcelMalformatException(error_message)
                        if len(not_in_types) > 0:
                            error_message = _(FAILURE_MESSAGES["sheet_has_extra_property"]).format(
                                tag=tag,
                                property=not_in_types[0],
                                field=field.field_name
                            )
                            raise ExcelMalformatException(error_message)
                        # check that fields with properties are numbered
                        if type(di['field'][field.field_name]) != list:
                            error_message = _(FAILURE_MESSAGES["invalid_field_with_property"]).format(field=field.field_name)
                            raise ExcelMalformatException(error_message)
                        field_prop_len = len(di['field'][field.field_name])
                        for prop in sheet_props:
                            if type(sheet_props[prop]) != list:
                                error_message = _(FAILURE_MESSAGES["invalid_property"]).format(
                                    field=field.field_name,
                                    prop=prop
                                )
                                raise ExcelMalformatException(error_message)
                            if len(sheet_props[prop]) != field_prop_len:
                                error_message = _(FAILURE_MESSAGES["wrong_field_property_combos"]).format(
                                    field=field.field_name,
                                    prop=prop
                                )
                                raise ExcelMalformatException(error_message)

                # excel format check should have been covered by this line. Can make assumptions about data now
                type_fields = data_type.fields
                item_fields = {}
                for field in type_fields:
                    # if field doesn't have properties
                    if len(field.properties) == 0:
                        item_fields[field.field_name] = FieldList(
                            field_list=[FixtureItemField(
                                # using unicode here, to cast ints, and multi-language strings
                                field_value=unicode(di['field'][field.field_name]),
                                properties={}
                            )]
                        )
                    else:
                        field_list = []
                        field_prop_combos = di['field'][field.field_name]
                        prop_combo_len = len(field_prop_combos)
                        prop_dict = di[field.field_name]
                        for x in range(0, prop_combo_len):
                            fix_item_field = FixtureItemField(
                                field_value=unicode(field_prop_combos[x]),
                                properties={prop: unicode(prop_dict[prop][x]) for prop in prop_dict}
                            )
                            field_list.append(fix_item_field)
                        item_fields[field.field_name] = FieldList(
                            field_list=field_list
                        )

                new_data_item = FixtureDataItem(
                    domain=domain,
                    data_type_id=data_type.get_id,
                    fields=item_fields,
                    sort_key=sort_key
                )
                try:
                    if di['UID'] and not replace:
                        old_data_item = FixtureDataItem.get(di['UID'])
                    else:
                        old_data_item = new_data_item
                        pass
                    old_data_item.fields = item_fields
                    if old_data_item.domain != domain or not old_data_item.data_type_id == data_type.get_id:
                        old_data_item = new_data_item
                        return_val.errors.append(_("'%(UID)s' is not a valid UID. But the new item is created.") % {'UID': di['UID']})
                    assert old_data_item.doc_type == FixtureDataItem._doc_type
                    if di[DELETE_HEADER] == "Y" or di[DELETE_HEADER] == "y":
                        old_data_item.recursive_delete(transaction)
                        continue
                except (ResourceNotFound, KeyError):
                    old_data_item = new_data_item
                transaction.save(old_data_item)

                old_groups = old_data_item.groups
                for group in old_groups:
                    old_data_item.remove_group(group)
                old_users = old_data_item.users
                for user in old_users:
                    old_data_item.remove_user(user)

                for group_name in di.get('group', []):
                    group = group_memoizer.by_name(group_name)
                    if group:
                        old_data_item.add_group(group, transaction=transaction)
                    else:
                        return_val.errors.append(_("Unknown group: '%(name)s'. But the row is successfully added") % {'name': group_name})

                for raw_username in di.get('user', []):
                    try:
                        username = normalize_username(str(raw_username), domain)
                    except ValidationError:
                        return_val.errors.append(_("Invalid username: '%(name)s'. Row is not added") % {'name': raw_username})
                        continue
                    user = CommCareUser.get_by_username(username)
                    if user:
                        old_data_item.add_user(user)
                    else:
                        return_val.errors.append(_("Unknown user: '%(name)s'. But the row is successfully added") % {'name': raw_username})

    return return_val
<|endoftext|>


<commit_before>__version__ = "0.4.0"
<commit_msg>Bump version to 0.4.1<commit_after>__version__ = "0.4.1"
<|endoftext|>


<commit_before># -*- encoding: UTF-8 -*-
from cvn import settings as stCVN

def parse_scope(treeXML):
    dataCVN = {}
    if treeXML:
        dataCVN['ambito'] = unicode(stCVN.SCOPE[treeXML.find(
            'Type/Item').text.strip()])
        if dataCVN['ambito'] == u'Otros' and treeXML.find('Others/Item'):
            dataCVN['otro_ambito'] = unicode(treeXML.find(
                'Others/Item').text.strip())
    return dataCVN

def parse_duration(duration):
    """
         Format: P<num_years>Y<num_months>M<num_days>D
    """
    dataCVN = {}
    number = ''
    for item in duration[1:]:
        if item.isdigit():
            number = item
        else:
            if item == 'Y':
                dataCVN['duracion_anyos'] = unicode(number)
            if item == 'M':
                dataCVN['duracion_meses'] = unicode(number)
            if item == 'D':
                dataCVN['duracion_dias'] = unicode(number)
    return dataCVN

def parse_nif(xml):
    nif = ''
    id_node = xml.find(
        'Agent/Identification/PersonalIdentification/OfficialId')
    nif_node = id_node.find('DNI/Item')
    if nif_node is None:
        nif_node = id_node.find('NIE/Item')

    if nif_node is not None:
        nif = nif_node.text.strip()

    return nif

def parse_authors(author_list):
    authors = ''
    for author in author_list:
        authorItem = ''
        if (author.find("GivenName/Item") and
           author.find("GivenName/Item").text):
            authorItem = unicode(author.find(
                "GivenName/Item").text.strip())
        if (author.find("FirstFamilyName/Item") and
           author.find("FirstFamilyName/Item").text):
            authorItem += u' ' + unicode(author.find(
                "FirstFamilyName/Item").text.strip())
        if (author.find("SecondFamilyName/Item") and
           author.find("SecondFamilyName/Item").text):
            authorItem += u' ' + unicode(
                author.find("SecondFamilyName/Item").text.strip())
        if author.find("Signature/Item").text:
            if authorItem:
                authors += unicode(
                    authorItem + ' (' + author.find(
                        "Signature/Item").text.strip() + '); ')
            else:
                authors += unicode(author.find(
                    "Signature/Item").text.strip() + '; ')
        else:
            authors += authorItem + '; '
    return authors[:-2]

def parse_produccion_type(xml):
    cvn_key = xml.find('CvnItemID/CVNPK/Item').text.strip()
    if not cvn_key in stCVN.FECYT_CODE:
        return ''
    return stCVN.FECYT_CODE[cvn_key]

def parse_produccion_subtype(xml):
    subtype = xml.find('Subtype/SubType1/Item')
    if subtype is None:
        return ''
    subtype = subtype.text.strip()
    if subtype in stCVN.FECYT_CODE_SUBTYPE:
        return stCVN.FECYT_CODE_SUBTYPE[subtype]
    return ''

def parse_publicacion_location(treeXML):
    data = {}
    if treeXML:
        volume = treeXML.find('Volume/Item')
        if volume is not None and volume.text is not None:
            data['volumen'] = volume.text.strip()
        number = treeXML.find('Number/Item')
        if number is not None and number.text is not None:
            data['numero'] = number.text.strip()
        page = treeXML.find('InitialPage/Item')
        if page is not None and page.text is not None:
            data['pagina_inicial'] = page.text.strip()
        page = treeXML.find('FinalPage/Item')
        if page is not None and page.text is not None:
            data['pagina_final'] = page.text.strip()
    return data
<commit_msg>Added comments with xml nodes needed in the parser functions<commit_after># -*- encoding: UTF-8 -*-
from cvn import settings as stCVN

def parse_scope(treeXML):
    '''Input: Scope node
       Example: CvnItem/Link/Scope'''
    dataCVN = {}
    if treeXML:
        dataCVN['ambito'] = unicode(stCVN.SCOPE[treeXML.find(
            'Type/Item').text.strip()])
        if dataCVN['ambito'] == u'Otros' and treeXML.find('Others/Item'):
            dataCVN['otro_ambito'] = unicode(treeXML.find(
                'Others/Item').text.strip())
    return dataCVN

def parse_duration(duration):
    '''Input: Duration/Item node
       Example: CvnItem/Date/Duration/Item
    '''
    dataCVN = {}
    number = ''
    for item in duration[1:]:
        if item.isdigit():
            number = item
        else:
            if item == 'Y':
                dataCVN['duracion_anyos'] = unicode(number)
            if item == 'M':
                dataCVN['duracion_meses'] = unicode(number)
            if item == 'D':
                dataCVN['duracion_dias'] = unicode(number)
    return dataCVN

def parse_nif(xml):
    '''Input: root node'''
    nif = ''
    id_node = xml.find(
        'Agent/Identification/PersonalIdentification/OfficialId')
    nif_node = id_node.find('DNI/Item')
    if nif_node is None:
        nif_node = id_node.find('NIE/Item')

    if nif_node is not None:
        nif = nif_node.text.strip()

    return nif

def parse_authors(author_list):
    '''Input: A list of Author nodes'''
    authors = ''
    for author in author_list:
        authorItem = ''
        if (author.find("GivenName/Item") and
           author.find("GivenName/Item").text):
            authorItem = unicode(author.find(
                "GivenName/Item").text.strip())
        if (author.find("FirstFamilyName/Item") and
           author.find("FirstFamilyName/Item").text):
            authorItem += u' ' + unicode(author.find(
                "FirstFamilyName/Item").text.strip())
        if (author.find("SecondFamilyName/Item") and
           author.find("SecondFamilyName/Item").text):
            authorItem += u' ' + unicode(
                author.find("SecondFamilyName/Item").text.strip())
        if author.find("Signature/Item").text:
            if authorItem:
                authors += unicode(
                    authorItem + ' (' + author.find(
                        "Signature/Item").text.strip() + '); ')
            else:
                authors += unicode(author.find(
                    "Signature/Item").text.strip() + '; ')
        else:
            authors += authorItem + '; '
    return authors[:-2]

def parse_produccion_type(xml):
    '''Input: CvnItem node'''
    cvn_key = xml.find('CvnItemID/CVNPK/Item').text.strip()
    if not cvn_key in stCVN.FECYT_CODE:
        return ''
    return stCVN.FECYT_CODE[cvn_key]

def parse_produccion_subtype(xml):
    '''Input: CvnItem node'''
    subtype = xml.find('Subtype/SubType1/Item')
    if subtype is None:
        return ''
    subtype = subtype.text.strip()
    if subtype in stCVN.FECYT_CODE_SUBTYPE:
        return stCVN.FECYT_CODE_SUBTYPE[subtype]
    return ''

def parse_publicacion_location(treeXML):
    '''Input: Location node'''
    data = {}
    if treeXML:
        volume = treeXML.find('Volume/Item')
        if volume is not None and volume.text is not None:
            data['volumen'] = volume.text.strip()
        number = treeXML.find('Number/Item')
        if number is not None and number.text is not None:
            data['numero'] = number.text.strip()
        page = treeXML.find('InitialPage/Item')
        if page is not None and page.text is not None:
            data['pagina_inicial'] = page.text.strip()
        page = treeXML.find('FinalPage/Item')
        if page is not None and page.text is not None:
            data['pagina_final'] = page.text.strip()
    return data
<|endoftext|>


<commit_before>from __future__ import division
import numpy as np
from numpy import inf, pi
import scipy.integrate
import scipy.misc
import warnings

from .CxDerivative import CxDeriv

def prod(C, f, df=None, phi=None, psi=None, absTol=1e-12, relTol=1e-12, divMin=5, divMax=10, m=2, method='quad', verbose=False, callback=None, integerTol=inf):
	r"""
	Compute the symmetric bilinear form used in (1.12) of [KB]

	.. math::

		<\phi,\psi> = \frac{1}{2i\pi} \oint_C \phi(z) \psi(z) \frac{f'(z)}{f(z)} dz.

    Parameters
    ----------
	C : Contour
		The enclosed_roots function returns the number of roots of f(z) within C
	f : function
		Function of a single variable f(x)
	df : function, optional
		Function of a single variable, df(x), providing the derivative of the function f(x)
		at the point x.  If not provided then df is approximated using a finite difference
		method.
	phi : function, optional
		Function of a single variable phi(x).  If not provided then phi=1.
	psi : function, optional
		Function of a single variable psi(x).  If not provided then psi=1.
	absTol : float, optional
		Absolute error tolerance.
	relTol : float, optional
		Relative error tolerance.
 	divMin : int, optional
 		Minimum number of divisions before the Romberg integration routine is allowed
 		to exit.  Only used if method='romb'.
	divMax : int, optional
		The maximum number of divisions before the Romberg integration routine of a
		path exits.  Only used if method='romb'.
    m : int, optional
    	Only used if df=None.  If method='romb' then m defines the stencil size for the
    	numerical differentiation of f, passed to numdifftools.fornberg.fd_derivative.
    	The stencil size is of 2*m+1 points in the interior, and 2*m+2 points for each
    	of the 2*m boundary points.  If instead method='quad' then m must is the order of
    	the error term in the Taylor approximation used which must be even.  The argument
    	order=m is passed to numdifftools.Derivative.
	method : {'quad', 'romb'}, optional
		If 'quad' then scipy.integrate.quad is used to perform the integral.  If 'romb'
		then Romberg integraion, using scipy.integrate.romb, is performed instead.

	References
	----------
	[KB] "Computing the zeros of analytic functions" by Peter Kravanja, Marc Van Barel, Springer 2000
	"""
	N = 1
	k = 0
	I = []
	integrandMax = []

	approx_df = False
	if df is None:
		approx_df = True
		import numdifftools.fornberg as ndf
		import numdifftools

	if method == 'romb':
		# XXX: Better way to characterise err than abs(I[-2] - I[-1])?
		while (len(I) < divMin or (abs(I[-2] - I[-1]) > absTol and abs(I[-2] - I[-1]) > relTol*abs(I[-1])) or abs(int(round(I[-1].real)) - I[-1].real) > integerTol or abs(I[-1].imag) > integerTol) and k < divMax:
			N = 2*N
			t = np.linspace(0,1,N+1)
			k = int(np.log2(len(t)-1))
			dt = t[1]-t[0]

			integrals = []
			for segment in C.segments:
				# compute/retrieve function evaluations
				fVal = segment.trapValues(f,k)

				if approx_df:
					### approximate df/dz with finite difference, see: numdifftools.fornberg
					used_m = m
					if 2*m+1 > len(t):
						# not enough points to accommodate stencil size
						# temporarily reduce m
						used_m = (len(t)-1)//2

					dfdt = ndf.fd_derivative(fVal, t, n=1, m=used_m)
					dfVal = dfdt/segment.dzdt(t)

				else:
					dfVal = segment.trapValues(df,k)

				segment_integrand = dfVal/fVal*segment.dzdt(t)
				if phi is not None:
					segment_integrand = segment.trapValues(phi,k)*segment_integrand
				if psi is not None:
					segment_integrand = segment.trapValues(psi,k)*segment_integrand

				segment_integral = scipy.integrate.romb(segment_integrand, dx=dt, axis=-1)/(2j*pi)
				integrals.append(segment_integral)

			I.append(sum(integrals))

			if verbose:
				if k > 1:
					print(k, 'I', I[-1], 'err', I[-2] - I[-1])
				else:
					print(k, 'I', I[-1])

			if callback is not None:
				if callback(I):
					break

		return I[-1], abs(I[-2] - I[-1])

	elif method == 'quad':
		if approx_df:
			df = numdifftools.Derivative(f, order=m)
			# df = lambda z: scipy.misc.derivative(f, z, dx=1e-8, n=1, order=3)
			# df = CxDeriv(f) # too slow

		I, err = 0, 0
		for segment in C.segments:
			integrand_cache = {}
			def integrand(t):
				if t in integrand_cache.keys():
					i = integrand_cache[t]
				else:
					z = segment(t)
					i = (df(z)/f(z))/(2j*pi) * segment.dzdt(t)
					if phi is not None:
						i = phi(z)*i
					if psi is not None:
						i = psi(z)*i
					integrand_cache[t] = i
				return i

			# integrate real part
			integrand_real = lambda t: np.real(integrand(t))
			result_real = scipy.integrate.quad(integrand_real, 0, 1, full_output=1, epsabs=absTol, epsrel=relTol)
			I_real, abserr_real, infodict_real = result_real[:3]

			# integrate imaginary part
			integrand_imag = lambda t: np.imag(integrand(t))
			result_imag = scipy.integrate.quad(integrand_imag, 0, 1, full_output=1, epsabs=absTol, epsrel=relTol)
			I_imag, abserr_imag, infodict_imag = result_imag[:3]

			I   += I_real + 1j*I_imag
			err += abserr_real + 1j*abserr_imag

		return I, abs(err)

class RootError(RuntimeError):
	pass

def count_enclosed_roots(C, f, df=None, NintAbsTol=0.07, integerTol=0.2, divMin=5, divMax=20, m=2, method='quad', verbose=False):
	r"""
	For a function of one complex variable, f(z), which is analytic in and within the contour C,
	return the number of zeros (counting multiplicities) within the contour calculated, using
	Cauchy's argument principle, as

	.. math::

		\frac{1}{2i\pi} \oint_C \frac{f'(z)}{f(z)} dz.

	If df(z), the derivative of f(z), is provided then the above integral is computed directly.
	Otherwise the derivative is approximated using a finite difference approximation implemented
	in Numdifftools <https://pypi.python.org/pypi/Numdifftools>`_.

	The number of roots is then the closest integer to the final value of the integral
	and the result is only accepted if the final value of the integral is within integerTol
	of the closest integer.

	Parameters
	----------
	C : Contour
		The enclosed_roots function returns the number of roots of f(z) within C
	f : function
		Function of a single variable f(x)
	df : function, optional
		Function of a single variable, df(x), providing the derivative of the function f(x)
		at the point x.
	NintAbsTol : float, optional
		Required absolute error for the integration.
	integerTol : float, optional
		The evaluation of the Cauchy integral will be accepted if the difference between successive
		iterations is < integerTol and the value of the integral is within integerTol of the
		closest integer.  Since the Cauchy integral must be an integer it is only necessary to
		distinguish which integer the integral is converging towards.  For this
		reason the integerTol can be set fairly large.
 	divMin : int, optional
 		Minimum number of divisions before the Romberg integration routine is allowed
 		to exit.  Only used if method='romb'.
	divMax : int, optional
		The maximum number of divisions before the Romberg integration routine of a
		path exits.  Only used if method='romb'.
    m : int, optional
    	Only used if df=None.  If method='romb' then m defines the stencil size for the
    	numerical differentiation of f, passed to numdifftools.fornberg.fd_derivative.
    	The stencil size is of 2*m+1 points in the interior, and 2*m+2 points for each
    	of the 2*m boundary points.  If instead method='quad' then m must is the order of
    	the error term in the Taylor approximation used which must be even.  The argument
    	order=m is passed to numdifftools.Derivative.
	method : {'quad', 'romb'}, optional
		If 'quad' then scipy.integrate.quad is used to perform the integral.  If 'romb'
		then Romberg integraion, using scipy.integrate.romb, is performed instead.

	Returns
	-------
	int
		The number of zeros of f (counting multiplicities) which lie within the contour

	References
	----------
	[DL] "A Numerical Method for Locating the Zeros of an Analytic function",
		L.M.Delves, J.N.Lyness, Mathematics of Computation (1967), Vol.21, Issue 100
	"""
	if verbose:
		print('Computing number of roots within', C)

	with warnings.catch_warnings():
		# ignore warnings and catch if I is NaN later
		warnings.simplefilter("ignore")
		I, err = prod(C, f, df, absTol=NintAbsTol, relTol=0, divMin=divMin, divMax=divMax, m=m, method=method, verbose=verbose, integerTol=integerTol)

	if np.isnan(I):
		raise RootError("Result of integral is an invalid value.  Most likely because of a divide by zero error.")

	elif abs(int(round(I.real)) - I.real) < integerTol and abs(I.imag) < integerTol:
		# integral is sufficiently close to an integer
		numberOfZeros = int(round(I.real))
		return numberOfZeros

	else:
		raise RootError("The number of enclosed roots has not converged to an integer")

<commit_msg>Remove CxDeriv import<commit_after>from __future__ import division
import numpy as np
from numpy import inf, pi
import scipy.integrate
import scipy.misc
import warnings

def prod(C, f, df=None, phi=None, psi=None, absTol=1e-12, relTol=1e-12, divMin=5, divMax=10, m=2, method='quad', verbose=False, callback=None, integerTol=inf):
	r"""
	Compute the symmetric bilinear form used in (1.12) of [KB]

	.. math::

		<\phi,\psi> = \frac{1}{2i\pi} \oint_C \phi(z) \psi(z) \frac{f'(z)}{f(z)} dz.

    Parameters
    ----------
	C : Contour
		The enclosed_roots function returns the number of roots of f(z) within C
	f : function
		Function of a single variable f(x)
	df : function, optional
		Function of a single variable, df(x), providing the derivative of the function f(x)
		at the point x.  If not provided then df is approximated using a finite difference
		method.
	phi : function, optional
		Function of a single variable phi(x).  If not provided then phi=1.
	psi : function, optional
		Function of a single variable psi(x).  If not provided then psi=1.
	absTol : float, optional
		Absolute error tolerance.
	relTol : float, optional
		Relative error tolerance.
 	divMin : int, optional
 		Minimum number of divisions before the Romberg integration routine is allowed
 		to exit.  Only used if method='romb'.
	divMax : int, optional
		The maximum number of divisions before the Romberg integration routine of a
		path exits.  Only used if method='romb'.
    m : int, optional
    	Only used if df=None.  If method='romb' then m defines the stencil size for the
    	numerical differentiation of f, passed to numdifftools.fornberg.fd_derivative.
    	The stencil size is of 2*m+1 points in the interior, and 2*m+2 points for each
    	of the 2*m boundary points.  If instead method='quad' then m must is the order of
    	the error term in the Taylor approximation used which must be even.  The argument
    	order=m is passed to numdifftools.Derivative.
	method : {'quad', 'romb'}, optional
		If 'quad' then scipy.integrate.quad is used to perform the integral.  If 'romb'
		then Romberg integraion, using scipy.integrate.romb, is performed instead.

	References
	----------
	[KB] "Computing the zeros of analytic functions" by Peter Kravanja, Marc Van Barel, Springer 2000
	"""
	N = 1
	k = 0
	I = []
	integrandMax = []

	approx_df = False
	if df is None:
		approx_df = True
		import numdifftools.fornberg as ndf
		import numdifftools

	if method == 'romb':
		# XXX: Better way to characterise err than abs(I[-2] - I[-1])?
		while (len(I) < divMin or (abs(I[-2] - I[-1]) > absTol and abs(I[-2] - I[-1]) > relTol*abs(I[-1])) or abs(int(round(I[-1].real)) - I[-1].real) > integerTol or abs(I[-1].imag) > integerTol) and k < divMax:
			N = 2*N
			t = np.linspace(0,1,N+1)
			k = int(np.log2(len(t)-1))
			dt = t[1]-t[0]

			integrals = []
			for segment in C.segments:
				# compute/retrieve function evaluations
				fVal = segment.trapValues(f,k)

				if approx_df:
					### approximate df/dz with finite difference, see: numdifftools.fornberg
					used_m = m
					if 2*m+1 > len(t):
						# not enough points to accommodate stencil size
						# temporarily reduce m
						used_m = (len(t)-1)//2

					dfdt = ndf.fd_derivative(fVal, t, n=1, m=used_m)
					dfVal = dfdt/segment.dzdt(t)

				else:
					dfVal = segment.trapValues(df,k)

				segment_integrand = dfVal/fVal*segment.dzdt(t)
				if phi is not None:
					segment_integrand = segment.trapValues(phi,k)*segment_integrand
				if psi is not None:
					segment_integrand = segment.trapValues(psi,k)*segment_integrand

				segment_integral = scipy.integrate.romb(segment_integrand, dx=dt, axis=-1)/(2j*pi)
				integrals.append(segment_integral)

			I.append(sum(integrals))

			if verbose:
				if k > 1:
					print(k, 'I', I[-1], 'err', I[-2] - I[-1])
				else:
					print(k, 'I', I[-1])

			if callback is not None:
				if callback(I):
					break

		return I[-1], abs(I[-2] - I[-1])

	elif method == 'quad':
		if approx_df:
			df = numdifftools.Derivative(f, order=m)
			# df = lambda z: scipy.misc.derivative(f, z, dx=1e-8, n=1, order=3)

		I, err = 0, 0
		for segment in C.segments:
			integrand_cache = {}
			def integrand(t):
				if t in integrand_cache.keys():
					i = integrand_cache[t]
				else:
					z = segment(t)
					i = (df(z)/f(z))/(2j*pi) * segment.dzdt(t)
					if phi is not None:
						i = phi(z)*i
					if psi is not None:
						i = psi(z)*i
					integrand_cache[t] = i
				return i

			# integrate real part
			integrand_real = lambda t: np.real(integrand(t))
			result_real = scipy.integrate.quad(integrand_real, 0, 1, full_output=1, epsabs=absTol, epsrel=relTol)
			I_real, abserr_real, infodict_real = result_real[:3]

			# integrate imaginary part
			integrand_imag = lambda t: np.imag(integrand(t))
			result_imag = scipy.integrate.quad(integrand_imag, 0, 1, full_output=1, epsabs=absTol, epsrel=relTol)
			I_imag, abserr_imag, infodict_imag = result_imag[:3]

			I   += I_real + 1j*I_imag
			err += abserr_real + 1j*abserr_imag

		return I, abs(err)

class RootError(RuntimeError):
	pass

def count_enclosed_roots(C, f, df=None, NintAbsTol=0.07, integerTol=0.2, divMin=5, divMax=20, m=2, method='quad', verbose=False):
	r"""
	For a function of one complex variable, f(z), which is analytic in and within the contour C,
	return the number of zeros (counting multiplicities) within the contour calculated, using
	Cauchy's argument principle, as

	.. math::

		\frac{1}{2i\pi} \oint_C \frac{f'(z)}{f(z)} dz.

	If df(z), the derivative of f(z), is provided then the above integral is computed directly.
	Otherwise the derivative is approximated using a finite difference approximation implemented
	in Numdifftools <https://pypi.python.org/pypi/Numdifftools>`_.

	The number of roots is then the closest integer to the final value of the integral
	and the result is only accepted if the final value of the integral is within integerTol
	of the closest integer.

	Parameters
	----------
	C : Contour
		The enclosed_roots function returns the number of roots of f(z) within C
	f : function
		Function of a single variable f(x)
	df : function, optional
		Function of a single variable, df(x), providing the derivative of the function f(x)
		at the point x.
	NintAbsTol : float, optional
		Required absolute error for the integration.
	integerTol : float, optional
		The evaluation of the Cauchy integral will be accepted if the difference between successive
		iterations is < integerTol and the value of the integral is within integerTol of the
		closest integer.  Since the Cauchy integral must be an integer it is only necessary to
		distinguish which integer the integral is converging towards.  For this
		reason the integerTol can be set fairly large.
 	divMin : int, optional
 		Minimum number of divisions before the Romberg integration routine is allowed
 		to exit.  Only used if method='romb'.
	divMax : int, optional
		The maximum number of divisions before the Romberg integration routine of a
		path exits.  Only used if method='romb'.
    m : int, optional
    	Only used if df=None.  If method='romb' then m defines the stencil size for the
    	numerical differentiation of f, passed to numdifftools.fornberg.fd_derivative.
    	The stencil size is of 2*m+1 points in the interior, and 2*m+2 points for each
    	of the 2*m boundary points.  If instead method='quad' then m must is the order of
    	the error term in the Taylor approximation used which must be even.  The argument
    	order=m is passed to numdifftools.Derivative.
	method : {'quad', 'romb'}, optional
		If 'quad' then scipy.integrate.quad is used to perform the integral.  If 'romb'
		then Romberg integraion, using scipy.integrate.romb, is performed instead.

	Returns
	-------
	int
		The number of zeros of f (counting multiplicities) which lie within the contour

	References
	----------
	[DL] "A Numerical Method for Locating the Zeros of an Analytic function",
		L.M.Delves, J.N.Lyness, Mathematics of Computation (1967), Vol.21, Issue 100
	"""
	if verbose:
		print('Computing number of roots within', C)

	with warnings.catch_warnings():
		# ignore warnings and catch if I is NaN later
		warnings.simplefilter("ignore")
		I, err = prod(C, f, df, absTol=NintAbsTol, relTol=0, divMin=divMin, divMax=divMax, m=m, method=method, verbose=verbose, integerTol=integerTol)

	if np.isnan(I):
		raise RootError("Result of integral is an invalid value.  Most likely because of a divide by zero error.")

	elif abs(int(round(I.real)) - I.real) < integerTol and abs(I.imag) < integerTol:
		# integral is sufficiently close to an integer
		numberOfZeros = int(round(I.real))
		return numberOfZeros

	else:
		raise RootError("The number of enclosed roots has not converged to an integer")

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

class State:
    STAND = 'stand'
    WALK = 'walk'

    state = STAND
<commit_msg>changes in StateMachine<commit_after># -*- coding: utf-8 -*-

class EntityState:
    pass

class StateMachine:
    def __init__(self, entity):
        self.entity = entity
        self.states = {}
        self.current_state = None

    def create_state(self, name):
        state = EntityState()
        self.states[name] = state
        return state

    def add_state(self, name, state):
        self.states[name] = state

    def change_state(self, name):
        if name not in self.states:
            raise Exception('No such %s state' % name)

        new_state = self.states[name]

        if new_state == self.current_state:
            return

        self.current_state = new_state
<|endoftext|>


<commit_before>from copy import copy
from datetime import datetime
from couchdbkit import MultipleResultsFound
from couchdbkit.ext.django.schema import StringProperty, Document,\
    DateTimeProperty, BooleanProperty
from django.db import models
from django.db.models import Q
from dimagi.utils.couch.database import is_bigcouch, bigcouch_quorum_count

XFORMS_SESSION_SMS = "SMS"
XFORMS_SESSION_IVR = "IVR"
XFORMS_SESSION_TYPES = [XFORMS_SESSION_SMS, XFORMS_SESSION_IVR]

class XFormsSession(Document):
    """
    Keeps information about an SMS XForm session.
    """
    # generic properties
    connection_id = StringProperty()
    session_id = StringProperty()
    form_xmlns = StringProperty()
    start_time = DateTimeProperty()
    modified_time = DateTimeProperty()
    end_time = DateTimeProperty()
    completed = BooleanProperty()

    # HQ specific properties
    domain = StringProperty()
    user_id = StringProperty()
    app_id = StringProperty()
    submission_id = StringProperty()
    survey_incentive = StringProperty()
    session_type = StringProperty(choices=XFORMS_SESSION_TYPES, default=XFORMS_SESSION_SMS)
    workflow = StringProperty() # One of the corehq.apps.sms.models.WORKFLOW_* constants describing what kind of workflow this session was a part of
    reminder_id = StringProperty() # Points to the _id of an instance of corehq.apps.reminders.models.CaseReminder that this session is tied to

    def save(self, *args, **kwargs):
        if is_bigcouch() and "w" not in kwargs:
            # Force a write to all nodes before returning
            kwargs["w"] = bigcouch_quorum_count()
        super(XFormsSession, self).save(*args, **kwargs)
        sync_sql_session_from_couch_session(self)

    def __unicode__(self):
        return 'Form %(form)s in domain %(domain)s. Last modified: %(mod)s' % \
            {"form": self.form_xmlns,
             "domain": self.domain,
             "mod": self.modified_time}

    def end(self, completed):
        """
        Marks this as ended (by setting end time).
        """
        self.completed = completed
        self.modified_time = self.end_time = datetime.utcnow()

    @property
    def is_open(self):
        """
        True if this session is still open, False otherwise.
        """
        return self.end_time is None

    @classmethod
    def get_all_open_sms_sessions(cls, domain, contact_id):
        sessions = cls.view("smsforms/open_sms_sessions_by_connection",
                            key=[domain, contact_id],
                            include_docs=True).all()
        return sessions

    @classmethod
    def close_all_open_sms_sessions(cls, domain, contact_id):
        sessions = cls.get_all_open_sms_sessions(domain, contact_id)
        for session in sessions:
            session.end(False)
            session.save()

    @classmethod
    def by_session_id(cls, id):
        return XFormsSession.view("smsforms/sessions_by_touchforms_id",
                                  key=id, include_docs=True).one()

    @classmethod
    def get_open_sms_session(cls, domain, contact_id):
        """
        Looks up the open sms survey session for the given domain and contact_id.
        Only one session is expected to be open at a time.
        Raises MultipleResultsFound if more than one session is open.
        """
        session = cls.view("smsforms/open_sms_sessions_by_connection",
                           key=[domain, contact_id],
                           include_docs=True).one()
        return session

class SQLXFormsSession(models.Model):
    """
    Keeps information about an SMS XForm session.
    """
    # generic properties
    couch_id = models.CharField(null=True, blank=True, db_index=True, max_length=50)
    connection_id = models.CharField(null=True, blank=True, db_index=True, max_length=50)
    session_id = models.CharField(null=True, blank=True, db_index=True, max_length=50)
    form_xmlns = models.CharField(null=True, blank=True, max_length=100)
    start_time = models.DateTimeField()
    modified_time = models.DateTimeField()
    end_time = models.DateTimeField(null=True)
    completed = models.BooleanField()

    # HQ specific properties
    domain = models.CharField(null=True, blank=True, db_index=True, max_length=100)
    user_id = models.CharField(null=True, blank=True, max_length=50)
    app_id = models.CharField(null=True, blank=True, max_length=50)
    submission_id = models.CharField(null=True, blank=True, max_length=50)
    survey_incentive = models.CharField(null=True, blank=True, max_length=100)
    session_type = models.CharField(max_length=10, choices=zip(XFORMS_SESSION_TYPES, XFORMS_SESSION_TYPES),
                                    default=XFORMS_SESSION_SMS)
    workflow = models.CharField(null=True, blank=True, max_length=20)
    reminder_id = models.CharField(null=True, blank=True, max_length=50)

    def __unicode__(self):
        return 'Form %(form)s in domain %(domain)s. Last modified: %(mod)s' % \
            {"form": self.form_xmlns,
             "domain": self.domain,
             "mod": self.modified_time}

    def end(self, completed):
        """
        Marks this as ended (by setting end time).
        """
        self.completed = completed
        self.modified_time = self.end_time = datetime.utcnow()

    @property
    def is_open(self):
        """
        True if this session is still open, False otherwise.
        """
        return self.end_time is None

    @classmethod
    def get_all_open_sms_sessions(cls, domain, contact_id):
        return cls.objects.filter(
            Q(session_type__isnull=True) | Q(session_type=XFORMS_SESSION_SMS),
            domain=domain,
            connection_id=contact_id,
            end_time__isnull=True,
        )

    @classmethod
    def close_all_open_sms_sessions(cls, domain, contact_id):
        sessions = cls.get_all_open_sms_sessions(domain, contact_id)
        for session in sessions:
            session.end(False)
            session.save()

    @classmethod
    def by_session_id(cls, id):
        try:
            return cls.get(session_id=id)
        except SQLXFormsSession.DoesNotExist:
            return None

    @classmethod
    def get_open_sms_session(cls, domain, contact_id):
        """
        Looks up the open sms survey session for the given domain and contact_id.
        Only one session is expected to be open at a time.
        Raises MultipleResultsFound if more than one session is open.
        """
        objs = cls.get_all_open_sms_sessions(domain, contact_id).all()
        if len(objs) > 1:
            raise MultipleResultsFound('more than 1 ({}) session found for domain {} and contact {}'.format(
                len(objs), domain, contact_id
            ))
        elif len(objs) == 0:
            return None
        return objs[0]

def sync_sql_session_from_couch_session(couch_session):
    data = copy(couch_session._doc)
    couch_id = data.pop('_id')
    data.pop('_rev', None)
    try:
        sql_session = SQLXFormsSession.objects.get(couch_id=couch_id)
    except SQLXFormsSession.DoesNotExist:
        sql_session = SQLXFormsSession(couch_id=couch_id)

    for attr, value in data.items():
        setattr(sql_session, attr, value)
    sql_session.save()

from . import signals
<commit_msg>add method to try sql when returning a session object and log errors<commit_after>from copy import copy
from datetime import datetime
import logging
from couchdbkit import MultipleResultsFound
from couchdbkit.ext.django.schema import StringProperty, Document,\
    DateTimeProperty, BooleanProperty
from django.db import models
from django.db.models import Q
from dimagi.utils.couch.database import is_bigcouch, bigcouch_quorum_count

XFORMS_SESSION_SMS = "SMS"
XFORMS_SESSION_IVR = "IVR"
XFORMS_SESSION_TYPES = [XFORMS_SESSION_SMS, XFORMS_SESSION_IVR]

class XFormsSession(Document):
    """
    Keeps information about an SMS XForm session.
    """
    # generic properties
    connection_id = StringProperty()
    session_id = StringProperty()
    form_xmlns = StringProperty()
    start_time = DateTimeProperty()
    modified_time = DateTimeProperty()
    end_time = DateTimeProperty()
    completed = BooleanProperty()

    # HQ specific properties
    domain = StringProperty()
    user_id = StringProperty()
    app_id = StringProperty()
    submission_id = StringProperty()
    survey_incentive = StringProperty()
    session_type = StringProperty(choices=XFORMS_SESSION_TYPES, default=XFORMS_SESSION_SMS)
    workflow = StringProperty() # One of the corehq.apps.sms.models.WORKFLOW_* constants describing what kind of workflow this session was a part of
    reminder_id = StringProperty() # Points to the _id of an instance of corehq.apps.reminders.models.CaseReminder that this session is tied to

    def save(self, *args, **kwargs):
        if is_bigcouch() and "w" not in kwargs:
            # Force a write to all nodes before returning
            kwargs["w"] = bigcouch_quorum_count()
        super(XFormsSession, self).save(*args, **kwargs)
        sync_sql_session_from_couch_session(self)

    def __unicode__(self):
        return 'Form %(form)s in domain %(domain)s. Last modified: %(mod)s' % \
            {"form": self.form_xmlns,
             "domain": self.domain,
             "mod": self.modified_time}

    def end(self, completed):
        """
        Marks this as ended (by setting end time).
        """
        self.completed = completed
        self.modified_time = self.end_time = datetime.utcnow()

    @property
    def is_open(self):
        """
        True if this session is still open, False otherwise.
        """
        return self.end_time is None

    @classmethod
    def get_all_open_sms_sessions(cls, domain, contact_id):
        sessions = cls.view("smsforms/open_sms_sessions_by_connection",
                            key=[domain, contact_id],
                            include_docs=True).all()
        return sessions

    @classmethod
    def close_all_open_sms_sessions(cls, domain, contact_id):
        sessions = cls.get_all_open_sms_sessions(domain, contact_id)
        for session in sessions:
            session.end(False)
            session.save()

    @classmethod
    def by_session_id(cls, id):
        return cls.view("smsforms/sessions_by_touchforms_id",
                        key=id, include_docs=True).one()

    @classmethod
    def get_open_sms_session(cls, domain, contact_id):
        """
        Looks up the open sms survey session for the given domain and contact_id.
        Only one session is expected to be open at a time.
        Raises MultipleResultsFound if more than one session is open.
        """
        session = cls.view("smsforms/open_sms_sessions_by_connection",
                           key=[domain, contact_id],
                           include_docs=True).one()
        return session

class SQLXFormsSession(models.Model):
    """
    Keeps information about an SMS XForm session.
    """
    # generic properties
    couch_id = models.CharField(null=True, blank=True, db_index=True, max_length=50)
    connection_id = models.CharField(null=True, blank=True, db_index=True, max_length=50)
    session_id = models.CharField(null=True, blank=True, db_index=True, max_length=50)
    form_xmlns = models.CharField(null=True, blank=True, max_length=100)
    start_time = models.DateTimeField()
    modified_time = models.DateTimeField()
    end_time = models.DateTimeField(null=True)
    completed = models.BooleanField()

    # HQ specific properties
    domain = models.CharField(null=True, blank=True, db_index=True, max_length=100)
    user_id = models.CharField(null=True, blank=True, max_length=50)
    app_id = models.CharField(null=True, blank=True, max_length=50)
    submission_id = models.CharField(null=True, blank=True, max_length=50)
    survey_incentive = models.CharField(null=True, blank=True, max_length=100)
    session_type = models.CharField(max_length=10, choices=zip(XFORMS_SESSION_TYPES, XFORMS_SESSION_TYPES),
                                    default=XFORMS_SESSION_SMS)
    workflow = models.CharField(null=True, blank=True, max_length=20)
    reminder_id = models.CharField(null=True, blank=True, max_length=50)

    def __unicode__(self):
        return 'Form %(form)s in domain %(domain)s. Last modified: %(mod)s' % \
            {"form": self.form_xmlns,
             "domain": self.domain,
             "mod": self.modified_time}

    def end(self, completed):
        """
        Marks this as ended (by setting end time).
        """
        self.completed = completed
        self.modified_time = self.end_time = datetime.utcnow()

    @property
    def is_open(self):
        """
        True if this session is still open, False otherwise.
        """
        return self.end_time is None

    @classmethod
    def get_all_open_sms_sessions(cls, domain, contact_id):
        return cls.objects.filter(
            Q(session_type__isnull=True) | Q(session_type=XFORMS_SESSION_SMS),
            domain=domain,
            connection_id=contact_id,
            end_time__isnull=True,
        )

    @classmethod
    def close_all_open_sms_sessions(cls, domain, contact_id):
        sessions = cls.get_all_open_sms_sessions(domain, contact_id)
        for session in sessions:
            session.end(False)
            session.save()

    @classmethod
    def by_session_id(cls, id):
        try:
            return cls.get(session_id=id)
        except SQLXFormsSession.DoesNotExist:
            return None

    @classmethod
    def get_open_sms_session(cls, domain, contact_id):
        """
        Looks up the open sms survey session for the given domain and contact_id.
        Only one session is expected to be open at a time.
        Raises MultipleResultsFound if more than one session is open.
        """
        objs = cls.get_all_open_sms_sessions(domain, contact_id).all()
        if len(objs) > 1:
            raise MultipleResultsFound('more than 1 ({}) session found for domain {} and contact {}'.format(
                len(objs), domain, contact_id
            ))
        elif len(objs) == 0:
            return None
        return objs[0]

def get_session_by_session_id(id):
    """
    Utility method to first try and get a session in sql, then failing that get it in couch
    and log an error.
    """
    sql_session = SQLXFormsSession.by_session_id(id)
    if sql_session:
        return sql_session

    couch_session = XFormsSession.by_session_id(id)
    if couch_session:
        logging.error('session {} could not be found in sql.'.format(couch_session._id))
    return couch_session

def sync_sql_session_from_couch_session(couch_session):
    data = copy(couch_session._doc)
    couch_id = data.pop('_id')
    data.pop('_rev', None)
    try:
        sql_session = SQLXFormsSession.objects.get(couch_id=couch_id)
    except SQLXFormsSession.DoesNotExist:
        sql_session = SQLXFormsSession(couch_id=couch_id)

    for attr, value in data.items():
        setattr(sql_session, attr, value)
    sql_session.save()

from . import signals
<|endoftext|>