<commit_before>from collections import defaultdict
import functools
import json
import itertools
import uuid
from corehq.apps.app_manager.exceptions import SuiteError
from corehq.apps.builds.models import CommCareBuildConfig
from corehq.apps.app_manager.tasks import create_user_cases
from corehq.util.soft_assert import soft_assert
from couchdbkit.exceptions import DocTypeError
from corehq import Domain
from corehq.apps.app_manager.const import (
    CT_REQUISITION_MODE_3,
    CT_LEDGER_STOCK,
    CT_LEDGER_REQUESTED,
    CT_REQUISITION_MODE_4,
    CT_LEDGER_APPROVED,
    CT_LEDGER_PREFIX,
    USERCASE_TYPE,
    USERCASE_ID,
    USERCASE_PREFIX)
from corehq.apps.app_manager.xform import XForm, XFormException, parse_xml
from dimagi.utils.couch import CriticalSection
import re
from dimagi.utils.decorators.memoized import memoized
from django.core.cache import cache
import logging

logger = logging.getLogger(__name__)

def get_app_id(form):
    """
    Given an XForm instance, try to grab the app id, returning
    None if not available. This is just a shortcut since the app_id
    might not always be set.
    """
    return getattr(form, "app_id", None)

def split_path(path):
    path_parts = path.split('/')
    name = path_parts.pop(-1)
    path = '/'.join(path_parts)
    return path, name

def save_xform(app, form, xml):
    def change_xmlns(xform, replacing):
        data = xform.data_node.render()
        xmlns = "http://openrosa.org/formdesigner/%s" % form.get_unique_id()
        data = data.replace(replacing, xmlns, 1)
        xform.instance_node.remove(xform.data_node.xml)
        xform.instance_node.append(parse_xml(data))
        xml = xform.render()
        return xform, xml

    try:
        xform = XForm(xml)
    except XFormException:
        pass
    else:
        duplicates = app.get_xmlns_map()[xform.data_node.tag_xmlns]
        for duplicate in duplicates:
            if form == duplicate:
                continue
            else:
                xform, xml = change_xmlns(xform, xform.data_node.tag_xmlns)
                break

        GENERIC_XMLNS = "http://www.w3.org/2002/xforms"
        if not xform.data_node.tag_xmlns or xform.data_node.tag_xmlns == GENERIC_XMLNS:  #no xmlns
            xform, xml = change_xmlns(xform, GENERIC_XMLNS)

    form.source = xml

CASE_TYPE_REGEX = r'^[\w-]+$'
_case_type_regex = re.compile(CASE_TYPE_REGEX)

def is_valid_case_type(case_type):
    """
    >>> is_valid_case_type('foo')
    True
    >>> is_valid_case_type('foo-bar')
    True
    >>> is_valid_case_type('foo bar')
    False
    >>> is_valid_case_type('')
    False
    >>> is_valid_case_type(None)
    False
    """
    return bool(_case_type_regex.match(case_type or ''))

class ParentCasePropertyBuilder(object):
    def __init__(self, app, defaults=()):
        self.app = app
        self.defaults = defaults

    @property
    @memoized
    def forms_info(self):
        # unfortunate, but biggest speed issue is accessing couchdbkit properties
        # so compute them once

        forms_info = []
        if self.app.doc_type == 'RemoteApp':
            return forms_info
        for module in self.app.get_modules():
            for form in module.get_forms():
                forms_info.append((module.case_type, form))
        return forms_info

    @memoized
    def get_parent_types_and_contributed_properties(self, case_type):
        parent_types = set()
        case_properties = set()
        for m_case_type, form in self.forms_info:
            p_types, c_props = form.get_parent_types_and_contributed_properties(m_case_type, case_type)
            parent_types.update(p_types)
            case_properties.update(c_props)
        return parent_types, case_properties

    def get_parent_types(self, case_type):
        parent_types, _ = \
            self.get_parent_types_and_contributed_properties(case_type)
        return set(p[0] for p in parent_types)

    @memoized
    def get_other_case_sharing_apps_in_domain(self):
        from corehq.apps.app_manager.models import get_apps_in_domain
        apps = get_apps_in_domain(self.app.domain, include_remote=False)
        return [a for a in apps if a.case_sharing and a.id != self.app.id]

    @memoized
    def get_properties(self, case_type, already_visited=(),
                       include_shared_properties=True):
        if case_type in already_visited:
            return ()

        get_properties_recursive = functools.partial(
            self.get_properties,
            already_visited=already_visited + (case_type,),
            include_shared_properties=include_shared_properties
        )

        case_properties = set(self.defaults)

        for m_case_type, form in self.forms_info:
            case_properties.update(self.get_case_updates(form, case_type))

        parent_types, contributed_properties = \
            self.get_parent_types_and_contributed_properties(case_type)
        case_properties.update(contributed_properties)
        for parent_type in parent_types:
            for property in get_properties_recursive(parent_type[0]):
                case_properties.add('%s/%s' % (parent_type[1], property))
        if self.app.case_sharing and include_shared_properties:
            from corehq.apps.app_manager.models import get_apps_in_domain
            for app in self.get_other_case_sharing_apps_in_domain():
                case_properties.update(
                    get_case_properties(
                        app, [case_type], include_shared_properties=False
                    ).get(case_type, [])
                )
        return case_properties

    @memoized
    def get_case_updates(self, form, case_type):
        return form.get_case_updates(case_type)

    def get_parent_type_map(self, case_types):
        parent_map = defaultdict(dict)
        for case_type in case_types:
            parent_types, _ = self.get_parent_types_and_contributed_properties(case_type)
            rel_map = defaultdict(list)
            for parent_type, relationship in parent_types:
                rel_map[relationship].append(parent_type)

            for relationship, types in rel_map.items():
                if len(types) > 1:
                    logger.error(
                        "Case Type '%s' has multiple parents for relationship '%s': %s",
                        case_type, relationship, types
                    )
                parent_map[case_type][relationship] = types[0] if types else []

        return parent_map

    def get_case_property_map(self, case_types, include_shared_properties=True):
        case_types = sorted(case_types)
        return {
            case_type: sorted(self.get_properties(
                case_type, include_shared_properties=include_shared_properties
            ))
            for case_type in case_types
        }

def get_case_properties(app, case_types, defaults=(),
                        include_shared_properties=True):
    builder = ParentCasePropertyBuilder(app, defaults)
    return builder.get_case_property_map(
        case_types, include_shared_properties=include_shared_properties
    )

def is_usercase_in_use(domain_name):
    domain = Domain.get_by_name(domain_name) if domain_name else None
    return domain and domain.usercase_enabled

def get_all_case_properties(app):
    return get_case_properties(
        app,
        set(itertools.chain.from_iterable(m.get_case_types() for m in app.modules)),
        defaults=('name',)
    )

def get_usercase_properties(app):
    # No need to check toggles.USER_AS_A_CASE. This function is only called
    # from app_manager.views, and it checks the toggle.
    if is_usercase_in_use(app.domain):
        return get_case_properties(app, [USERCASE_TYPE])
    return {USERCASE_TYPE: []}

def get_settings_values(app):
    try:
        profile = app.profile
    except AttributeError:
        profile = {}
    hq_settings = dict([
        (attr, app[attr])
        for attr in app.properties() if not hasattr(app[attr], 'pop')
    ])
    if getattr(app, 'use_custom_suite', False):
        hq_settings.update({'custom_suite': getattr(app, 'custom_suite', None)})

    hq_settings['build_spec'] = app.build_spec.to_string()
    # the admin_password hash shouldn't be sent to the client
    hq_settings.pop('admin_password', None)

    domain = Domain.get_by_name(app.domain)
    return {
        'properties': profile.get('properties', {}),
        'features': profile.get('features', {}),
        'hq': hq_settings,
        '$parent': {
            'doc_type': app.get_doc_type(),
            '_id': app.get_id,
            'domain': app.domain,
            'commtrack_enabled': domain.commtrack_enabled,
        }
    }

def add_odk_profile_after_build(app_build):
    """caller must save"""

    profile = app_build.create_profile(is_odk=True)
    app_build.lazy_put_attachment(profile, 'files/profile.ccpr')
    # hack this in for records
    app_build.odk_profile_created_after_build = True

def create_temp_sort_column(field, index):
    """
    Used to create a column for the sort only properties to
    add the field to the list of properties and app strings but
    not persist anything to the detail data.
    """
    from corehq.apps.app_manager.models import SortOnlyDetailColumn
    return SortOnlyDetailColumn(
        model='case',
        field=field,
        format='invisible',
        header=None,
    )

def is_sort_only_column(column):
    from corehq.apps.app_manager.models import SortOnlyDetailColumn
    return isinstance(column, SortOnlyDetailColumn)

def get_correct_app_class(doc):
    from corehq.apps.app_manager.models import Application, RemoteApp
    try:
        return {
            'Application': Application,
            'Application-Deleted': Application,
            "RemoteApp": RemoteApp,
            "RemoteApp-Deleted": RemoteApp,
        }[doc['doc_type']]
    except KeyError:
        raise DocTypeError()

def all_apps_by_domain(domain):
    from corehq.apps.app_manager.models import ApplicationBase
    rows = ApplicationBase.get_db().view(
        'app_manager/applications',
        startkey=[domain, None],
        endkey=[domain, None, {}],
        include_docs=True,
    ).all()
    for row in rows:
        doc = row['doc']
        yield get_correct_app_class(doc).wrap(doc)

def new_careplan_module(app, name, lang, target_module):
    from corehq.apps.app_manager.models import CareplanModule, CareplanGoalForm, CareplanTaskForm
    module = app.add_module(CareplanModule.new_module(
        app,
        name,
        lang,
        target_module.unique_id,
        target_module.case_type)
    )

    forms = [form_class.new_form(lang, name, mode)
                for form_class in [CareplanGoalForm, CareplanTaskForm]
                for mode in ['create', 'update']]

    for form, source in forms:
        module.forms.append(form)
        form = module.get_form(-1)
        form.source = source

    return module

def languages_mapping():
    mapping = cache.get('__languages_mapping')
    if not mapping:
        with open('submodules/langcodes/langs.json') as langs_file:
            lang_data = json.load(langs_file)
            mapping = dict([(l["two"], l["names"]) for l in lang_data])
        mapping["default"] = ["Default Language"]
        cache.set('__languages_mapping', mapping, 12*60*60)
    return mapping

def commtrack_ledger_sections(mode):
    sections = [CT_LEDGER_STOCK]
    if mode == CT_REQUISITION_MODE_3:
        sections += [CT_LEDGER_REQUESTED]
    elif mode == CT_REQUISITION_MODE_4:
        sections += [CT_LEDGER_REQUESTED, CT_LEDGER_APPROVED]

    return ['{}{}'.format(CT_LEDGER_PREFIX, s) for s in sections]

def version_key(ver):
    """
    A key function that takes a version and returns a numeric value that can
    be used for sorting

    >>> version_key('2')
    2000000
    >>> version_key('2.9')
    2009000
    >>> version_key('2.10')
    2010000
    >>> version_key('2.9.1')
    2009001
    >>> version_key('2.9.1.1')
    2009001
    >>> version_key('2.9B')
    Traceback (most recent call last):
      ...
    ValueError: invalid literal for int() ...

    """
    padded = ver + '.0.0'
    values = padded.split('.')
    return int(values[0]) * 1000000 + int(values[1]) * 1000 + int(values[2])

def get_commcare_versions(request_user):
    versions = [i.build.version for i in CommCareBuildConfig.fetch().menu
                if request_user.is_superuser or not i.superuser_only]
    return sorted(versions, key=version_key)

def actions_use_usercase(actions):
    return (('update_usercase' in actions and actions['update_usercase'].update) or
            ('usercase_preload' in actions and actions['usercase_preload'].preload))

def enable_usercase(domain_name):
    with CriticalSection(['enable_usercase_' + domain_name]):
        domain = Domain.get_by_name(domain_name, strict=True)
        if not domain.usercase_enabled:
            domain.usercase_enabled = True
            domain.save()
            create_user_cases.delay(domain_name)

def prefix_usercase_properties(properties):
    return {'{}{}'.format(USERCASE_PREFIX, prop) for prop in properties}

def get_cloudcare_session_data(suite_gen, domain_name, form, couch_user):
    from corehq.apps.hqcase.utils import get_case_by_domain_hq_user_id

    datums = suite_gen.get_new_case_id_datums_meta(form)
    session_data = {datum['datum'].id: uuid.uuid4().hex for datum in datums}
    if couch_user.doc_type == 'CommCareUser':  # smsforms.app.start_session could pass a CommCareCase
        try:
            extra_datums = suite_gen.get_extra_case_id_datums(form)
        except SuiteError as err:
            _assert = soft_assert(['nhooper_at_dimagi_dot_com'.replace('_at_', '@').replace('_dot_', '.')])
            _assert(False, 'Domain "%s": %s' % (domain_name, err))
        else:
            if suite_gen.any_usercase_datums(extra_datums):
                usercase = get_case_by_domain_hq_user_id(domain_name, couch_user.get_id, include_docs=False)
                if usercase:
                    session_data[USERCASE_ID] = usercase['id']
    return session_data
<commit_msg>include usercase properties in get_call_case_properties<commit_after>from collections import defaultdict
import functools
import json
import itertools
import uuid
from corehq.apps.app_manager.exceptions import SuiteError
from corehq.apps.builds.models import CommCareBuildConfig
from corehq.apps.app_manager.tasks import create_user_cases
from corehq.util.soft_assert import soft_assert
from couchdbkit.exceptions import DocTypeError
from corehq import Domain
from corehq.apps.app_manager.const import (
    CT_REQUISITION_MODE_3,
    CT_LEDGER_STOCK,
    CT_LEDGER_REQUESTED,
    CT_REQUISITION_MODE_4,
    CT_LEDGER_APPROVED,
    CT_LEDGER_PREFIX,
    USERCASE_TYPE,
    USERCASE_ID,
    USERCASE_PREFIX)
from corehq.apps.app_manager.xform import XForm, XFormException, parse_xml
from dimagi.utils.couch import CriticalSection
import re
from dimagi.utils.decorators.memoized import memoized
from django.core.cache import cache
import logging

logger = logging.getLogger(__name__)

def get_app_id(form):
    """
    Given an XForm instance, try to grab the app id, returning
    None if not available. This is just a shortcut since the app_id
    might not always be set.
    """
    return getattr(form, "app_id", None)

def split_path(path):
    path_parts = path.split('/')
    name = path_parts.pop(-1)
    path = '/'.join(path_parts)
    return path, name

def save_xform(app, form, xml):
    def change_xmlns(xform, replacing):
        data = xform.data_node.render()
        xmlns = "http://openrosa.org/formdesigner/%s" % form.get_unique_id()
        data = data.replace(replacing, xmlns, 1)
        xform.instance_node.remove(xform.data_node.xml)
        xform.instance_node.append(parse_xml(data))
        xml = xform.render()
        return xform, xml

    try:
        xform = XForm(xml)
    except XFormException:
        pass
    else:
        duplicates = app.get_xmlns_map()[xform.data_node.tag_xmlns]
        for duplicate in duplicates:
            if form == duplicate:
                continue
            else:
                xform, xml = change_xmlns(xform, xform.data_node.tag_xmlns)
                break

        GENERIC_XMLNS = "http://www.w3.org/2002/xforms"
        if not xform.data_node.tag_xmlns or xform.data_node.tag_xmlns == GENERIC_XMLNS:  #no xmlns
            xform, xml = change_xmlns(xform, GENERIC_XMLNS)

    form.source = xml

CASE_TYPE_REGEX = r'^[\w-]+$'
_case_type_regex = re.compile(CASE_TYPE_REGEX)

def is_valid_case_type(case_type):
    """
    >>> is_valid_case_type('foo')
    True
    >>> is_valid_case_type('foo-bar')
    True
    >>> is_valid_case_type('foo bar')
    False
    >>> is_valid_case_type('')
    False
    >>> is_valid_case_type(None)
    False
    """
    return bool(_case_type_regex.match(case_type or ''))

class ParentCasePropertyBuilder(object):
    def __init__(self, app, defaults=()):
        self.app = app
        self.defaults = defaults

    @property
    @memoized
    def forms_info(self):
        # unfortunate, but biggest speed issue is accessing couchdbkit properties
        # so compute them once

        forms_info = []
        if self.app.doc_type == 'RemoteApp':
            return forms_info
        for module in self.app.get_modules():
            for form in module.get_forms():
                forms_info.append((module.case_type, form))
        return forms_info

    @memoized
    def get_parent_types_and_contributed_properties(self, case_type):
        parent_types = set()
        case_properties = set()
        for m_case_type, form in self.forms_info:
            p_types, c_props = form.get_parent_types_and_contributed_properties(m_case_type, case_type)
            parent_types.update(p_types)
            case_properties.update(c_props)
        return parent_types, case_properties

    def get_parent_types(self, case_type):
        parent_types, _ = \
            self.get_parent_types_and_contributed_properties(case_type)
        return set(p[0] for p in parent_types)

    @memoized
    def get_other_case_sharing_apps_in_domain(self):
        from corehq.apps.app_manager.models import get_apps_in_domain
        apps = get_apps_in_domain(self.app.domain, include_remote=False)
        return [a for a in apps if a.case_sharing and a.id != self.app.id]

    @memoized
    def get_properties(self, case_type, already_visited=(),
                       include_shared_properties=True):
        if case_type in already_visited:
            return ()

        get_properties_recursive = functools.partial(
            self.get_properties,
            already_visited=already_visited + (case_type,),
            include_shared_properties=include_shared_properties
        )

        case_properties = set(self.defaults)

        for m_case_type, form in self.forms_info:
            case_properties.update(self.get_case_updates(form, case_type))

        parent_types, contributed_properties = \
            self.get_parent_types_and_contributed_properties(case_type)
        case_properties.update(contributed_properties)
        for parent_type in parent_types:
            for property in get_properties_recursive(parent_type[0]):
                case_properties.add('%s/%s' % (parent_type[1], property))
        if self.app.case_sharing and include_shared_properties:
            from corehq.apps.app_manager.models import get_apps_in_domain
            for app in self.get_other_case_sharing_apps_in_domain():
                case_properties.update(
                    get_case_properties(
                        app, [case_type], include_shared_properties=False
                    ).get(case_type, [])
                )
        return case_properties

    @memoized
    def get_case_updates(self, form, case_type):
        return form.get_case_updates(case_type)

    def get_parent_type_map(self, case_types):
        parent_map = defaultdict(dict)
        for case_type in case_types:
            parent_types, _ = self.get_parent_types_and_contributed_properties(case_type)
            rel_map = defaultdict(list)
            for parent_type, relationship in parent_types:
                rel_map[relationship].append(parent_type)

            for relationship, types in rel_map.items():
                if len(types) > 1:
                    logger.error(
                        "Case Type '%s' has multiple parents for relationship '%s': %s",
                        case_type, relationship, types
                    )
                parent_map[case_type][relationship] = types[0] if types else []

        return parent_map

    def get_case_property_map(self, case_types, include_shared_properties=True):
        case_types = sorted(case_types)
        return {
            case_type: sorted(self.get_properties(
                case_type, include_shared_properties=include_shared_properties
            ))
            for case_type in case_types
        }

def get_case_properties(app, case_types, defaults=(),
                        include_shared_properties=True):
    builder = ParentCasePropertyBuilder(app, defaults)
    return builder.get_case_property_map(
        case_types, include_shared_properties=include_shared_properties
    )

def is_usercase_in_use(domain_name):
    domain = Domain.get_by_name(domain_name) if domain_name else None
    return domain and domain.usercase_enabled

def get_all_case_properties(app):
    extra_types = set()
    if is_usercase_in_use(app.domain):
        extra_types.add(USERCASE_TYPE)

    return get_case_properties(
        app,
        set(itertools.chain.from_iterable(m.get_case_types() for m in app.modules)) | extra_types,
        defaults=('name',)
    )

def get_usercase_properties(app):
    # No need to check toggles.USER_AS_A_CASE. This function is only called
    # from app_manager.views, and it checks the toggle.
    if is_usercase_in_use(app.domain):
        return get_case_properties(app, [USERCASE_TYPE])
    return {USERCASE_TYPE: []}

def get_settings_values(app):
    try:
        profile = app.profile
    except AttributeError:
        profile = {}
    hq_settings = dict([
        (attr, app[attr])
        for attr in app.properties() if not hasattr(app[attr], 'pop')
    ])
    if getattr(app, 'use_custom_suite', False):
        hq_settings.update({'custom_suite': getattr(app, 'custom_suite', None)})

    hq_settings['build_spec'] = app.build_spec.to_string()
    # the admin_password hash shouldn't be sent to the client
    hq_settings.pop('admin_password', None)

    domain = Domain.get_by_name(app.domain)
    return {
        'properties': profile.get('properties', {}),
        'features': profile.get('features', {}),
        'hq': hq_settings,
        '$parent': {
            'doc_type': app.get_doc_type(),
            '_id': app.get_id,
            'domain': app.domain,
            'commtrack_enabled': domain.commtrack_enabled,
        }
    }

def add_odk_profile_after_build(app_build):
    """caller must save"""

    profile = app_build.create_profile(is_odk=True)
    app_build.lazy_put_attachment(profile, 'files/profile.ccpr')
    # hack this in for records
    app_build.odk_profile_created_after_build = True

def create_temp_sort_column(field, index):
    """
    Used to create a column for the sort only properties to
    add the field to the list of properties and app strings but
    not persist anything to the detail data.
    """
    from corehq.apps.app_manager.models import SortOnlyDetailColumn
    return SortOnlyDetailColumn(
        model='case',
        field=field,
        format='invisible',
        header=None,
    )

def is_sort_only_column(column):
    from corehq.apps.app_manager.models import SortOnlyDetailColumn
    return isinstance(column, SortOnlyDetailColumn)

def get_correct_app_class(doc):
    from corehq.apps.app_manager.models import Application, RemoteApp
    try:
        return {
            'Application': Application,
            'Application-Deleted': Application,
            "RemoteApp": RemoteApp,
            "RemoteApp-Deleted": RemoteApp,
        }[doc['doc_type']]
    except KeyError:
        raise DocTypeError()

def all_apps_by_domain(domain):
    from corehq.apps.app_manager.models import ApplicationBase
    rows = ApplicationBase.get_db().view(
        'app_manager/applications',
        startkey=[domain, None],
        endkey=[domain, None, {}],
        include_docs=True,
    ).all()
    for row in rows:
        doc = row['doc']
        yield get_correct_app_class(doc).wrap(doc)

def new_careplan_module(app, name, lang, target_module):
    from corehq.apps.app_manager.models import CareplanModule, CareplanGoalForm, CareplanTaskForm
    module = app.add_module(CareplanModule.new_module(
        app,
        name,
        lang,
        target_module.unique_id,
        target_module.case_type)
    )

    forms = [form_class.new_form(lang, name, mode)
                for form_class in [CareplanGoalForm, CareplanTaskForm]
                for mode in ['create', 'update']]

    for form, source in forms:
        module.forms.append(form)
        form = module.get_form(-1)
        form.source = source

    return module

def languages_mapping():
    mapping = cache.get('__languages_mapping')
    if not mapping:
        with open('submodules/langcodes/langs.json') as langs_file:
            lang_data = json.load(langs_file)
            mapping = dict([(l["two"], l["names"]) for l in lang_data])
        mapping["default"] = ["Default Language"]
        cache.set('__languages_mapping', mapping, 12*60*60)
    return mapping

def commtrack_ledger_sections(mode):
    sections = [CT_LEDGER_STOCK]
    if mode == CT_REQUISITION_MODE_3:
        sections += [CT_LEDGER_REQUESTED]
    elif mode == CT_REQUISITION_MODE_4:
        sections += [CT_LEDGER_REQUESTED, CT_LEDGER_APPROVED]

    return ['{}{}'.format(CT_LEDGER_PREFIX, s) for s in sections]

def version_key(ver):
    """
    A key function that takes a version and returns a numeric value that can
    be used for sorting

    >>> version_key('2')
    2000000
    >>> version_key('2.9')
    2009000
    >>> version_key('2.10')
    2010000
    >>> version_key('2.9.1')
    2009001
    >>> version_key('2.9.1.1')
    2009001
    >>> version_key('2.9B')
    Traceback (most recent call last):
      ...
    ValueError: invalid literal for int() ...

    """
    padded = ver + '.0.0'
    values = padded.split('.')
    return int(values[0]) * 1000000 + int(values[1]) * 1000 + int(values[2])

def get_commcare_versions(request_user):
    versions = [i.build.version for i in CommCareBuildConfig.fetch().menu
                if request_user.is_superuser or not i.superuser_only]
    return sorted(versions, key=version_key)

def actions_use_usercase(actions):
    return (('update_usercase' in actions and actions['update_usercase'].update) or
            ('usercase_preload' in actions and actions['usercase_preload'].preload))

def enable_usercase(domain_name):
    with CriticalSection(['enable_usercase_' + domain_name]):
        domain = Domain.get_by_name(domain_name, strict=True)
        if not domain.usercase_enabled:
            domain.usercase_enabled = True
            domain.save()
            create_user_cases.delay(domain_name)

def prefix_usercase_properties(properties):
    return {'{}{}'.format(USERCASE_PREFIX, prop) for prop in properties}

def get_cloudcare_session_data(suite_gen, domain_name, form, couch_user):
    from corehq.apps.hqcase.utils import get_case_by_domain_hq_user_id

    datums = suite_gen.get_new_case_id_datums_meta(form)
    session_data = {datum['datum'].id: uuid.uuid4().hex for datum in datums}
    if couch_user.doc_type == 'CommCareUser':  # smsforms.app.start_session could pass a CommCareCase
        try:
            extra_datums = suite_gen.get_extra_case_id_datums(form)
        except SuiteError as err:
            _assert = soft_assert(['nhooper_at_dimagi_dot_com'.replace('_at_', '@').replace('_dot_', '.')])
            _assert(False, 'Domain "%s": %s' % (domain_name, err))
        else:
            if suite_gen.any_usercase_datums(extra_datums):
                usercase = get_case_by_domain_hq_user_id(domain_name, couch_user.get_id, include_docs=False)
                if usercase:
                    session_data[USERCASE_ID] = usercase['id']
    return session_data
<|endoftext|>


<commit_before>#!/usr/bin/env python3

from progressbar import ProgressBar
from pretty import *
import argparse
import lasagne
import theano
import theano.tensor as T
import pickle
import numpy
import time

import os
import os.path

from lasagne.layers.normalization import LocalResponseNormalization2DLayer
from lasagne.nonlinearities import rectify, softmax

Conv2DLayer = lasagne.layers.Conv2DLayer
MaxPool2DLayer = lasagne.layers.MaxPool2DLayer
if theano.config.device.startswith("gpu"):
    import lasagne.layers.dnn
    # Force GPU implementations if a GPU is available.
    # Do not know why Theano is not selecting these impls
    # by default as advertised.
    if theano.sandbox.cuda.dnn.dnn_available():
        Conv2DLayer = lasagne.layers.dnn.Conv2DDNNLayer
        MaxPool2DLayer = lasagne.layers.dnn.MaxPool2DDNNLayer

parser = argparse.ArgumentParser()
parser.add_argument('tagged', help='path to directory containing prepared files')
parser.add_argument('-m', '--momentum', type=float, help='momentum', default=0.9)
parser.add_argument('-b', '--batchsize', type=int, help='size of each mini batch', default=256)
parser.add_argument('-s', '--batch-stop', type=int, help='stop after this many batches each epoch', default=0)
parser.add_argument('-e', '--epoch-stop', type=int, help='stop after this many epochs', default=0)
parser.add_argument('-c', '--cache', type=argparse.FileType('ab+'), help='store/resume network state in/from this file', default=None)
parser.add_argument('-p', '--plot', type=argparse.FileType('ab+'), help='plot network performance to this png file', default=None)
parser.add_argument('-l', '--labels', type=argparse.FileType('wb+'), help='record test set predictions to this file', default=None)
parser.add_argument('-v', '--verbose', action='count')
args = parser.parse_args()

imsz = 128
cropsz = 117

section("Setup")
task("Loading data")
subtask("Loading training set")
y_train = numpy.memmap(os.path.join(args.tagged, "train.labels.db"), dtype=numpy.int32, mode='r')
X_train = numpy.memmap(os.path.join(args.tagged, "train.images.db"), dtype=numpy.float32, mode='r', shape=(len(y_train), 3, imsz, imsz))
cats = numpy.max(y_train)+1
subtask("Loading validation set")
y_val = numpy.memmap(os.path.join(args.tagged, "val.labels.db"), dtype=numpy.int32, mode='r')
X_val = numpy.memmap(os.path.join(args.tagged, "val.images.db"), dtype=numpy.float32, mode='r', shape=(len(y_val), 3, imsz, imsz))

if args.labels is not None:
    subtask("Loading test set")
    y_test = numpy.memmap(os.path.join(args.tagged, "test.labels.db"), dtype=numpy.int32, mode='r')
    X_test = numpy.memmap(os.path.join(args.tagged, "test.images.db"), dtype=numpy.float32, mode='r', shape=(len(y_test), 3, imsz, imsz))

task("Building model and compiling functions")
# create Theano variables for input and target minibatch
learning_rates = numpy.logspace(-1.7, -4, 60, dtype=theano.config.floatX)
learning_rates_var = theano.shared(learning_rates)
learning_rate = theano.shared(learning_rates[0])
epochi = T.iscalar('e')
input_var = T.tensor4('X')
target_var = T.ivector('y')

# parameters
flip_var = T.iscalar('f')
crop_var = T.ivector('c') # ycrop, xcrop
center = numpy.zeros((2,), dtype=numpy.int32)
center.fill(numpy.floor((imsz - cropsz)/2))

# crop+flip
cropped = input_var[:, :, crop_var[0]:crop_var[0]+cropsz, crop_var[1]:crop_var[1]+cropsz]
prepared = cropped[:,:,:,::flip_var]

# create a small convolutional neural network
network = lasagne.layers.InputLayer((None, 3, cropsz, cropsz), prepared)
# 1st
network = Conv2DLayer(network, 64, (8, 8), stride=2, nonlinearity=rectify)
network = LocalResponseNormalization2DLayer(network, n=5, k=1, beta=0.75, alpha=0.0001/5)
network = MaxPool2DLayer(network, (3, 3), stride=2)
# 2nd
network = Conv2DLayer(network, 96, (5, 5), stride=1, pad='same', nonlinearity=rectify)
network = LocalResponseNormalization2DLayer(network, n=5, k=1, beta=0.75, alpha=0.0001/5)
network = MaxPool2DLayer(network, (3, 3), stride=2)
# 3rd
network = Conv2DLayer(network, 128, (3, 3), stride=1, pad='same', nonlinearity=rectify)
network = LocalResponseNormalization2DLayer(network, n=5, k=1, beta=0.75, alpha=0.0001/5)
network = MaxPool2DLayer(network, (3, 3), stride=2)
# 4th
network = lasagne.layers.DenseLayer(network, 512, nonlinearity=rectify)
network = lasagne.layers.DropoutLayer(network)
# 5th
network = lasagne.layers.DenseLayer(network, cats, nonlinearity=softmax)

# Output
prediction = lasagne.layers.get_output(network)

# create loss function
from lasagne.regularization import regularize_network_params, l2, l1
loss = lasagne.objectives.categorical_crossentropy(prediction, target_var).mean()
loss += regularize_network_params(network, l2) * 1e-3

# create parameter update expressions
params = lasagne.layers.get_all_params(network, trainable=True)
updates = lasagne.updates.nesterov_momentum(loss, params, learning_rate=learning_rate, momentum=args.momentum)
updates[learning_rate] = learning_rates_var[epochi]

# compile training function that updates parameters and returns training loss
train_fn = theano.function([epochi, flip_var, crop_var, input_var, target_var], loss, updates=updates)

# Create a loss expression for validation/testing. The crucial difference here
# is that we do a deterministic forward pass through the network, disabling
# dropout layers.
test_prediction = lasagne.layers.get_output(network, deterministic=True)
test_loss = lasagne.objectives.categorical_crossentropy(test_prediction,
                                                        target_var)
test_loss = test_loss.mean()
# As a bonus, also create an expression for the classification accuracy:
test_1_acc = T.mean(lasagne.objectives.categorical_accuracy(test_prediction, target_var, top_k=1))
test_5_acc = T.mean(lasagne.objectives.categorical_accuracy(test_prediction, target_var, top_k=5))

# compile a second function computing the validation loss and accuracy:
val_fn = theano.function([flip_var, crop_var, input_var, target_var], [test_loss, test_1_acc, test_5_acc])

# and a final one for test output
top5_pred = T.argsort(test_prediction, axis=1)[:, -5:]
test_fn = theano.function([flip_var, crop_var, input_var], [top5_pred])

def iterate_minibatches(inputs, targets, batchsize, shuffle=False, test=False):
    assert len(inputs) == len(targets)
    if shuffle:
        indices = numpy.arange(len(inputs))
        numpy.random.shuffle(indices)
    for start_idx in range(0, len(inputs) - batchsize + 1, batchsize):
        if shuffle:
            excerpt = indices[start_idx:start_idx + batchsize]
        else:
            excerpt = slice(start_idx, start_idx + batchsize)
        yield inputs[excerpt], targets[excerpt]

training = []
validation = []
end = len(learning_rates)
if args.epoch_stop != 0:
    end = args.epoch_stop

if args.plot is not None:
    args.plot.close()
    import matplotlib
    matplotlib.use('Agg') # avoid the need for X

def replot():
    if args.plot is None:
        return

    global training
    global validation
    if len(validation) == 0:
        return

    import seaborn as sns
    import matplotlib.pyplot as plt
    sns.set(style="ticks", color_codes=True)

    fig = plt.figure()
    ax_loss = fig.add_subplot(1, 2, 1)
    ax_err = fig.add_subplot(1, 2, 2)

    # styles
    ax_loss.grid(True)
    ax_err.grid(True)
    ax_loss.set_yscale('log')
    #ax_err.set_yscale('log')

    # limits
    global end
    ax_loss.set_xlim(0, end)
    ax_err.set_xlim(0, end)
    ax_err.set_ylim(0, 1)
    #ax_err.set_ylim(1e-5, 1)

    # plot loss
    xend = len(training)+1
    ax_loss.plot(range(1, xend), [dp[0] for dp in training], 'b', marker='o', markersize=4)
    ax_loss.plot(range(1, xend), [dp[0] for dp in validation], 'r--', marker='o', markersize=4)
    ax_loss.legend(['Training loss', 'Validation loss'])
    ax_loss.set_title('Model loss')

    # plot error
    ax_err.plot(range(1, xend), [1-dp[1] for dp in training], 'b', marker='o', markersize=4)
    ax_err.plot(range(1, xend), [1-dp[2] for dp in training], 'r', marker='o', markersize=4)
    ax_err.plot(range(1, xend), [1-dp[1] for dp in validation], 'y--', marker='s', markersize=4)
    ax_err.plot(range(1, xend), [1-dp[2] for dp in validation], 'm--', marker='s', markersize=4)
    ax_err.legend(['Training exact', 'Training top 5', 'Validation exact', 'Validation top 5'])
    ax_err.set_title('Match error')

    import tempfile
    with tempfile.NamedTemporaryFile(delete=False, dir=os.path.dirname(args.plot.name)) as fp:
        fig.savefig(fp, format='png', dpi=192)
        plt.close(fig)
        fp.close()
        os.rename(fp.name, args.plot.name)

start = 0
if args.cache is not None:
    try:
        section("Restoring state")
        args.cache.seek(0)
        task("Loading parameters")
        state = pickle.load(args.cache)
        task("Loading epoch information")
        epoch = pickle.load(args.cache)
        training = pickle.load(args.cache)
        validation = pickle.load(args.cache)
        task("Restoring parameter values")
        for p, v in zip(params, state):
            p.set_value(v)
        learning_rate.set_value(learning_rates[epoch])
        subtask("Resuming at epoch {}".format(epoch+1))
        start = epoch+1
    except EOFError:
        task("No model state stored; starting afresh")

section("Training")
# Finally, launch the training loop.
for epoch in range(start, end):
    replot()

    task("Starting training epoch {}".format(epoch))
    start_time = time.time()

    # How much work will we have to do?
    train_batches = len(range(0, len(X_train) - args.batchsize + 1, args.batchsize))
    val_batches = len(range(0, len(X_val) - args.batchsize + 1, args.batchsize))
    train_test_batches = val_batches

    # In each epoch, we do a pass over minibatches of the training data:
    train_loss = 0
    p = progress(train_batches)
    i = 1
    frame = numpy.zeros((2,), dtype=numpy.int32)
    for batch in iterate_minibatches(X_train, y_train, args.batchsize, shuffle=True):
        flip = numpy.random.randint(0, 2) and 1 or -1
        frame[0] = numpy.random.randint(0, imsz - cropsz)
        frame[1] = numpy.random.randint(0, imsz - cropsz)
        train_loss += train_fn(epoch, flip, frame, batch[0], batch[1])
        p.update(i)
        i = i+1
        if args.batch_stop != 0 and i > args.batch_stop:
            p.update(train_batches)
            break

    # Only do forward pass on a subset of the training data
    subtask("Doing forward pass on training data (size: {})".format(len(X_val)))
    p = progress(train_test_batches)
    i = 0
    train_acc1 = 0
    train_acc5 = 0
    for batch in iterate_minibatches(X_train, y_train, args.batchsize, shuffle=True):
        i = i+1
        _, acc1, acc5 = val_fn(1, center, batch[0], batch[1])
        p.update(i)
        train_acc1 += acc1
        train_acc5 += acc5
        if i == train_test_batches:
            break

    subtask("Doing forward pass on validation data (size: {})".format(len(X_val)))
    # Also do a validation data forward pass
    val_loss = 0
    val_acc1 = 0
    val_acc5 = 0
    p = progress(val_batches)
    i = 1
    for batch in iterate_minibatches(X_val, y_val, args.batchsize, shuffle=False):
        loss, acc1, acc5 = val_fn(1, center, batch[0], batch[1])
        val_loss += loss
        val_acc1 += acc1
        val_acc5 += acc5
        p.update(i)
        i = i+1

    # record performance
    training.append((train_loss/train_batches, train_acc1/train_test_batches, train_acc5/train_test_batches))
    validation.append((val_loss/val_batches, val_acc1/val_batches, val_acc5/val_batches))

    # store model state
    if args.cache is not None:
        subtask("Storing trained parameters")
        args.cache.seek(0)
        args.cache.truncate()
        pickle.dump([p.get_value() for p in params], args.cache)
        pickle.dump(epoch, args.cache)
        pickle.dump(training, args.cache)
        pickle.dump(validation, args.cache)

    # Then we print the results for this epoch:
    subtask("Epoch results: {:.2f}%/{:.2f}% (t5acc, v5acc)".format(
        training[-1][2] * 100,
        validation[-1][2] * 100,
    ))

replot()

if args.labels is not None:
    section("Evaluation")
    task("Evaluating performance on test data set")
    pred_out = numpy.memmap(args.labels, dtype=numpy.int32, shape=(len(X_test), 5))

    test_batches = len(range(0, len(X_test) - args.batchsize + 1, args.batchsize))
    p = progress(test_batches)

    i = 1
    for batch in iterate_minibatches(X_test, y_test, args.batchsize, shuffle=False):
        s = (i-1)*args.batchsize
        pred_out[s:s+args.batchsize, :] = test_fn(1, center, batch[0])[0]
        p.update(i)
        i = i+1
    del pred_out
<commit_msg>Replace local response with batchnorm<commit_after>#!/usr/bin/env python3

from progressbar import ProgressBar
from pretty import *
import argparse
import lasagne
import theano
import theano.tensor as T
import pickle
import numpy
import time

import os
import os.path

from lasagne.layers.normalization import BatchNormLayer
from lasagne.nonlinearities import rectify, softmax

Conv2DLayer = lasagne.layers.Conv2DLayer
MaxPool2DLayer = lasagne.layers.MaxPool2DLayer
if theano.config.device.startswith("gpu"):
    import lasagne.layers.dnn
    # Force GPU implementations if a GPU is available.
    # Do not know why Theano is not selecting these impls
    # by default as advertised.
    if theano.sandbox.cuda.dnn.dnn_available():
        Conv2DLayer = lasagne.layers.dnn.Conv2DDNNLayer
        MaxPool2DLayer = lasagne.layers.dnn.MaxPool2DDNNLayer

parser = argparse.ArgumentParser()
parser.add_argument('tagged', help='path to directory containing prepared files')
parser.add_argument('-m', '--momentum', type=float, help='momentum', default=0.9)
parser.add_argument('-b', '--batchsize', type=int, help='size of each mini batch', default=256)
parser.add_argument('-s', '--batch-stop', type=int, help='stop after this many batches each epoch', default=0)
parser.add_argument('-e', '--epoch-stop', type=int, help='stop after this many epochs', default=0)
parser.add_argument('-c', '--cache', type=argparse.FileType('ab+'), help='store/resume network state in/from this file', default=None)
parser.add_argument('-p', '--plot', type=argparse.FileType('ab+'), help='plot network performance to this png file', default=None)
parser.add_argument('-l', '--labels', type=argparse.FileType('wb+'), help='record test set predictions to this file', default=None)
parser.add_argument('-v', '--verbose', action='count')
args = parser.parse_args()

imsz = 128
cropsz = 117

section("Setup")
task("Loading data")
subtask("Loading training set")
y_train = numpy.memmap(os.path.join(args.tagged, "train.labels.db"), dtype=numpy.int32, mode='r')
X_train = numpy.memmap(os.path.join(args.tagged, "train.images.db"), dtype=numpy.float32, mode='r', shape=(len(y_train), 3, imsz, imsz))
cats = numpy.max(y_train)+1
subtask("Loading validation set")
y_val = numpy.memmap(os.path.join(args.tagged, "val.labels.db"), dtype=numpy.int32, mode='r')
X_val = numpy.memmap(os.path.join(args.tagged, "val.images.db"), dtype=numpy.float32, mode='r', shape=(len(y_val), 3, imsz, imsz))

if args.labels is not None:
    subtask("Loading test set")
    y_test = numpy.memmap(os.path.join(args.tagged, "test.labels.db"), dtype=numpy.int32, mode='r')
    X_test = numpy.memmap(os.path.join(args.tagged, "test.images.db"), dtype=numpy.float32, mode='r', shape=(len(y_test), 3, imsz, imsz))

task("Building model and compiling functions")
# create Theano variables for input and target minibatch
learning_rates = numpy.logspace(-1.5, -4, 30, dtype=theano.config.floatX)
learning_rates_var = theano.shared(learning_rates)
learning_rate = theano.shared(learning_rates[0])
epochi = T.iscalar('e')
input_var = T.tensor4('X')
target_var = T.ivector('y')

# parameters
flip_var = T.iscalar('f')
crop_var = T.ivector('c') # ycrop, xcrop
center = numpy.zeros((2,), dtype=numpy.int32)
center.fill(numpy.floor((imsz - cropsz)/2))

# crop+flip
cropped = input_var[:, :, crop_var[0]:crop_var[0]+cropsz, crop_var[1]:crop_var[1]+cropsz]
prepared = cropped[:,:,:,::flip_var]

# create a small convolutional neural network
network = lasagne.layers.InputLayer((None, 3, cropsz, cropsz), prepared)
# 1st
network = Conv2DLayer(network, 64, (8, 8), stride=2, nonlinearity=rectify)
network = BatchNormLayer(network, nonlinearity=rectify)
network = MaxPool2DLayer(network, (3, 3), stride=2)
# 2nd
network = Conv2DLayer(network, 96, (5, 5), stride=1, pad='same')
network = BatchNormLayer(network, nonlinearity=rectify)
network = MaxPool2DLayer(network, (3, 3), stride=2)
# 3rd
network = Conv2DLayer(network, 128, (3, 3), stride=1, pad='same')
network = BatchNormLayer(network, nonlinearity=rectify)
network = MaxPool2DLayer(network, (3, 3), stride=2)
# 4th
network = lasagne.layers.DenseLayer(network, 512, nonlinearity=rectify)
# 5th
network = lasagne.layers.DenseLayer(network, cats, nonlinearity=softmax)

# Output
prediction = lasagne.layers.get_output(network)

# create loss function
from lasagne.regularization import regularize_network_params, l2, l1
loss = lasagne.objectives.categorical_crossentropy(prediction, target_var).mean()
loss += regularize_network_params(network, l2) * 1e-3

# create parameter update expressions
params = lasagne.layers.get_all_params(network, trainable=True)
updates = lasagne.updates.nesterov_momentum(loss, params, learning_rate=learning_rate, momentum=args.momentum)
updates[learning_rate] = learning_rates_var[epochi]

# compile training function that updates parameters and returns training loss
train_fn = theano.function([epochi, flip_var, crop_var, input_var, target_var], loss, updates=updates)

# Create a loss expression for validation/testing. The crucial difference here
# is that we do a deterministic forward pass through the network, disabling
# dropout layers.
test_prediction = lasagne.layers.get_output(network, deterministic=True)
test_loss = lasagne.objectives.categorical_crossentropy(test_prediction,
                                                        target_var)
test_loss = test_loss.mean()
# As a bonus, also create an expression for the classification accuracy:
test_1_acc = T.mean(lasagne.objectives.categorical_accuracy(test_prediction, target_var, top_k=1))
test_5_acc = T.mean(lasagne.objectives.categorical_accuracy(test_prediction, target_var, top_k=5))

# compile a second function computing the validation loss and accuracy:
val_fn = theano.function([flip_var, crop_var, input_var, target_var], [test_loss, test_1_acc, test_5_acc])

# and a final one for test output
top5_pred = T.argsort(test_prediction, axis=1)[:, -5:]
test_fn = theano.function([flip_var, crop_var, input_var], [top5_pred])

def iterate_minibatches(inputs, targets, batchsize, shuffle=False, test=False):
    assert len(inputs) == len(targets)
    if shuffle:
        indices = numpy.arange(len(inputs))
        numpy.random.shuffle(indices)
    for start_idx in range(0, len(inputs) - batchsize + 1, batchsize):
        if shuffle:
            excerpt = indices[start_idx:start_idx + batchsize]
        else:
            excerpt = slice(start_idx, start_idx + batchsize)
        yield inputs[excerpt], targets[excerpt]

training = []
validation = []
end = len(learning_rates)
if args.epoch_stop != 0:
    end = args.epoch_stop

if args.plot is not None:
    args.plot.close()
    import matplotlib
    matplotlib.use('Agg') # avoid the need for X

def replot():
    if args.plot is None:
        return

    global training
    global validation
    if len(validation) == 0:
        return

    import seaborn as sns
    import matplotlib.pyplot as plt
    sns.set(style="ticks", color_codes=True)

    fig = plt.figure()
    ax_loss = fig.add_subplot(1, 2, 1)
    ax_err = fig.add_subplot(1, 2, 2)

    # styles
    ax_loss.grid(True)
    ax_err.grid(True)
    ax_loss.set_yscale('log')
    #ax_err.set_yscale('log')

    # limits
    global end
    ax_loss.set_xlim(0, end)
    ax_err.set_xlim(0, end)
    ax_err.set_ylim(0, 1)
    #ax_err.set_ylim(1e-5, 1)

    # plot loss
    xend = len(training)+1
    ax_loss.plot(range(1, xend), [dp[0] for dp in training], 'b', marker='o', markersize=4)
    ax_loss.plot(range(1, xend), [dp[0] for dp in validation], 'r--', marker='o', markersize=4)
    ax_loss.legend(['Training loss', 'Validation loss'])
    ax_loss.set_title('Model loss')

    # plot error
    ax_err.plot(range(1, xend), [1-dp[1] for dp in training], 'b', marker='o', markersize=4)
    ax_err.plot(range(1, xend), [1-dp[2] for dp in training], 'r', marker='o', markersize=4)
    ax_err.plot(range(1, xend), [1-dp[1] for dp in validation], 'y--', marker='s', markersize=4)
    ax_err.plot(range(1, xend), [1-dp[2] for dp in validation], 'm--', marker='s', markersize=4)
    ax_err.legend(['Training exact', 'Training top 5', 'Validation exact', 'Validation top 5'])
    ax_err.set_title('Match error')

    import tempfile
    with tempfile.NamedTemporaryFile(delete=False, dir=os.path.dirname(args.plot.name)) as fp:
        fig.savefig(fp, format='png', dpi=192)
        plt.close(fig)
        fp.close()
        os.rename(fp.name, args.plot.name)

start = 0
if args.cache is not None:
    try:
        section("Restoring state")
        args.cache.seek(0)
        task("Loading parameters")
        state = pickle.load(args.cache)
        task("Loading epoch information")
        epoch = pickle.load(args.cache)
        training = pickle.load(args.cache)
        validation = pickle.load(args.cache)
        task("Restoring parameter values")
        for p, v in zip(params, state):
            p.set_value(v)
        learning_rate.set_value(learning_rates[epoch])
        subtask("Resuming at epoch {}".format(epoch+1))
        start = epoch+1
    except EOFError:
        task("No model state stored; starting afresh")

section("Training")
# Finally, launch the training loop.
for epoch in range(start, end):
    replot()

    task("Starting training epoch {}".format(epoch))
    start_time = time.time()

    # How much work will we have to do?
    train_batches = len(range(0, len(X_train) - args.batchsize + 1, args.batchsize))
    val_batches = len(range(0, len(X_val) - args.batchsize + 1, args.batchsize))
    train_test_batches = val_batches

    # In each epoch, we do a pass over minibatches of the training data:
    train_loss = 0
    p = progress(train_batches)
    i = 1
    frame = numpy.zeros((2,), dtype=numpy.int32)
    for batch in iterate_minibatches(X_train, y_train, args.batchsize, shuffle=True):
        flip = numpy.random.randint(0, 2) and 1 or -1
        frame[0] = numpy.random.randint(0, imsz - cropsz)
        frame[1] = numpy.random.randint(0, imsz - cropsz)
        train_loss += train_fn(epoch, flip, frame, batch[0], batch[1])
        p.update(i)
        i = i+1
        if args.batch_stop != 0 and i > args.batch_stop:
            p.update(train_batches)
            break

    # Only do forward pass on a subset of the training data
    subtask("Doing forward pass on training data (size: {})".format(len(X_val)))
    p = progress(train_test_batches)
    i = 0
    train_acc1 = 0
    train_acc5 = 0
    for batch in iterate_minibatches(X_train, y_train, args.batchsize, shuffle=True):
        i = i+1
        _, acc1, acc5 = val_fn(1, center, batch[0], batch[1])
        p.update(i)
        train_acc1 += acc1
        train_acc5 += acc5
        if i == train_test_batches:
            break

    subtask("Doing forward pass on validation data (size: {})".format(len(X_val)))
    # Also do a validation data forward pass
    val_loss = 0
    val_acc1 = 0
    val_acc5 = 0
    p = progress(val_batches)
    i = 1
    for batch in iterate_minibatches(X_val, y_val, args.batchsize, shuffle=False):
        loss, acc1, acc5 = val_fn(1, center, batch[0], batch[1])
        val_loss += loss
        val_acc1 += acc1
        val_acc5 += acc5
        p.update(i)
        i = i+1

    # record performance
    training.append((train_loss/train_batches, train_acc1/train_test_batches, train_acc5/train_test_batches))
    validation.append((val_loss/val_batches, val_acc1/val_batches, val_acc5/val_batches))

    # store model state
    if args.cache is not None:
        subtask("Storing trained parameters")
        args.cache.seek(0)
        args.cache.truncate()
        pickle.dump([p.get_value() for p in params], args.cache)
        pickle.dump(epoch, args.cache)
        pickle.dump(training, args.cache)
        pickle.dump(validation, args.cache)

    # Then we print the results for this epoch:
    subtask("Epoch results: {:.2f}%/{:.2f}% (t5acc, v5acc)".format(
        training[-1][2] * 100,
        validation[-1][2] * 100,
    ))

replot()

if args.labels is not None:
    section("Evaluation")
    task("Evaluating performance on test data set")
    pred_out = numpy.memmap(args.labels, dtype=numpy.int32, shape=(len(X_test), 5))

    test_batches = len(range(0, len(X_test) - args.batchsize + 1, args.batchsize))
    p = progress(test_batches)

    i = 1
    for batch in iterate_minibatches(X_test, y_test, args.batchsize, shuffle=False):
        s = (i-1)*args.batchsize
        pred_out[s:s+args.batchsize, :] = test_fn(1, center, batch[0])[0]
        p.update(i)
        i = i+1
    del pred_out
<|endoftext|>


<commit_before>import pygame as pg
from constants import WINDOW_WIDTH, WINDOW_HEIGHT
from models.player import Player
from pygame import Rect

def main():
    """ Set up the game and run the main game loop """
    pg.init()      # Prepare the pygame module for use
    surface_height = WINDOW_WIDTH   # Desired physical surface size, in pixels.
    surface_width = WINDOW_HEIGHT    # Desired physical surface size, in pixels.

    surface_height = 700   # Desired physical surface size, in pixels.
    surface_width = 700    # Desired physical surface size, in pixels.
    # Create surface of (width, height), and its window.
    main_surface = pg.display.set_mode((surface_height, surface_width))

    some_color = (0, 0, 255)  # A color is a mix of (Red, Green, Blue)
    some_other_color = (255, 255, 255)
    # Set up some data to describe a small rectangle and its color
    triangle_points = (0,200),(100,0),(200,200)

    player = Player()
    while True:
        ev = pg.event.poll()    # Look for any event
        if ev.type == pg.QUIT:  # Window close button clicked?
            break                   #   .   .. leave game loop

        # Update your game objects and data structures here...

        # We draw everything from scratch on each frame.
        # So first fill everything with the background color

        main_surface.fill(some_color)

        pg.draw.polygon(main_surface, some_other_color, triangle_points, 0)

        player.draw()
        # main_surface.fill(some_other_color, small_box)

        # Overpaint a smaller rectangle on the main surface

        # Now the surface is ready, tell pygame to display it!
        pg.display.flip()

    pygame.quit()     # Once we leave the loop, close the window.

main()
<commit_msg>Game crashed when it was supposed to quit<commit_after>import pygame as pg
from constants import WINDOW_WIDTH, WINDOW_HEIGHT
from models.player import Player
from pygame import Rect

def main():
    """ Set up the game and run the main game loop """
    pg.init()      # Prepare the pygame module for use
    surface_height = WINDOW_WIDTH   # Desired physical surface size, in pixels.
    surface_width = WINDOW_HEIGHT    # Desired physical surface size, in pixels.

    surface_height = 700   # Desired physical surface size, in pixels.
    surface_width = 700    # Desired physical surface size, in pixels.
    # Create surface of (width, height), and its window.
    main_surface = pg.display.set_mode((surface_height, surface_width))

    some_color = (0, 0, 255)  # A color is a mix of (Red, Green, Blue)
    some_other_color = (255, 255, 255)
    # Set up some data to describe a small rectangle and its color
    triangle_points = (0,200),(100,0),(200,200)

    player = Player()
    while True:
        ev = pg.event.poll()    # Look for any event
        if ev.type == pg.QUIT:  # Window close button clicked?
            break                   #   .   .. leave game loop

        # Update your game objects and data structures here...

        # We draw everything from scratch on each frame.
        # So first fill everything with the background color

        main_surface.fill(some_color)

        pg.draw.polygon(main_surface, some_other_color, triangle_points, 0)

        player.draw()
        # main_surface.fill(some_other_color, small_box)

        # Overpaint a smaller rectangle on the main surface

        # Now the surface is ready, tell pygame to display it!
        pg.display.flip()

    pg.quit()     # Once we leave the loop, close the window.

main()
<|endoftext|>


<commit_before>#!/var/www/opendc.ewi.tudelft.nl/web-server/venv/bin/python

import json
import os
import sys
import traceback

from flask import Flask, abort, request, send_from_directory, session, jsonify
import flask_socketio
from oauth2client import client, crypt

from opendc.models.user import User
from opendc.util import exceptions, rest, path_parser

if len(sys.argv) < 2:
    print "config file path not given as argument"
    sys.exit(1)

# Get keys from config file
with open(sys.argv[1]) as file:
    KEYS = json.load(file)

STATIC_ROOT = os.path.join(KEYS['ROOT_DIR'], 'opendc-frontend', 'build')

FLASK_CORE_APP = Flask(__name__, static_url_path='')
FLASK_CORE_APP.config['SECREY_KEY'] = KEYS['FLASK_SECRET']

SOCKET_IO_CORE = flask_socketio.SocketIO(FLASK_CORE_APP)

@FLASK_CORE_APP.errorhandler(404)
def page_not_found(e):
    return send_from_directory(STATIC_ROOT, '404.html')

@FLASK_CORE_APP.route('/')
def serve_splash():
    """Serve the splash page on /"""

    return send_from_directory(STATIC_ROOT, 'index.html')

@FLASK_CORE_APP.route('/app')
def serve_app():
    """Serve the app on /app."""

    return send_from_directory(STATIC_ROOT, 'app.html')

@FLASK_CORE_APP.route('/profile')
def serve_profile():
    """Serve profile page."""

    return send_from_directory(STATIC_ROOT, 'profile.html')

@FLASK_CORE_APP.route('/projects')
def serve_projects():
    """Serve the projects page."""

    return send_from_directory(STATIC_ROOT, 'projects.html')

@FLASK_CORE_APP.route('/web-server-test')
def serve_web_server_test():
    """Serve the web server test."""

    return send_from_directory(os.path.join(KEYS['ROOT_DIR'], 'opendc-web-server', 'static'), 'index.html')

@FLASK_CORE_APP.route('/<path:folder>/<path:filepath>')
def serve_static(folder, filepath):
    """Serve static files from the build directory"""

    if not folder in ['bower_components', 'img', 'scripts', 'styles']:
        abort(404)

    return send_from_directory(os.path.join(STATIC_ROOT, folder), filepath)

@FLASK_CORE_APP.route('/tokensignin', methods=['POST'])
def sign_in():
    """Authenticate a user with Google sign in"""

    try:
        token = request.form['idtoken']
    except KeyError:
        return 'No idtoken provided', 401

    try:
        idinfo = client.verify_id_token(token, KEYS['OAUTH_CLIENT_ID'])

        if idinfo['aud'] != KEYS['OAUTH_CLIENT_ID']:
            raise crypt.AppIdentityError('Unrecognized client.')

        if idinfo['iss'] not in ['accounts.google.com', 'https://accounts.google.com']:
            raise crypt.AppIdentityError('Wrong issuer.')

    except crypt.AppIdentityError, e:
        return 'Did not successfully authenticate'

    user = User.from_google_id(idinfo['sub'])

    data = {
        'isNewUser': not user.exists()
    }

    if user.exists():
        data['userId'] = user.id

    return jsonify(**data)

@FLASK_CORE_APP.route('/api/<string:version>/<path:endpoint_path>')
def api_call(version, endpoint_path):
    """Call an API endpoint directly over HTTP"""

    (path, path_parameters) = path_parser.parse(version, endpoint_path)

    message = {
        'id': 0,
        'method': request.method,
        'path': path,
        'parameters': {
            'path': path_parameters
        }
    }

    return jsonify(message)

@SOCKET_IO_CORE.on('request')
def receive_message(message):
    """"Receive a SocketIO request"""

    try:
        request = rest.Request(message)
        response = request.process()

        flask_socketio.emit('response', response.to_JSON(), json=True)

        print 'Socket: {} to `/{}` resulted in {}: {}'.format(
            request.method,
            request.path,
            response.status['code'],
            response.status['description']
        )

        return

    except exceptions.AuthorizationTokenError as e:
        response = rest.Response(401, 'Authorization error')
        response.id = message['id']

        flask_socketio.emit('response', response.to_JSON(), json=True)

    except exceptions.RequestInitializationError as e:
        response = rest.Response(400, e.message)
        response.id = message['id']

        flask_socketio.emit('response', response.to_JSON(), json=True)

    except Exception as e:
        response = rest.Response(500, 'Internal server error')
        response.id = message['id']

        flask_socketio.emit('response', response.to_JSON(), json=True)
        traceback.print_exc()

    print 'Socket: {} to `{}` resulted in {}: {}'.format(
        message['method'],
        message['path'],
        response.status['code'],
        response.status['description']
    )

SOCKET_IO_CORE.run(FLASK_CORE_APP, host='0.0.0.0', port=8081)
<commit_msg>Add query parameter parsing<commit_after>#!/var/www/opendc.ewi.tudelft.nl/web-server/venv/bin/python

import json
import os
import sys
import traceback

from flask import Flask, abort, request, send_from_directory, session, jsonify
import flask_socketio
from oauth2client import client, crypt

from opendc.models.user import User
from opendc.util import exceptions, rest, path_parser

if len(sys.argv) < 2:
    print "config file path not given as argument"
    sys.exit(1)

# Get keys from config file
with open(sys.argv[1]) as file:
    KEYS = json.load(file)

STATIC_ROOT = os.path.join(KEYS['ROOT_DIR'], 'opendc-frontend', 'build')

FLASK_CORE_APP = Flask(__name__, static_url_path='')
FLASK_CORE_APP.config['SECREY_KEY'] = KEYS['FLASK_SECRET']

SOCKET_IO_CORE = flask_socketio.SocketIO(FLASK_CORE_APP)

@FLASK_CORE_APP.errorhandler(404)
def page_not_found(e):
    return send_from_directory(STATIC_ROOT, '404.html')

@FLASK_CORE_APP.route('/')
def serve_splash():
    """Serve the splash page on /"""

    return send_from_directory(STATIC_ROOT, 'index.html')

@FLASK_CORE_APP.route('/app')
def serve_app():
    """Serve the app on /app."""

    return send_from_directory(STATIC_ROOT, 'app.html')

@FLASK_CORE_APP.route('/profile')
def serve_profile():
    """Serve profile page."""

    return send_from_directory(STATIC_ROOT, 'profile.html')

@FLASK_CORE_APP.route('/projects')
def serve_projects():
    """Serve the projects page."""

    return send_from_directory(STATIC_ROOT, 'projects.html')

@FLASK_CORE_APP.route('/web-server-test')
def serve_web_server_test():
    """Serve the web server test."""

    return send_from_directory(os.path.join(KEYS['ROOT_DIR'], 'opendc-web-server', 'static'), 'index.html')

@FLASK_CORE_APP.route('/<path:folder>/<path:filepath>')
def serve_static(folder, filepath):
    """Serve static files from the build directory"""

    if not folder in ['bower_components', 'img', 'scripts', 'styles']:
        abort(404)

    return send_from_directory(os.path.join(STATIC_ROOT, folder), filepath)

@FLASK_CORE_APP.route('/tokensignin', methods=['POST'])
def sign_in():
    """Authenticate a user with Google sign in"""

    try:
        token = request.form['idtoken']
    except KeyError:
        return 'No idtoken provided', 401

    try:
        idinfo = client.verify_id_token(token, KEYS['OAUTH_CLIENT_ID'])

        if idinfo['aud'] != KEYS['OAUTH_CLIENT_ID']:
            raise crypt.AppIdentityError('Unrecognized client.')

        if idinfo['iss'] not in ['accounts.google.com', 'https://accounts.google.com']:
            raise crypt.AppIdentityError('Wrong issuer.')

    except crypt.AppIdentityError, e:
        return 'Did not successfully authenticate'

    user = User.from_google_id(idinfo['sub'])

    data = {
        'isNewUser': not user.exists()
    }

    if user.exists():
        data['userId'] = user.id

    return jsonify(**data)

@FLASK_CORE_APP.route('/api/<string:version>/<path:endpoint_path>')
def api_call(version, endpoint_path):
    """Call an API endpoint directly over HTTP"""

    (path, path_parameters) = path_parser.parse(version, endpoint_path)

    query_parameters = request.args.to_dict()
    for param in query_parameters:
        try:
            query_parameters[param] = int(query_parameters[param])
        except:
            pass

    message = {
        'id': 0,
        'method': request.method,
        'parameters': {
            'path': path_parameters,
            'query': query_parameters
        },
        'path': path
    }

    return jsonify(message)

@SOCKET_IO_CORE.on('request')
def receive_message(message):
    """"Receive a SocketIO request"""

    try:
        request = rest.Request(message)
        response = request.process()

        flask_socketio.emit('response', response.to_JSON(), json=True)

        print 'Socket: {} to `/{}` resulted in {}: {}'.format(
            request.method,
            request.path,
            response.status['code'],
            response.status['description']
        )

        return

    except exceptions.AuthorizationTokenError as e:
        response = rest.Response(401, 'Authorization error')
        response.id = message['id']

        flask_socketio.emit('response', response.to_JSON(), json=True)

    except exceptions.RequestInitializationError as e:
        response = rest.Response(400, e.message)
        response.id = message['id']

        flask_socketio.emit('response', response.to_JSON(), json=True)

    except Exception as e:
        response = rest.Response(500, 'Internal server error')
        response.id = message['id']

        flask_socketio.emit('response', response.to_JSON(), json=True)
        traceback.print_exc()

    print 'Socket: {} to `{}` resulted in {}: {}'.format(
        message['method'],
        message['path'],
        response.status['code'],
        response.status['description']
    )

SOCKET_IO_CORE.run(FLASK_CORE_APP, host='0.0.0.0', port=8081)
<|endoftext|>


<commit_before>import collections
import hashlib

from couchdbkit import ResourceNotFound
from django_prbac.utils import has_privilege

from corehq import privileges, toggles
from corehq.apps.app_manager.dbaccessors import get_apps_in_domain
from corehq.apps.hqwebapp.templatetags.hq_shared_tags import toggle_enabled
from corehq.apps.linked_domain.util import is_linked_report
from corehq.apps.userreports.adapter import IndicatorAdapterLoadTracker
from corehq.apps.userreports.const import REPORT_BUILDER_EVENTS_KEY, TEMP_REPORT_PREFIX
from corehq.apps.userreports.exceptions import BadSpecError, ReportConfigurationNotFoundError, \
    DataSourceConfigurationNotFoundError
from corehq.apps.userreports.models import is_data_registry_report
from corehq.toggles import ENABLE_UCR_MIRRORS
from corehq.util import reverse
from corehq.util.couch import DocumentNotFound
from corehq.util.metrics.load_counters import ucr_load_counter
from dimagi.utils.couch.undo import is_deleted, remove_deleted_doc_type_suffix

UCR_TABLE_PREFIX = 'ucr_'
LEGACY_UCR_TABLE_PREFIX = 'config_report_'

def localize(value, lang):
    """
    Localize the given value.

    This function is intended to be used within UCR to localize user supplied
    translations.

    :param value: A dict-like object or string
    :param lang: A language code.
    """
    if isinstance(value, collections.Mapping) and len(value):
        return (
            value.get(lang, None) or
            value.get(default_language(), None) or
            value[sorted(value.keys())[0]]
        )
    return value

def default_language():
    return "en"

def has_report_builder_add_on_privilege(request):
    return any(
        has_privilege(request, p) for p in privileges.REPORT_BUILDER_ADD_ON_PRIVS
    )

def has_report_builder_access(request):
    builder_enabled = toggle_enabled(request, toggles.REPORT_BUILDER)
    legacy_builder_priv = has_privilege(request, privileges.REPORT_BUILDER)
    beta_group_enabled = toggle_enabled(request, toggles.REPORT_BUILDER_BETA_GROUP)
    has_add_on_priv = has_report_builder_add_on_privilege(request)

    return (builder_enabled and legacy_builder_priv) or beta_group_enabled or has_add_on_priv

def add_event(request, event):
    events = request.session.get(REPORT_BUILDER_EVENTS_KEY, [])
    request.session[REPORT_BUILDER_EVENTS_KEY] = events + [event]

def has_report_builder_trial(request):
    return has_privilege(request, privileges.REPORT_BUILDER_TRIAL)

def can_edit_report(request, report):
    return _can_edit_report(request, report) and not is_linked_report(report)

def can_delete_report(request, report):
    return _can_edit_report(request, report)

def _can_edit_report(request, report):
    if not request.can_access_all_locations:
        return False

    if not request.couch_user.can_edit_reports():
        return False

    ucr_toggle = toggle_enabled(request, toggles.USER_CONFIGURABLE_REPORTS)
    report_builder_toggle = toggle_enabled(request, toggles.REPORT_BUILDER)
    report_builder_beta_toggle = toggle_enabled(request, toggles.REPORT_BUILDER_BETA_GROUP)
    data_registry_ucr_toggle = toggle_enabled(request, toggles.DATA_REGISTRY_UCR)
    add_on_priv = has_report_builder_add_on_privilege(request)
    created_by_builder = report.report_meta.created_by_builder

    if is_data_registry_report(report) and not data_registry_ucr_toggle:
        # disable editing or deleting DR reports
        return False

    if created_by_builder:
        return report_builder_toggle or report_builder_beta_toggle or add_on_priv
    else:
        return ucr_toggle

def get_referring_apps(domain, report_id):
    to_ret = []
    apps = get_apps_in_domain(domain)
    for app in apps:
        app_url = reverse('view_app', args=[domain, app.id])
        for module in app.get_report_modules():
            module_url = reverse('view_module', args=[domain, app.id, module.unique_id])
            for config in module.report_configs:
                if config.report_id == report_id:
                    to_ret.append({
                        "app_url": app_url,
                        "app_name": app.name,
                        "module_url": module_url,
                        "module_name": module.default_name()
                    })
    return to_ret

def allowed_report_builder_reports(request):
    """
    Return the number of report builder reports allowed
    """
    builder_enabled = toggle_enabled(request, toggles.REPORT_BUILDER)
    legacy_builder_priv = has_privilege(request, privileges.REPORT_BUILDER)
    beta_group_enabled = toggle_enabled(request, toggles.REPORT_BUILDER_BETA_GROUP)

    if toggle_enabled(request, toggles.UNLIMITED_REPORT_BUILDER_REPORTS):
        return float("inf")
    if has_privilege(request, privileges.REPORT_BUILDER_30):
        return 30
    if has_privilege(request, privileges.REPORT_BUILDER_15):
        return 15
    if (
        has_privilege(request, privileges.REPORT_BUILDER_TRIAL) or
        has_privilege(request, privileges.REPORT_BUILDER_5) or
        beta_group_enabled or
        (builder_enabled and legacy_builder_priv)
    ):
        return 5
    return 0

def get_configurable_and_static_reports(domain):
    from corehq.apps.userreports.models import StaticReportConfiguration
    return get_existing_reports(domain) + StaticReportConfiguration.by_domain(domain)

def get_existing_reports(domain):
    from corehq.apps.userreports.models import ReportConfiguration
    existing_reports = ReportConfiguration.by_domain(domain)
    return [
        report for report in existing_reports
        if not (report.title and report.title.startswith(TEMP_REPORT_PREFIX))
    ]

def number_of_report_builder_reports(domain):
    builder_reports = [
        report for report in get_existing_reports(domain)
        if report.report_meta.created_by_builder
    ]
    return len(builder_reports)

def number_of_ucr_reports(domain):
    ucr_reports = [
        report for report in get_existing_reports(domain)
        if not report.report_meta.created_by_builder
    ]
    return len(ucr_reports)

def get_indicator_adapter(config, raise_errors=False, load_source="unknown"):
    from corehq.apps.userreports.sql.adapter import IndicatorSqlAdapter, ErrorRaisingIndicatorSqlAdapter, \
        MultiDBSqlAdapter, ErrorRaisingMultiDBAdapter
    requires_mirroring = config.mirrored_engine_ids
    if requires_mirroring and ENABLE_UCR_MIRRORS.enabled(config.domain):
        adapter_cls = ErrorRaisingMultiDBAdapter if raise_errors else MultiDBSqlAdapter
    else:
        adapter_cls = ErrorRaisingIndicatorSqlAdapter if raise_errors else IndicatorSqlAdapter
    adapter = adapter_cls(config)
    track_load = ucr_load_counter(config.engine_id, load_source, config.domain)
    return IndicatorAdapterLoadTracker(adapter, track_load)

def get_table_name(domain, table_id, max_length=50, prefix=UCR_TABLE_PREFIX):
    """
    :param domain:
    :param table_id:
    :param max_length: Max allowable length of table. Default of 50 to prevent long table
                       name issues with CitusDB. PostgreSQL max is 63.
    :param prefix: Table prefix. Configurable to allow migration from old to new prefix.
    :return:
    """
    def _hash(domain, table_id):
        return hashlib.sha1(
            '{}_{}'.format(
                hashlib.sha1(domain.encode('utf-8')).hexdigest(),
                table_id
            ).encode('utf-8')
        ).hexdigest()[:8]

    domain = domain.encode('unicode-escape').decode('utf-8')
    table_id = table_id.encode('unicode-escape').decode('utf-8')
    return truncate_value(
        '{}{}_{}_{}'.format(prefix, domain, table_id, _hash(domain, table_id)),
        max_length=max_length,
        from_left=False
    )

def truncate_value(value, max_length=63, from_left=True):
    """
    Truncate a value (typically a column name) to a certain number of characters,
    using a hash to ensure uniqueness.
    """
    hash_length = 8
    truncated_length = max_length - hash_length - 1
    value = value.encode('unicode-escape')
    if from_left:
        truncated_value = value[-truncated_length:]
    else:
        truncated_value = value[:truncated_length]

    if len(value) > max_length:
        short_hash = hashlib.sha1(value).hexdigest()[:hash_length]
        return '{}_{}'.format(truncated_value.decode('utf-8'), short_hash)
    return value.decode('utf-8')

def get_ucr_class_name(id):
    """
    This returns the module and class name for a ucr from its id as used in report permissions.
    It takes an id and returns the string that needed for `user.can_view_report(string)`.
    The class name comes from corehq.reports._make_report_class, if something breaks, look there first.
    :param id: the id of the ucr report config
    :return: string class name
    """
    return 'corehq.reports.DynamicReport{}'.format(id)

def get_async_indicator_modify_lock_key(doc_id):
    return 'async_indicator_save-{}'.format(doc_id)

def get_static_report_mapping(from_domain, to_domain):
    from corehq.apps.userreports.models import StaticReportConfiguration, STATIC_PREFIX, \
        CUSTOM_REPORT_PREFIX

    report_map = {}

    for static_report in StaticReportConfiguration.by_domain(from_domain):
        if static_report.get_id.startswith(STATIC_PREFIX):
            report_id = static_report.get_id.replace(
                STATIC_PREFIX + from_domain + '-',
                ''
            )
            is_custom_report = False
        else:
            report_id = static_report.get_id.replace(
                CUSTOM_REPORT_PREFIX + from_domain + '-',
                ''
            )
            is_custom_report = True
        new_id = StaticReportConfiguration.get_doc_id(
            to_domain, report_id, is_custom_report
        )
        # check that new report is in new domain's list of static reports
        try:
            StaticReportConfiguration.by_id(new_id, to_domain)
        except (BadSpecError, DocumentNotFound):
            pass
        else:
            report_map[static_report.get_id] = new_id
    return report_map

def add_tabbed_text(text):
    return '\t' + '\n\t'.join(text.splitlines(False))

def get_report_config_or_not_found(domain, config_id):
    from corehq.apps.userreports.models import ReportConfiguration
    try:
        doc = ReportConfiguration.get_db().get(config_id)
        config = wrap_report_config_by_type(doc)
    except (ResourceNotFound, KeyError, ReportConfigurationNotFoundError):
        raise DocumentNotFound()

    if config.domain != domain:
        raise DocumentNotFound()

    return config

def get_ucr_datasource_config_by_id(indicator_config_id, allow_deleted=False):
    from corehq.apps.userreports.models import (
        id_is_static,
        StaticDataSourceConfiguration,
        DataSourceConfiguration,
    )
    if id_is_static(indicator_config_id):
        return StaticDataSourceConfiguration.by_id(indicator_config_id)
    else:
        doc = DataSourceConfiguration.get_db().get(indicator_config_id)
        return _wrap_data_source_by_doc_type(doc, allow_deleted)

def _wrap_data_source_by_doc_type(doc, allow_deleted=False):
    from corehq.apps.userreports.models import (
        DataSourceConfiguration,
        RegistryDataSourceConfiguration,
    )
    if is_deleted(doc) and not allow_deleted:
        raise DataSourceConfigurationNotFoundError()

    doc_type = remove_deleted_doc_type_suffix(doc["doc_type"])
    return {
        "DataSourceConfiguration": DataSourceConfiguration,
        "RegistryDataSourceConfiguration": RegistryDataSourceConfiguration,
    }[doc_type].wrap(doc)

def wrap_report_config_by_type(config):
    from corehq.apps.userreports.models import (
        ReportConfiguration,
        RegistryReportConfiguration,
    )
    try:
        return {
            "ReportConfiguration": ReportConfiguration,
            "RegistryReportConfiguration": RegistryReportConfiguration,
        }[config["doc_type"]].wrap(config)
    except KeyError:
        raise ReportConfigurationNotFoundError()
<commit_msg>only restrict editing, not deleting<commit_after>import collections
import hashlib

from couchdbkit import ResourceNotFound
from django_prbac.utils import has_privilege

from corehq import privileges, toggles
from corehq.apps.app_manager.dbaccessors import get_apps_in_domain
from corehq.apps.hqwebapp.templatetags.hq_shared_tags import toggle_enabled
from corehq.apps.linked_domain.util import is_linked_report
from corehq.apps.userreports.adapter import IndicatorAdapterLoadTracker
from corehq.apps.userreports.const import REPORT_BUILDER_EVENTS_KEY, TEMP_REPORT_PREFIX
from corehq.apps.userreports.exceptions import BadSpecError, ReportConfigurationNotFoundError, \
    DataSourceConfigurationNotFoundError
from corehq.apps.userreports.models import is_data_registry_report
from corehq.toggles import ENABLE_UCR_MIRRORS
from corehq.util import reverse
from corehq.util.couch import DocumentNotFound
from corehq.util.metrics.load_counters import ucr_load_counter
from dimagi.utils.couch.undo import is_deleted, remove_deleted_doc_type_suffix

UCR_TABLE_PREFIX = 'ucr_'
LEGACY_UCR_TABLE_PREFIX = 'config_report_'

def localize(value, lang):
    """
    Localize the given value.

    This function is intended to be used within UCR to localize user supplied
    translations.

    :param value: A dict-like object or string
    :param lang: A language code.
    """
    if isinstance(value, collections.Mapping) and len(value):
        return (
            value.get(lang, None) or
            value.get(default_language(), None) or
            value[sorted(value.keys())[0]]
        )
    return value

def default_language():
    return "en"

def has_report_builder_add_on_privilege(request):
    return any(
        has_privilege(request, p) for p in privileges.REPORT_BUILDER_ADD_ON_PRIVS
    )

def has_report_builder_access(request):
    builder_enabled = toggle_enabled(request, toggles.REPORT_BUILDER)
    legacy_builder_priv = has_privilege(request, privileges.REPORT_BUILDER)
    beta_group_enabled = toggle_enabled(request, toggles.REPORT_BUILDER_BETA_GROUP)
    has_add_on_priv = has_report_builder_add_on_privilege(request)

    return (builder_enabled and legacy_builder_priv) or beta_group_enabled or has_add_on_priv

def add_event(request, event):
    events = request.session.get(REPORT_BUILDER_EVENTS_KEY, [])
    request.session[REPORT_BUILDER_EVENTS_KEY] = events + [event]

def has_report_builder_trial(request):
    return has_privilege(request, privileges.REPORT_BUILDER_TRIAL)

def can_edit_report(request, report):
    return (
        _can_edit_report(request, report)
        and not is_linked_report(report)
        and (
            not is_data_registry_report(report)
            or toggles.DATA_REGISTRY_UCR.enabled_for_request(request)
        ))

def can_delete_report(request, report):
    return _can_edit_report(request, report)

def _can_edit_report(request, report):
    if not request.can_access_all_locations:
        return False

    if not request.couch_user.can_edit_reports():
        return False

    ucr_toggle = toggle_enabled(request, toggles.USER_CONFIGURABLE_REPORTS)
    report_builder_toggle = toggle_enabled(request, toggles.REPORT_BUILDER)
    report_builder_beta_toggle = toggle_enabled(request, toggles.REPORT_BUILDER_BETA_GROUP)
    add_on_priv = has_report_builder_add_on_privilege(request)
    created_by_builder = report.report_meta.created_by_builder

    if created_by_builder:
        return report_builder_toggle or report_builder_beta_toggle or add_on_priv
    else:
        return ucr_toggle

def get_referring_apps(domain, report_id):
    to_ret = []
    apps = get_apps_in_domain(domain)
    for app in apps:
        app_url = reverse('view_app', args=[domain, app.id])
        for module in app.get_report_modules():
            module_url = reverse('view_module', args=[domain, app.id, module.unique_id])
            for config in module.report_configs:
                if config.report_id == report_id:
                    to_ret.append({
                        "app_url": app_url,
                        "app_name": app.name,
                        "module_url": module_url,
                        "module_name": module.default_name()
                    })
    return to_ret

def allowed_report_builder_reports(request):
    """
    Return the number of report builder reports allowed
    """
    builder_enabled = toggle_enabled(request, toggles.REPORT_BUILDER)
    legacy_builder_priv = has_privilege(request, privileges.REPORT_BUILDER)
    beta_group_enabled = toggle_enabled(request, toggles.REPORT_BUILDER_BETA_GROUP)

    if toggle_enabled(request, toggles.UNLIMITED_REPORT_BUILDER_REPORTS):
        return float("inf")
    if has_privilege(request, privileges.REPORT_BUILDER_30):
        return 30
    if has_privilege(request, privileges.REPORT_BUILDER_15):
        return 15
    if (
        has_privilege(request, privileges.REPORT_BUILDER_TRIAL) or
        has_privilege(request, privileges.REPORT_BUILDER_5) or
        beta_group_enabled or
        (builder_enabled and legacy_builder_priv)
    ):
        return 5
    return 0

def get_configurable_and_static_reports(domain):
    from corehq.apps.userreports.models import StaticReportConfiguration
    return get_existing_reports(domain) + StaticReportConfiguration.by_domain(domain)

def get_existing_reports(domain):
    from corehq.apps.userreports.models import ReportConfiguration
    existing_reports = ReportConfiguration.by_domain(domain)
    return [
        report for report in existing_reports
        if not (report.title and report.title.startswith(TEMP_REPORT_PREFIX))
    ]

def number_of_report_builder_reports(domain):
    builder_reports = [
        report for report in get_existing_reports(domain)
        if report.report_meta.created_by_builder
    ]
    return len(builder_reports)

def number_of_ucr_reports(domain):
    ucr_reports = [
        report for report in get_existing_reports(domain)
        if not report.report_meta.created_by_builder
    ]
    return len(ucr_reports)

def get_indicator_adapter(config, raise_errors=False, load_source="unknown"):
    from corehq.apps.userreports.sql.adapter import IndicatorSqlAdapter, ErrorRaisingIndicatorSqlAdapter, \
        MultiDBSqlAdapter, ErrorRaisingMultiDBAdapter
    requires_mirroring = config.mirrored_engine_ids
    if requires_mirroring and ENABLE_UCR_MIRRORS.enabled(config.domain):
        adapter_cls = ErrorRaisingMultiDBAdapter if raise_errors else MultiDBSqlAdapter
    else:
        adapter_cls = ErrorRaisingIndicatorSqlAdapter if raise_errors else IndicatorSqlAdapter
    adapter = adapter_cls(config)
    track_load = ucr_load_counter(config.engine_id, load_source, config.domain)
    return IndicatorAdapterLoadTracker(adapter, track_load)

def get_table_name(domain, table_id, max_length=50, prefix=UCR_TABLE_PREFIX):
    """
    :param domain:
    :param table_id:
    :param max_length: Max allowable length of table. Default of 50 to prevent long table
                       name issues with CitusDB. PostgreSQL max is 63.
    :param prefix: Table prefix. Configurable to allow migration from old to new prefix.
    :return:
    """
    def _hash(domain, table_id):
        return hashlib.sha1(
            '{}_{}'.format(
                hashlib.sha1(domain.encode('utf-8')).hexdigest(),
                table_id
            ).encode('utf-8')
        ).hexdigest()[:8]

    domain = domain.encode('unicode-escape').decode('utf-8')
    table_id = table_id.encode('unicode-escape').decode('utf-8')
    return truncate_value(
        '{}{}_{}_{}'.format(prefix, domain, table_id, _hash(domain, table_id)),
        max_length=max_length,
        from_left=False
    )

def truncate_value(value, max_length=63, from_left=True):
    """
    Truncate a value (typically a column name) to a certain number of characters,
    using a hash to ensure uniqueness.
    """
    hash_length = 8
    truncated_length = max_length - hash_length - 1
    value = value.encode('unicode-escape')
    if from_left:
        truncated_value = value[-truncated_length:]
    else:
        truncated_value = value[:truncated_length]

    if len(value) > max_length:
        short_hash = hashlib.sha1(value).hexdigest()[:hash_length]
        return '{}_{}'.format(truncated_value.decode('utf-8'), short_hash)
    return value.decode('utf-8')

def get_ucr_class_name(id):
    """
    This returns the module and class name for a ucr from its id as used in report permissions.
    It takes an id and returns the string that needed for `user.can_view_report(string)`.
    The class name comes from corehq.reports._make_report_class, if something breaks, look there first.
    :param id: the id of the ucr report config
    :return: string class name
    """
    return 'corehq.reports.DynamicReport{}'.format(id)

def get_async_indicator_modify_lock_key(doc_id):
    return 'async_indicator_save-{}'.format(doc_id)

def get_static_report_mapping(from_domain, to_domain):
    from corehq.apps.userreports.models import StaticReportConfiguration, STATIC_PREFIX, \
        CUSTOM_REPORT_PREFIX

    report_map = {}

    for static_report in StaticReportConfiguration.by_domain(from_domain):
        if static_report.get_id.startswith(STATIC_PREFIX):
            report_id = static_report.get_id.replace(
                STATIC_PREFIX + from_domain + '-',
                ''
            )
            is_custom_report = False
        else:
            report_id = static_report.get_id.replace(
                CUSTOM_REPORT_PREFIX + from_domain + '-',
                ''
            )
            is_custom_report = True
        new_id = StaticReportConfiguration.get_doc_id(
            to_domain, report_id, is_custom_report
        )
        # check that new report is in new domain's list of static reports
        try:
            StaticReportConfiguration.by_id(new_id, to_domain)
        except (BadSpecError, DocumentNotFound):
            pass
        else:
            report_map[static_report.get_id] = new_id
    return report_map

def add_tabbed_text(text):
    return '\t' + '\n\t'.join(text.splitlines(False))

def get_report_config_or_not_found(domain, config_id):
    from corehq.apps.userreports.models import ReportConfiguration
    try:
        doc = ReportConfiguration.get_db().get(config_id)
        config = wrap_report_config_by_type(doc)
    except (ResourceNotFound, KeyError, ReportConfigurationNotFoundError):
        raise DocumentNotFound()

    if config.domain != domain:
        raise DocumentNotFound()

    return config

def get_ucr_datasource_config_by_id(indicator_config_id, allow_deleted=False):
    from corehq.apps.userreports.models import (
        id_is_static,
        StaticDataSourceConfiguration,
        DataSourceConfiguration,
    )
    if id_is_static(indicator_config_id):
        return StaticDataSourceConfiguration.by_id(indicator_config_id)
    else:
        doc = DataSourceConfiguration.get_db().get(indicator_config_id)
        return _wrap_data_source_by_doc_type(doc, allow_deleted)

def _wrap_data_source_by_doc_type(doc, allow_deleted=False):
    from corehq.apps.userreports.models import (
        DataSourceConfiguration,
        RegistryDataSourceConfiguration,
    )
    if is_deleted(doc) and not allow_deleted:
        raise DataSourceConfigurationNotFoundError()

    doc_type = remove_deleted_doc_type_suffix(doc["doc_type"])
    return {
        "DataSourceConfiguration": DataSourceConfiguration,
        "RegistryDataSourceConfiguration": RegistryDataSourceConfiguration,
    }[doc_type].wrap(doc)

def wrap_report_config_by_type(config):
    from corehq.apps.userreports.models import (
        ReportConfiguration,
        RegistryReportConfiguration,
    )
    try:
        return {
            "ReportConfiguration": ReportConfiguration,
            "RegistryReportConfiguration": RegistryReportConfiguration,
        }[config["doc_type"]].wrap(config)
    except KeyError:
        raise ReportConfigurationNotFoundError()
<|endoftext|>


<commit_before>import pygame
from Tile import Grass, Path

def main():
    """ Set up the game and run the main game loop """
    pygame.init()      # Prepare the pygame module for use
    surface_sz = 500   # Desired physical surface size, in pixels.

    # Create surface of (width, height), and its window.
    main_surface = pygame.display.set_mode((surface_sz, surface_sz))

    map_bw = pygame.image.load("map_bw.png")

    # Set up some data to describe a small rectangle and its color
    grid = [[None for x in range(10)] for y in range(10)]
    for x in range(10):
        for y in range(10):
            if map_bw.get_at((x,y))[0] <= 50:
                grid[x][y] = Path(x,y)
            else:
                grid[x][y] = Grass(x,y)

    while True:
        ev = pygame.event.poll()    # Look for any event
        if ev.type == pygame.QUIT:  # Window close button clicked?
            break                   #   ... leave game loop

        # Update your game objects and data structures here...

        # We draw everything from scratch on each frame.
        # So first fill everything with the background color
        main_surface.fill((0, 200, 255))

        # Overpaint a smaller rectangle on the main surface
        for row in grid:
            for cell in row:
                main_surface.fill(cell.color, (cell.pxPos,cell.size))
        #main_surface.fill(some_color, small_rect)

        # Now the surface is ready, tell pygame to display it!
        pygame.display.flip()

    pygame.quit()     # Once we leave the loop, close the window.

main()<commit_msg>Bitte markdown benutzen und kein docx/etc<commit_after>#!/usr/bin/env python3

import pygame
from Tile import Grass, Path

def main():
    """ Set up the game and run the main game loop """
    pygame.init()      # Prepare the pygame module for use
    surface_sz = 500   # Desired physical surface size, in pixels.

    # Create surface of (width, height), and its window.
    main_surface = pygame.display.set_mode((surface_sz, surface_sz))

    map_bw = pygame.image.load("map_bw.png")

    # Set up some data to describe a small rectangle and its color
    grid = [[None for x in range(10)] for y in range(10)]
    for x in range(10):
        for y in range(10):
            if map_bw.get_at((x,y))[0] <= 50:
                grid[x][y] = Path(x,y)
            else:
                grid[x][y] = Grass(x,y)

    while True:
        ev = pygame.event.poll()    # Look for any event
        if ev.type == pygame.QUIT:  # Window close button clicked?
            break                   #   ... leave game loop

        # Update your game objects and data structures here...

        # We draw everything from scratch on each frame.
        # So first fill everything with the background color
        main_surface.fill((0, 200, 255))

        # Overpaint a smaller rectangle on the main surface
        for row in grid:
            for cell in row:
                main_surface.fill(cell.color, (cell.pxPos,cell.size))
        #main_surface.fill(some_color, small_rect)

        # Now the surface is ready, tell pygame to display it!
        pygame.display.flip()

    pygame.quit()     # Once we leave the loop, close the window.

main()
<|endoftext|>