<commit_before>import arrow
import settings
from . import misc
from . import voting
from . import exceptions as exc

def merge_pr(api, urn, pr, votes, total, threshold):
    """ merge a pull request, if possible, and use a nice detailed merge commit
    message """

    pr_num = pr["number"]
    pr_title = pr['title']
    pr_description = pr['body']

    path = "/repos/{urn}/pulls/{pr}/merge".format(urn=urn, pr=pr_num)

    record = voting.friendly_voting_record(votes)
    if record:
        record = "Vote record:\n" + record

    votes_summary = formatted_votes_summary(votes, total, threshold)

    pr_url = "https://github.com/{urn}/pull/{pr}".format(urn=urn, pr=pr_num)

    title = "merging PR #{num}: {pr_title}".format(num=pr_num, pr_title=pr_title)
    desc = """
{pr_url}: {pr_title}

Description:
{pr_description}

:ok_woman: PR passed {summary}.

{record}
""".strip().format(
    pr_url=pr_url,
    pr_title=pr_title,
    pr_description=pr_description,
    summary=votes_summary,
    record=record,
    )

    data = {
        "commit_title": title,
        "commit_message": desc,
        "merge_method": "merge",

        # if some clever person attempts to submit more commits while we're
        # aggregating votes, this sha check will fail and no merge will occur
        "sha": pr["head"]["sha"],
    }
    try:
        resp = api("PUT", path, json=data)
        return resp["sha"]
    except HTTPError as e:
        resp = e.response
        # could not be merged
        if resp.status_code == 405:
            raise exc.CouldntMerge
        # someone trying to be sneaky and change their PR commits during voting
        elif resp.status_code == 409:
            raise exc.CouldntMerge
        else:
            raise

def formatted_votes_summary(votes, total, threshold):
    vfor = sum(v for v in votes.values() if v > 0)
    vagainst = abs(sum(v for v in votes.values() if v < 0))

    return "with a vote of {vfor} for and {vagainst} against, with a weighted total of {total:.1f} and a threshold of {threshold:.1f}" \
        .strip().format(vfor=vfor, vagainst=vagainst, total=total, threshold=threshold)

def formatted_votes_short_summary(votes, total, threshold):
    vfor = sum(v for v in votes.values() if v > 0)
    vagainst = abs(sum(v for v in votes.values() if v < 0))

    return "vote: {vfor}-{vagainst}, weighted total: {total:.1f}, threshold: {threshold:.1f}" \
        .strip().format(vfor=vfor, vagainst=vagainst, total=total, threshold=threshold)

def label_pr(api, urn, pr_num, labels):
    """ set a pr's labels (removes old labels) """
    if not isinstance(labels, (tuple, list)):
        labels = [labels]
    path = "/repos/{urn}/issues/{pr}/labels".format(urn=urn, pr=pr_num)
    data = labels
    resp = api("PUT", path, json=data)

def close_pr(api, urn, pr):
    """ https://developer.github.com/v3/pulls/#update-a-pull-request """
    path = "/repos/{urn}/pulls/{pr}".format(urn=urn, pr=pr["number"])
    data = {
        "state": "closed",
    }
    return api("patch", path, json=data)

def get_pr_last_updated(pr_data):
    """ a helper for finding the utc datetime of the last pr branch
    modifications """
    repo = pr_data["head"]["repo"]
    if repo:
        dt = repo["pushed_at"]
    else:
        dt = pr_data["created_at"]
    return arrow.get(dt)

def get_pr_comments(api, urn, pr_num):
    """ yield all comments on a pr, weirdly excluding the initial pr comment
    itself (the one the owner makes) """
    params = {
        "per_page": settings.DEFAULT_PAGINATION
    }
    path = "/repos/{urn}/issues/{pr}/comments".format(urn=urn, pr=pr_num)
    comments = api("get", path, params=params)
    for comment in comments:
        yield comment

def get_ready_prs(api, urn, window):
    """ yield mergeable, non-WIP prs that have had no modifications for longer
    than the voting window.  these are prs that are ready to be considered for
    merging """
    open_prs = get_open_prs(api, urn)
    for pr in open_prs:
        pr_num = pr["number"]

        now = arrow.utcnow()
        updated = get_pr_last_updated(pr)
        delta = (now - updated).total_seconds()

        is_wip = "WIP" in pr["title"]

        if not is_wip and delta > window:
            # we check if its mergeable if its outside the voting window,
            # because there seems to be a race where a freshly-created PR exists
            # in the paginated list of PRs, but 404s when trying to fetch it
            # directly
            mergeable = get_is_mergeable(api, urn, pr_num)
            if mergeable is True:
                label_pr(api, urn, pr_num, ["mergeable"])
                yield pr
            elif mergeable is False:
                label_pr(api, urn, pr_num, ["conflicts"])
            # mergeable can also be None, in which case we just skip it for now

def voting_window_remaining_seconds(pr, window):
    now = arrow.utcnow()
    updated = get_pr_last_updated(pr)
    delta = (now - updated).total_seconds()
    return window - delta

def is_pr_in_voting_window(pr, window):
    return voting_window_remaining_seconds(pr, window) <= 0

def get_pr_reviews(api, urn, pr_num):
    """ get all pr reviews on a pr
    https://help.github.com/articles/about-pull-request-reviews/ """
    params = {
        "per_page": settings.DEFAULT_PAGINATION
    }
    path = "/repos/{urn}/pulls/{pr}/reviews".format(urn=urn, pr=pr_num)
    data = api("get", path, params=params)
    return data

def get_is_mergeable(api, urn, pr_num):
    return get_pr(api, urn, pr_num)["mergeable"]

def get_pr(api, urn, pr_num):
    """ helper for fetching a pr.  necessary because the "mergeable" field does
    not exist on prs that come back from paginated endpoints, so we must fetch
    the pr directly """
    path = "/repos/{urn}/pulls/{pr}".format(urn=urn, pr=pr_num)
    pr = api("get", path)
    return pr

def get_open_prs(api, urn):
    params = {
        "state": "open",
        "sort": "updated",
        "direction": "asc",
        "per_page": settings.DEFAULT_PAGINATION,
    }
    path = "/repos/{urn}/pulls".format(urn=urn)
    data = api("get", path, params=params)
    return data

def get_reactions_for_pr(api, urn, pr):
    path = "/repos/{urn}/issues/{pr}/reactions".format(urn=urn, pr=pr)
    params = {"per_page": settings.DEFAULT_PAGINATION}
    reactions = api("get", path, params=params)
    for reaction in reactions:
        yield reaction

def post_accepted_status(api, urn, pr, voting_window, votes, total, threshold):
    sha = pr["head"]["sha"]

    remaining_seconds = voting_window_remaining_seconds(pr, voting_window)
    remaining_human = misc.seconds_to_human(remaining_seconds)
    votes_summary = formatted_votes_short_summary(votes, total, threshold)

    post_status(api, urn, sha, "success",
                "remaining: {time}, {summary}".format(time=remaining_human, summary=votes_summary))

def post_rejected_status(api, urn, pr, voting_window, votes, total, threshold):
    sha = pr["head"]["sha"]

    remaining_seconds = voting_window_remaining_seconds(pr, voting_window)
    remaining_human = misc.seconds_to_human(remaining_seconds)
    votes_summary = formatted_votes_short_summary(votes, total, threshold)

    post_status(api, urn, sha, "failure",
                "remaining: {time}, {summary}".format(time=remaining_human, summary=votes_summary))

def post_pending_status(api, urn, pr, voting_window, votes, total, threshold):
    sha = pr["head"]["sha"]

    remaining_seconds = voting_window_remaining_seconds(pr, voting_window)
    remaining_human = misc.seconds_to_human(remaining_seconds)
    votes_summary = formatted_votes_short_summary(votes, total, threshold)

    post_status(api, urn, sha, "pending",
                "remaining: {time}, {summary}".format(time=remaining_human, summary=votes_summary))

def post_status(api, urn, sha, state, description):
    """ apply an issue label to a pr """
    path = "/repos/{urn}/statuses/{sha}".format(urn=urn, sha=sha)
    data = {
        "state": state,
        "description": description,
        "context": "chaosbot"
    }
    api("POST", path, json=data)
<commit_msg>merging PR #217: Remove the mergeable tag<commit_after>import arrow
import settings
from . import misc
from . import voting
from . import exceptions as exc

def merge_pr(api, urn, pr, votes, total, threshold):
    """ merge a pull request, if possible, and use a nice detailed merge commit
    message """

    pr_num = pr["number"]
    pr_title = pr['title']
    pr_description = pr['body']

    path = "/repos/{urn}/pulls/{pr}/merge".format(urn=urn, pr=pr_num)

    record = voting.friendly_voting_record(votes)
    if record:
        record = "Vote record:\n" + record

    votes_summary = formatted_votes_summary(votes, total, threshold)

    pr_url = "https://github.com/{urn}/pull/{pr}".format(urn=urn, pr=pr_num)

    title = "merging PR #{num}: {pr_title}".format(num=pr_num, pr_title=pr_title)
    desc = """
{pr_url}: {pr_title}

Description:
{pr_description}

:ok_woman: PR passed {summary}.

{record}
""".strip().format(
    pr_url=pr_url,
    pr_title=pr_title,
    pr_description=pr_description,
    summary=votes_summary,
    record=record,
    )

    data = {
        "commit_title": title,
        "commit_message": desc,
        "merge_method": "merge",

        # if some clever person attempts to submit more commits while we're
        # aggregating votes, this sha check will fail and no merge will occur
        "sha": pr["head"]["sha"],
    }
    try:
        resp = api("PUT", path, json=data)
        return resp["sha"]
    except HTTPError as e:
        resp = e.response
        # could not be merged
        if resp.status_code == 405:
            raise exc.CouldntMerge
        # someone trying to be sneaky and change their PR commits during voting
        elif resp.status_code == 409:
            raise exc.CouldntMerge
        else:
            raise

def formatted_votes_summary(votes, total, threshold):
    vfor = sum(v for v in votes.values() if v > 0)
    vagainst = abs(sum(v for v in votes.values() if v < 0))

    return "with a vote of {vfor} for and {vagainst} against, with a weighted total of {total:.1f} and a threshold of {threshold:.1f}" \
        .strip().format(vfor=vfor, vagainst=vagainst, total=total, threshold=threshold)

def formatted_votes_short_summary(votes, total, threshold):
    vfor = sum(v for v in votes.values() if v > 0)
    vagainst = abs(sum(v for v in votes.values() if v < 0))

    return "vote: {vfor}-{vagainst}, weighted total: {total:.1f}, threshold: {threshold:.1f}" \
        .strip().format(vfor=vfor, vagainst=vagainst, total=total, threshold=threshold)

def label_pr(api, urn, pr_num, labels):
    """ set a pr's labels (removes old labels) """
    if not isinstance(labels, (tuple, list)):
        labels = [labels]
    path = "/repos/{urn}/issues/{pr}/labels".format(urn=urn, pr=pr_num)
    data = labels
    resp = api("PUT", path, json=data)

def close_pr(api, urn, pr):
    """ https://developer.github.com/v3/pulls/#update-a-pull-request """
    path = "/repos/{urn}/pulls/{pr}".format(urn=urn, pr=pr["number"])
    data = {
        "state": "closed",
    }
    return api("patch", path, json=data)

def get_pr_last_updated(pr_data):
    """ a helper for finding the utc datetime of the last pr branch
    modifications """
    repo = pr_data["head"]["repo"]
    if repo:
        dt = repo["pushed_at"]
    else:
        dt = pr_data["created_at"]
    return arrow.get(dt)

def get_pr_comments(api, urn, pr_num):
    """ yield all comments on a pr, weirdly excluding the initial pr comment
    itself (the one the owner makes) """
    params = {
        "per_page": settings.DEFAULT_PAGINATION
    }
    path = "/repos/{urn}/issues/{pr}/comments".format(urn=urn, pr=pr_num)
    comments = api("get", path, params=params)
    for comment in comments:
        yield comment

def get_ready_prs(api, urn, window):
    """ yield mergeable, non-WIP prs that have had no modifications for longer
    than the voting window.  these are prs that are ready to be considered for
    merging """
    open_prs = get_open_prs(api, urn)
    for pr in open_prs:
        pr_num = pr["number"]

        now = arrow.utcnow()
        updated = get_pr_last_updated(pr)
        delta = (now - updated).total_seconds()

        is_wip = "WIP" in pr["title"]

        if not is_wip and delta > window:
            # we check if its mergeable if its outside the voting window,
            # because there seems to be a race where a freshly-created PR exists
            # in the paginated list of PRs, but 404s when trying to fetch it
            # directly
            mergeable = get_is_mergeable(api, urn, pr_num)
            if mergeable is True:
                label_pr(api, urn, pr_num, [])
                yield pr
            elif mergeable is False:
                label_pr(api, urn, pr_num, ["conflicts"])
            # mergeable can also be None, in which case we just skip it for now

def voting_window_remaining_seconds(pr, window):
    now = arrow.utcnow()
    updated = get_pr_last_updated(pr)
    delta = (now - updated).total_seconds()
    return window - delta

def is_pr_in_voting_window(pr, window):
    return voting_window_remaining_seconds(pr, window) <= 0

def get_pr_reviews(api, urn, pr_num):
    """ get all pr reviews on a pr
    https://help.github.com/articles/about-pull-request-reviews/ """
    params = {
        "per_page": settings.DEFAULT_PAGINATION
    }
    path = "/repos/{urn}/pulls/{pr}/reviews".format(urn=urn, pr=pr_num)
    data = api("get", path, params=params)
    return data

def get_is_mergeable(api, urn, pr_num):
    return get_pr(api, urn, pr_num)["mergeable"]

def get_pr(api, urn, pr_num):
    """ helper for fetching a pr.  necessary because the "mergeable" field does
    not exist on prs that come back from paginated endpoints, so we must fetch
    the pr directly """
    path = "/repos/{urn}/pulls/{pr}".format(urn=urn, pr=pr_num)
    pr = api("get", path)
    return pr

def get_open_prs(api, urn):
    params = {
        "state": "open",
        "sort": "updated",
        "direction": "asc",
        "per_page": settings.DEFAULT_PAGINATION,
    }
    path = "/repos/{urn}/pulls".format(urn=urn)
    data = api("get", path, params=params)
    return data

def get_reactions_for_pr(api, urn, pr):
    path = "/repos/{urn}/issues/{pr}/reactions".format(urn=urn, pr=pr)
    params = {"per_page": settings.DEFAULT_PAGINATION}
    reactions = api("get", path, params=params)
    for reaction in reactions:
        yield reaction

def post_accepted_status(api, urn, pr, voting_window, votes, total, threshold):
    sha = pr["head"]["sha"]

    remaining_seconds = voting_window_remaining_seconds(pr, voting_window)
    remaining_human = misc.seconds_to_human(remaining_seconds)
    votes_summary = formatted_votes_short_summary(votes, total, threshold)

    post_status(api, urn, sha, "success",
                "remaining: {time}, {summary}".format(time=remaining_human, summary=votes_summary))

def post_rejected_status(api, urn, pr, voting_window, votes, total, threshold):
    sha = pr["head"]["sha"]

    remaining_seconds = voting_window_remaining_seconds(pr, voting_window)
    remaining_human = misc.seconds_to_human(remaining_seconds)
    votes_summary = formatted_votes_short_summary(votes, total, threshold)

    post_status(api, urn, sha, "failure",
                "remaining: {time}, {summary}".format(time=remaining_human, summary=votes_summary))

def post_pending_status(api, urn, pr, voting_window, votes, total, threshold):
    sha = pr["head"]["sha"]

    remaining_seconds = voting_window_remaining_seconds(pr, voting_window)
    remaining_human = misc.seconds_to_human(remaining_seconds)
    votes_summary = formatted_votes_short_summary(votes, total, threshold)

    post_status(api, urn, sha, "pending",
                "remaining: {time}, {summary}".format(time=remaining_human, summary=votes_summary))

def post_status(api, urn, sha, state, description):
    """ apply an issue label to a pr """
    path = "/repos/{urn}/statuses/{sha}".format(urn=urn, sha=sha)
    data = {
        "state": state,
        "description": description,
        "context": "chaosbot"
    }
    api("POST", path, json=data)
<|endoftext|>


<commit_before>import collections
import copy
import datetime

from jsonobject.exceptions import BadValueError

from casexml.apps.case.xform import extract_case_blocks, get_case_ids_from_form
from corehq.apps.change_feed import topics
from corehq.apps.change_feed.consumer.feed import KafkaChangeFeed, MultiTopicCheckpointEventHandler
from corehq.apps.receiverwrapper.util import get_app_version_info
from corehq.elastic import get_es_new
from corehq.form_processor.change_providers import SqlFormChangeProvider
from corehq.form_processor.utils.xform import add_couch_properties_to_sql_form_json
from corehq.pillows.mappings.xform_mapping import XFORM_MAPPING, XFORM_INDEX, XFORM_ES_TYPE
from corehq.pillows.utils import get_user_type
from couchforms.jsonobject_extensions import GeoPointProperty
from .base import HQPillow
from couchforms.const import RESERVED_WORDS, DEVICE_LOG_XMLNS
from couchforms.models import XFormInstance
from dateutil import parser
from pillowtop.checkpoints.manager import PillowCheckpoint, PillowCheckpointEventHandler
from pillowtop.es_utils import ElasticsearchIndexInfo, get_index_info_from_pillow
from pillowtop.pillow.interface import ConstructedPillow
from pillowtop.processors.elastic import ElasticProcessor
from pillowtop.processors.form import AppFormSubmissionTrackerProcessor
from pillowtop.reindexer.change_providers.couch import CouchViewChangeProvider
from pillowtop.reindexer.reindexer import get_default_reindexer_for_elastic_pillow, \
    ElasticPillowReindexer, ResumableBulkElasticPillowReindexer

UNKNOWN_VERSION = 'XXX'
UNKNOWN_UIVERSION = 'XXX'

def is_valid_date(txt):
    try:
        if txt and parser.parse(txt):
            return True
    except Exception:
        pass
    return False

# modified from: http://stackoverflow.com/questions/6027558/flatten-nested-python-dictionaries-compressing-keys
def flatten(d, parent_key='', delimiter='/'):
    items = []
    for k, v in d.items():
        if k in RESERVED_WORDS:
            continue
        new_key = parent_key + delimiter + k if parent_key else k
        if isinstance(v, collections.MutableMapping):
            items.extend(flatten(v, new_key, delimiter).items())
        elif not isinstance(v, list):
            items.append((new_key, v))
    return dict(items)

class XFormPillow(HQPillow):
    document_class = XFormInstance
    couch_filter = "couchforms/xforms"
    es_alias = "xforms"
    es_type = XFORM_ES_TYPE
    es_index = XFORM_INDEX
    include_docs = False

    # for simplicity, the handlers are managed on the domain level
    handler_domain_map = {}
    default_mapping = XFORM_MAPPING

    def change_transform(self, doc_dict, include_props=True):
        return transform_xform_for_elasticsearch(doc_dict, include_props)

def device_log_filter(doc_dict):
    """
    :return: True to filter out doc
    """
    return doc_dict.get('xmlns', None) == DEVICE_LOG_XMLNS

def transform_xform_for_elasticsearch(doc_dict, include_props=True):
    """
    Given an XFormInstance, return a copy that is ready to be sent to elasticsearch,
    or None, if the form should not be saved to elasticsearch
    """
    if doc_dict.get('domain', None) is None or doc_dict['form'] is None:
        # if there is no domain don't bother processing it
        return None
    else:
        doc_ret = copy.deepcopy(doc_dict)

        if 'meta' in doc_ret['form']:
            if not is_valid_date(doc_ret['form']['meta'].get('timeEnd', None)):
                doc_ret['form']['meta']['timeEnd'] = None
            if not is_valid_date(doc_ret['form']['meta'].get('timeStart', None)):
                doc_ret['form']['meta']['timeStart'] = None

            # Some docs have their @xmlns and #text here
            if isinstance(doc_ret['form']['meta'].get('appVersion'), dict):
                doc_ret['form']['meta']['appVersion'] = doc_ret['form']['meta']['appVersion'].get('#text')

            app_version_info = get_app_version_info(
                doc_ret['domain'],
                doc_ret.get('build_id'),
                doc_ret.get('version'),
                doc_ret['form']['meta'],
            )
            doc_ret['form']['meta']['commcare_version'] = app_version_info.commcare_version
            doc_ret['form']['meta']['app_build_version'] = app_version_info.build_version

            try:
                geo_point = GeoPointProperty().wrap(doc_ret['form']['meta']['location'])
                doc_ret['form']['meta']['geo_point'] = geo_point.lat_lon
            except (KeyError, BadValueError):
                doc_ret['form']['meta']['geo_point'] = None
                pass

        try:
            user_id = doc_ret['form']['meta']['userID']
        except KeyError:
            user_id = None
        doc_ret['user_type'] = get_user_type(user_id)
        doc_ret['inserted_at'] = datetime.datetime.utcnow().isoformat()

        case_blocks = extract_case_blocks(doc_ret)
        for case_dict in case_blocks:
            for date_modified_key in ['date_modified', '@date_modified']:
                if not is_valid_date(case_dict.get(date_modified_key, None)):
                    if case_dict.get(date_modified_key) == '':
                        case_dict[date_modified_key] = None
                    else:
                        case_dict.pop(date_modified_key, None)

            # convert all mapped dict properties to nulls if they are empty strings
            for object_key in ['index', 'attachment', 'create', 'update']:
                if object_key in case_dict and not isinstance(case_dict[object_key], dict):
                    case_dict[object_key] = None

        doc_ret["__retrieved_case_ids"] = list(get_case_ids_from_form(doc_dict))
        if include_props:
            form_props = ["%s:%s" % (k, v) for k, v in flatten(doc_ret['form']).iteritems()]
            doc_ret["__props_for_querying"] = form_props
        return doc_ret

def prepare_sql_form_json_for_elasticsearch(sql_form_json):
    prepped_form = transform_xform_for_elasticsearch(sql_form_json)
    if prepped_form:
        add_couch_properties_to_sql_form_json(prepped_form)

    return prepped_form

def get_sql_xform_to_elasticsearch_pillow(pillow_id='SqlXFormToElasticsearchPillow'):
    checkpoint = PillowCheckpoint(
        'sql-xforms-to-elasticsearch',
    )
    form_processor = ElasticProcessor(
        elasticsearch=get_es_new(),
        index_info=ElasticsearchIndexInfo(index=XFORM_INDEX, type=XFORM_ES_TYPE),
        doc_prep_fn=prepare_sql_form_json_for_elasticsearch
    )
    return ConstructedPillow(
        name=pillow_id,
        checkpoint=checkpoint,
        change_feed=KafkaChangeFeed(topics=[topics.FORM_SQL], group_id='sql-forms-to-es'),
        processor=form_processor,
        change_processed_event_handler=PillowCheckpointEventHandler(
            checkpoint=checkpoint, checkpoint_frequency=100,
        ),
    )

def get_couch_form_reindexer():
    return get_default_reindexer_for_elastic_pillow(
        pillow=XFormPillow(online=False),
        change_provider=CouchViewChangeProvider(
            couch_db=XFormInstance.get_db(),
            view_name='all_docs/by_doc_type',
            view_kwargs={
                'startkey': ['XFormInstance'],
                'endkey': ['XFormInstance', {}],
                'include_docs': True,
            }
        )
    )

def get_resumable_couch_form_reindexer():
    return ResumableBulkElasticPillowReindexer(
        name=XFormPillow(online=False).pillow_id,
        doc_types=[XFormInstance],
        elasticsearch=get_es_new(),
        index_info=_get_xform_index_info(),
        doc_filter=device_log_filter,
        doc_transform=transform_xform_for_elasticsearch
    )

def get_app_form_submission_tracker_pillow(pillow_id='AppFormSubmissionTrackerPillow'):
    """
    This gets a pillow which iterates through all forms and marks the corresponding app
    as having submissions. This could be expanded to be more generic and include
    other processing that needs to happen on each form
    """
    checkpoint = PillowCheckpoint('app-form-submission-tracker')
    form_processor = AppFormSubmissionTrackerProcessor()
    change_feed = KafkaChangeFeed(topics=[topics.FORM, topics.FORM_SQL], group_id='form-processsor')
    return ConstructedPillow(
        name=pillow_id,
        checkpoint=checkpoint,
        change_feed=change_feed,
        processor=form_processor,
        change_processed_event_handler=MultiTopicCheckpointEventHandler(
            checkpoint=checkpoint, checkpoint_frequency=100, change_feed=change_feed,
        ),
    )

def get_sql_form_reindexer():
    return ElasticPillowReindexer(
        pillow=get_sql_xform_to_elasticsearch_pillow(),
        change_provider=SqlFormChangeProvider(),
        elasticsearch=get_es_new(),
        index_info=_get_xform_index_info(),
    )

def _get_xform_index_info():
    return get_index_info_from_pillow(XFormPillow(online=False))
<commit_msg>remove unused kwarg<commit_after>import collections
import copy
import datetime

from jsonobject.exceptions import BadValueError

from casexml.apps.case.xform import extract_case_blocks, get_case_ids_from_form
from corehq.apps.change_feed import topics
from corehq.apps.change_feed.consumer.feed import KafkaChangeFeed, MultiTopicCheckpointEventHandler
from corehq.apps.receiverwrapper.util import get_app_version_info
from corehq.elastic import get_es_new
from corehq.form_processor.change_providers import SqlFormChangeProvider
from corehq.form_processor.utils.xform import add_couch_properties_to_sql_form_json
from corehq.pillows.mappings.xform_mapping import XFORM_MAPPING, XFORM_INDEX, XFORM_ES_TYPE
from corehq.pillows.utils import get_user_type
from couchforms.jsonobject_extensions import GeoPointProperty
from .base import HQPillow
from couchforms.const import RESERVED_WORDS, DEVICE_LOG_XMLNS
from couchforms.models import XFormInstance
from dateutil import parser
from pillowtop.checkpoints.manager import PillowCheckpoint, PillowCheckpointEventHandler
from pillowtop.es_utils import ElasticsearchIndexInfo, get_index_info_from_pillow
from pillowtop.pillow.interface import ConstructedPillow
from pillowtop.processors.elastic import ElasticProcessor
from pillowtop.processors.form import AppFormSubmissionTrackerProcessor
from pillowtop.reindexer.change_providers.couch import CouchViewChangeProvider
from pillowtop.reindexer.reindexer import get_default_reindexer_for_elastic_pillow, \
    ElasticPillowReindexer, ResumableBulkElasticPillowReindexer

UNKNOWN_VERSION = 'XXX'
UNKNOWN_UIVERSION = 'XXX'

def is_valid_date(txt):
    try:
        if txt and parser.parse(txt):
            return True
    except Exception:
        pass
    return False

# modified from: http://stackoverflow.com/questions/6027558/flatten-nested-python-dictionaries-compressing-keys
def flatten(d, parent_key='', delimiter='/'):
    items = []
    for k, v in d.items():
        if k in RESERVED_WORDS:
            continue
        new_key = parent_key + delimiter + k if parent_key else k
        if isinstance(v, collections.MutableMapping):
            items.extend(flatten(v, new_key, delimiter).items())
        elif not isinstance(v, list):
            items.append((new_key, v))
    return dict(items)

class XFormPillow(HQPillow):
    document_class = XFormInstance
    couch_filter = "couchforms/xforms"
    es_alias = "xforms"
    es_type = XFORM_ES_TYPE
    es_index = XFORM_INDEX
    include_docs = False

    # for simplicity, the handlers are managed on the domain level
    handler_domain_map = {}
    default_mapping = XFORM_MAPPING

    def change_transform(self, doc_dict):
        return transform_xform_for_elasticsearch(doc_dict)

def device_log_filter(doc_dict):
    """
    :return: True to filter out doc
    """
    return doc_dict.get('xmlns', None) == DEVICE_LOG_XMLNS

def transform_xform_for_elasticsearch(doc_dict, include_props=True):
    """
    Given an XFormInstance, return a copy that is ready to be sent to elasticsearch,
    or None, if the form should not be saved to elasticsearch
    """
    if doc_dict.get('domain', None) is None or doc_dict['form'] is None:
        # if there is no domain don't bother processing it
        return None
    else:
        doc_ret = copy.deepcopy(doc_dict)

        if 'meta' in doc_ret['form']:
            if not is_valid_date(doc_ret['form']['meta'].get('timeEnd', None)):
                doc_ret['form']['meta']['timeEnd'] = None
            if not is_valid_date(doc_ret['form']['meta'].get('timeStart', None)):
                doc_ret['form']['meta']['timeStart'] = None

            # Some docs have their @xmlns and #text here
            if isinstance(doc_ret['form']['meta'].get('appVersion'), dict):
                doc_ret['form']['meta']['appVersion'] = doc_ret['form']['meta']['appVersion'].get('#text')

            app_version_info = get_app_version_info(
                doc_ret['domain'],
                doc_ret.get('build_id'),
                doc_ret.get('version'),
                doc_ret['form']['meta'],
            )
            doc_ret['form']['meta']['commcare_version'] = app_version_info.commcare_version
            doc_ret['form']['meta']['app_build_version'] = app_version_info.build_version

            try:
                geo_point = GeoPointProperty().wrap(doc_ret['form']['meta']['location'])
                doc_ret['form']['meta']['geo_point'] = geo_point.lat_lon
            except (KeyError, BadValueError):
                doc_ret['form']['meta']['geo_point'] = None
                pass

        try:
            user_id = doc_ret['form']['meta']['userID']
        except KeyError:
            user_id = None
        doc_ret['user_type'] = get_user_type(user_id)
        doc_ret['inserted_at'] = datetime.datetime.utcnow().isoformat()

        case_blocks = extract_case_blocks(doc_ret)
        for case_dict in case_blocks:
            for date_modified_key in ['date_modified', '@date_modified']:
                if not is_valid_date(case_dict.get(date_modified_key, None)):
                    if case_dict.get(date_modified_key) == '':
                        case_dict[date_modified_key] = None
                    else:
                        case_dict.pop(date_modified_key, None)

            # convert all mapped dict properties to nulls if they are empty strings
            for object_key in ['index', 'attachment', 'create', 'update']:
                if object_key in case_dict and not isinstance(case_dict[object_key], dict):
                    case_dict[object_key] = None

        doc_ret["__retrieved_case_ids"] = list(get_case_ids_from_form(doc_dict))
        if include_props:
            form_props = ["%s:%s" % (k, v) for k, v in flatten(doc_ret['form']).iteritems()]
            doc_ret["__props_for_querying"] = form_props
        return doc_ret

def prepare_sql_form_json_for_elasticsearch(sql_form_json):
    prepped_form = transform_xform_for_elasticsearch(sql_form_json)
    if prepped_form:
        add_couch_properties_to_sql_form_json(prepped_form)

    return prepped_form

def get_sql_xform_to_elasticsearch_pillow(pillow_id='SqlXFormToElasticsearchPillow'):
    checkpoint = PillowCheckpoint(
        'sql-xforms-to-elasticsearch',
    )
    form_processor = ElasticProcessor(
        elasticsearch=get_es_new(),
        index_info=ElasticsearchIndexInfo(index=XFORM_INDEX, type=XFORM_ES_TYPE),
        doc_prep_fn=prepare_sql_form_json_for_elasticsearch
    )
    return ConstructedPillow(
        name=pillow_id,
        checkpoint=checkpoint,
        change_feed=KafkaChangeFeed(topics=[topics.FORM_SQL], group_id='sql-forms-to-es'),
        processor=form_processor,
        change_processed_event_handler=PillowCheckpointEventHandler(
            checkpoint=checkpoint, checkpoint_frequency=100,
        ),
    )

def get_couch_form_reindexer():
    return get_default_reindexer_for_elastic_pillow(
        pillow=XFormPillow(online=False),
        change_provider=CouchViewChangeProvider(
            couch_db=XFormInstance.get_db(),
            view_name='all_docs/by_doc_type',
            view_kwargs={
                'startkey': ['XFormInstance'],
                'endkey': ['XFormInstance', {}],
                'include_docs': True,
            }
        )
    )

def get_resumable_couch_form_reindexer():
    return ResumableBulkElasticPillowReindexer(
        name=XFormPillow(online=False).pillow_id,
        doc_types=[XFormInstance],
        elasticsearch=get_es_new(),
        index_info=_get_xform_index_info(),
        doc_filter=device_log_filter,
        doc_transform=transform_xform_for_elasticsearch
    )

def get_app_form_submission_tracker_pillow(pillow_id='AppFormSubmissionTrackerPillow'):
    """
    This gets a pillow which iterates through all forms and marks the corresponding app
    as having submissions. This could be expanded to be more generic and include
    other processing that needs to happen on each form
    """
    checkpoint = PillowCheckpoint('app-form-submission-tracker')
    form_processor = AppFormSubmissionTrackerProcessor()
    change_feed = KafkaChangeFeed(topics=[topics.FORM, topics.FORM_SQL], group_id='form-processsor')
    return ConstructedPillow(
        name=pillow_id,
        checkpoint=checkpoint,
        change_feed=change_feed,
        processor=form_processor,
        change_processed_event_handler=MultiTopicCheckpointEventHandler(
            checkpoint=checkpoint, checkpoint_frequency=100, change_feed=change_feed,
        ),
    )

def get_sql_form_reindexer():
    return ElasticPillowReindexer(
        pillow=get_sql_xform_to_elasticsearch_pillow(),
        change_provider=SqlFormChangeProvider(),
        elasticsearch=get_es_new(),
        index_info=_get_xform_index_info(),
    )

def _get_xform_index_info():
    return get_index_info_from_pillow(XFormPillow(online=False))
<|endoftext|>


<commit_before>"""
Copyright (c) 2013
Eric Casteleijn, <thisfred@gmail.com>
"""

__version__ = '0.2.0'

class NotValid(Exception):

    def __init__(self, msg):
        self.msg = msg

def parse_schema(schema):
    if isinstance(schema, Schema):
        return schema.validate

    if type(schema) is type:

        def type_validator(data):
            if isinstance(data, schema):
                return data

            raise NotValid('%r is not of type %r' % (data, schema))

        return type_validator

    if isinstance(schema, dict):

        optional = {}
        mandatory = {}
        types = {}
        for key, value in schema.items():
            if isinstance(key, Optional):
                optional[key.value] = parse_schema(value)
                continue

            if type(key) is type:
                types[key] = parse_schema(value)
                continue

            mandatory[key] = parse_schema(value)

        def dict_validator(data):
            if not isinstance(data, dict):
                raise NotValid('%r is not of type dict' % (data,))
            validated = {}
            to_validate = data.keys()
            for key, sub_schema in mandatory.items():
                if key not in data:
                    raise NotValid('missing key: %r' % (key,))
                try:
                    validated[key] = sub_schema(data[key])
                except NotValid, e:
                    raise NotValid('%s: %s' % (key, e.msg))
                to_validate.remove(key)
            for key in to_validate:
                value = data[key]
                if key in optional:
                    try:
                        validated[key] = optional[key](value)
                    except NotValid, e:
                        raise NotValid('%s: %s' % (key, e.msg))
                    continue  # pragma: nocover
                for key_schema, value_schema in types.items():
                    if not isinstance(key, key_schema):
                        continue
                    try:
                        validated[key] = value_schema(value)
                    except NotValid:
                        continue
                    else:
                        break
                else:
                    raise NotValid('key %r and value %s not matched' % (
                        key, value))

            return validated

        return dict_validator

    if type(schema) in (list, tuple, set):

        sub_schemas = [parse_schema(s) for s in schema]

        def item_validator(value):
            for sub in sub_schemas:
                try:
                    return sub(value)

                except NotValid:
                    pass

            raise NotValid('%r not validated by anything in %s.' % (
                value, schema))

        def collection_validator(data):
            if not type(data) is type(schema):
                raise NotValid('%r is not of type %s' % (data, type(schema)))

            return type(schema)(item_validator(value) for value in data)

        return collection_validator

    if callable(schema):

        def callable_validator(data):
            try:
                if schema(data):
                    return data
                else:
                    raise NotValid('%r does not satisfy %r' % (data, schema))
            except (TypeError, ValueError), e:
                raise NotValid(e)

        return callable_validator

    def static_validator(data):
        if data == schema:
            return data

        raise NotValid('%r is not equal to %r' % (data, schema))

    return static_validator

class Schema(object):

    def __init__(self, schema):
        self._orig_schema = schema
        self.schema = parse_schema(schema)

    def __repr__(self):
        return repr(self._orig_schema)

    def validate(self, data):
        return self.schema(data)

    def validates(self, data):
        try:
            self.validate(data)
            return True
        except NotValid:
            return False

class Optional(object):

    def __init__(self, value):
        self.value = value

    def __repr__(self):
        return "<Optional: %r>" % (self.value,)

class Or(Schema):

    def __init__(self, *values):
        self.values = values
        self.schemas = tuple(parse_schema(s) for s in values)

    def validate(self, data):
        for sub in self.schemas:
            try:
                return sub(data)
            except NotValid:
                pass
        raise NotValid('%r not validated by %r' % (data, self.values))

    def __repr__(self):
        return "<%s>" % (" or ".join(["%r" % (v,) for v in self.values]),)

class And(Schema):

    def __init__(self, *values):
        self.values = values
        self.schemas = tuple(parse_schema(s) for s in values)

    def validate(self, data):
        for sub in self.schemas:
            data = sub(data)
        return data

    def __repr__(self):
        return "<%s>" % (" and ".join(["%r" % (v,) for v in self.values]),)

class Convert(Schema):

    def __init__(self, conversion):
        self.convert = conversion

    def validate(self, data):
        try:
            return self.convert(data)
        except (TypeError, ValueError), e:
            raise NotValid(e)

    def __repr__(self):
        return '<Convert: %r>' % (self.convert,)

class Ordered(Schema):

    def __init__(self, schemas):
        self._orig_schema = schemas
        self.schemas = type(schemas)(Schema(s) for s in schemas)
        self.length = len(self.schemas)

    def validate(self, values):
        if self.length != len(values):
            raise NotValid(
                "Expected %d values, got %d" % (self.length, len(values)))
        return type(self.schemas)(
            self.schemas[i].validate(v) for i, v in enumerate(values))

    def __repr__(self):
        return '<Ordered: %r>' % (self.schemas,)
<commit_msg>more consistent validation error messages. show all the errors when Or validation fails.<commit_after>"""
Copyright (c) 2013
Eric Casteleijn, <thisfred@gmail.com>
"""

__version__ = '0.2.1'

class NotValid(Exception):

    def __init__(self, msg):
        self.msg = msg

def parse_schema(schema):
    if isinstance(schema, Schema):
        return schema.validate

    if type(schema) is type:

        def type_validator(data):
            if isinstance(data, schema):
                return data

            raise NotValid('%r is not of type %r' % (data, schema))

        return type_validator

    if isinstance(schema, dict):

        optional = {}
        mandatory = {}
        types = {}
        for key, value in schema.items():
            if isinstance(key, Optional):
                optional[key.value] = parse_schema(value)
                continue

            if type(key) is type:
                types[key] = parse_schema(value)
                continue

            mandatory[key] = parse_schema(value)

        def dict_validator(data):
            if not isinstance(data, dict):
                raise NotValid('%r is not of type dict' % (data,))
            validated = {}
            to_validate = data.keys()
            for key, sub_schema in mandatory.items():
                if key not in data:
                    raise NotValid('missing key: %r' % (key,))
                try:
                    validated[key] = sub_schema(data[key])
                except NotValid, e:
                    raise NotValid('%s: %s' % (key, e.msg))
                to_validate.remove(key)
            for key in to_validate:
                value = data[key]
                if key in optional:
                    try:
                        validated[key] = optional[key](value)
                    except NotValid, e:
                        raise NotValid('%s: %s' % (key, e.msg))
                    continue  # pragma: nocover
                for key_schema, value_schema in types.items():
                    if not isinstance(key, key_schema):
                        continue
                    try:
                        validated[key] = value_schema(value)
                    except NotValid:
                        continue
                    else:
                        break
                else:
                    raise NotValid('key %r and value %s not matched' % (
                        key, value))

            return validated

        return dict_validator

    if type(schema) in (list, tuple, set):

        sub_schemas = [parse_schema(s) for s in schema]

        def item_validator(value):
            for sub in sub_schemas:
                try:
                    return sub(value)

                except NotValid:
                    pass

            raise NotValid('%r not validated by anything in %s.' % (
                value, schema))

        def collection_validator(data):
            if not type(data) is type(schema):
                raise NotValid('%r is not of type %s' % (data, type(schema)))

            return type(schema)(item_validator(value) for value in data)

        return collection_validator

    if callable(schema):

        def callable_validator(data):
            try:
                if schema(data):
                    return data
                else:
                    raise NotValid('%r does not satisfy %r' % (data, schema))
            except (TypeError, ValueError), e:
                raise NotValid(e)

        return callable_validator

    def static_validator(data):
        if data == schema:
            return data

        raise NotValid('%r is not equal to %r' % (data, schema))

    return static_validator

class Schema(object):

    def __init__(self, schema):
        self._orig_schema = schema
        self.schema = parse_schema(schema)

    def __repr__(self):
        return repr(self._orig_schema)

    def validate(self, data):
        return self.schema(data)

    def validates(self, data):
        try:
            self.validate(data)
            return True
        except NotValid:
            return False

class Optional(object):

    def __init__(self, value):
        self.value = value

    def __repr__(self):
        return "<Optional: %r>" % (self.value,)

class Or(Schema):

    def __init__(self, *values):
        self.values = values
        self.schemas = tuple(parse_schema(s) for s in values)

    def validate(self, data):
        errors = []
        for sub in self.schemas:
            try:
                return sub(data)
            except NotValid, e:
                errors.append(e.msg)
        raise NotValid('%r not validated:\n\n  - %s' % (
            data, '\n  - '.join(errors)))

    def __repr__(self):
        return "<%s>" % (" or ".join(["%r" % (v,) for v in self.values]),)

class And(Schema):

    def __init__(self, *values):
        self.values = values
        self.schemas = tuple(parse_schema(s) for s in values)

    def validate(self, data):
        for sub in self.schemas:
            data = sub(data)
        return data

    def __repr__(self):
        return "<%s>" % (" and ".join(["%r" % (v,) for v in self.values]),)

class Convert(Schema):

    def __init__(self, conversion):
        self.convert = conversion

    def validate(self, data):
        try:
            return self.convert(data)
        except (TypeError, ValueError), e:
            raise NotValid(e)

    def __repr__(self):
        return '<Convert: %r>' % (self.convert,)

class Ordered(Schema):

    def __init__(self, schemas):
        self._orig_schema = schemas
        self.schemas = type(schemas)(Schema(s) for s in schemas)
        self.length = len(self.schemas)

    def validate(self, values):
        if self.length != len(values):
            raise NotValid(
                "Does not have exactly %d values. (Got %d.)" % (
                    self.length, len(values)))
        return type(self.schemas)(
            self.schemas[i].validate(v) for i, v in enumerate(values))

    def __repr__(self):
        return '<Ordered: %r>' % (self.schemas,)
<|endoftext|>


<commit_before>from __future__ import unicode_literals

import importlib

def lazy_load(module_name, element):
    def f(*args, **kwargs):
        module = importlib.import_module(module_name, "moto")
        return getattr(module, element)(*args, **kwargs)

    return f

mock_acm = lazy_load(".acm", "mock_acm")
mock_apigateway = lazy_load(".apigateway", "mock_apigateway")
mock_apigateway_deprecated = lazy_load(".apigateway", "mock_apigateway_deprecated")
mock_athena = lazy_load(".athena", "mock_athena")
mock_applicationautoscaling = lazy_load(
    ".applicationautoscaling", "mock_applicationautoscaling"
)
mock_autoscaling = lazy_load(".autoscaling", "mock_autoscaling")
mock_autoscaling_deprecated = lazy_load(".autoscaling", "mock_autoscaling_deprecated")
mock_lambda = lazy_load(".awslambda", "mock_lambda")
mock_lambda_deprecated = lazy_load(".awslambda", "mock_lambda_deprecated")
mock_batch = lazy_load(".batch", "mock_batch")
mock_batch = lazy_load(".batch", "mock_batch")
mock_cloudformation = lazy_load(".cloudformation", "mock_cloudformation")
mock_cloudformation_deprecated = lazy_load(
    ".cloudformation", "mock_cloudformation_deprecated"
)
mock_cloudwatch = lazy_load(".cloudwatch", "mock_cloudwatch")
mock_cloudwatch_deprecated = lazy_load(".cloudwatch", "mock_cloudwatch_deprecated")
mock_codecommit = lazy_load(".codecommit", "mock_codecommit")
mock_codepipeline = lazy_load(".codepipeline", "mock_codepipeline")
mock_cognitoidentity = lazy_load(".cognitoidentity", "mock_cognitoidentity")
mock_cognitoidentity_deprecated = lazy_load(
    ".cognitoidentity", "mock_cognitoidentity_deprecated"
)
mock_cognitoidp = lazy_load(".cognitoidp", "mock_cognitoidp")
mock_cognitoidp_deprecated = lazy_load(".cognitoidp", "mock_cognitoidp_deprecated")
mock_config = lazy_load(".config", "mock_config")
mock_datapipeline = lazy_load(".datapipeline", "mock_datapipeline")
mock_datapipeline_deprecated = lazy_load(
    ".datapipeline", "mock_datapipeline_deprecated"
)
mock_datasync = lazy_load(".datasync", "mock_datasync")
mock_dynamodb = lazy_load(".dynamodb", "mock_dynamodb")
mock_dynamodb_deprecated = lazy_load(".dynamodb", "mock_dynamodb_deprecated")
mock_dynamodb2 = lazy_load(".dynamodb2", "mock_dynamodb2")
mock_dynamodb2_deprecated = lazy_load(".dynamodb2", "mock_dynamodb2_deprecated")
mock_dynamodbstreams = lazy_load(".dynamodbstreams", "mock_dynamodbstreams")
mock_elasticbeanstalk = lazy_load(".elasticbeanstalk", "mock_elasticbeanstalk")
mock_ec2 = lazy_load(".ec2", "mock_ec2")
mock_ec2_deprecated = lazy_load(".ec2", "mock_ec2_deprecated")
mock_ec2instanceconnect = lazy_load(".ec2instanceconnect", "mock_ec2instanceconnect")
mock_ecr = lazy_load(".ecr", "mock_ecr")
mock_ecr_deprecated = lazy_load(".ecr", "mock_ecr_deprecated")
mock_ecs = lazy_load(".ecs", "mock_ecs")
mock_ecs_deprecated = lazy_load(".ecs", "mock_ecs_deprecated")
mock_elb = lazy_load(".elb", "mock_elb")
mock_elb_deprecated = lazy_load(".elb", "mock_elb_deprecated")
mock_elbv2 = lazy_load(".elbv2", "mock_elbv2")
mock_emr = lazy_load(".emr", "mock_emr")
mock_emr_deprecated = lazy_load(".emr", "mock_emr_deprecated")
mock_events = lazy_load(".events", "mock_events")
mock_forecast = lazy_load(".forecast", "mock_forecast")
mock_glacier = lazy_load(".glacier", "mock_glacier")
mock_glacier_deprecated = lazy_load(".glacier", "mock_glacier_deprecated")
mock_glue = lazy_load(".glue", "mock_glue")
mock_iam = lazy_load(".iam", "mock_iam")
mock_iam_deprecated = lazy_load(".iam", "mock_iam_deprecated")
mock_iot = lazy_load(".iot", "mock_iot")
mock_iotdata = lazy_load(".iotdata", "mock_iotdata")
mock_kinesis = lazy_load(".kinesis", "mock_kinesis")
mock_kinesis_deprecated = lazy_load(".kinesis", "mock_kinesis_deprecated")
mock_kms = lazy_load(".kms", "mock_kms")
mock_kms_deprecated = lazy_load(".kms", "mock_kms_deprecated")
mock_logs = lazy_load(".logs", "mock_logs")
mock_logs_deprecated = lazy_load(".logs", "mock_logs_deprecated")
mock_managedblockchain = lazy_load(".managedblockchain", "mock_managedblockchain")
mock_opsworks = lazy_load(".opsworks", "mock_opsworks")
mock_opsworks_deprecated = lazy_load(".opsworks", "mock_opsworks_deprecated")
mock_organizations = lazy_load(".organizations", "mock_organizations")
mock_polly = lazy_load(".polly", "mock_polly")
mock_ram = lazy_load(".ram", "mock_ram")
mock_rds = lazy_load(".rds", "mock_rds")
mock_rds_deprecated = lazy_load(".rds", "mock_rds_deprecated")
mock_rds2 = lazy_load(".rds2", "mock_rds2")
mock_rds2_deprecated = lazy_load(".rds2", "mock_rds2_deprecated")
mock_redshift = lazy_load(".redshift", "mock_redshift")
mock_redshift_deprecated = lazy_load(".redshift", "mock_redshift_deprecated")
mock_resourcegroups = lazy_load(".resourcegroups", "mock_resourcegroups")
mock_resourcegroupstaggingapi = lazy_load(
    ".resourcegroupstaggingapi", "mock_resourcegroupstaggingapi"
)
mock_route53 = lazy_load(".route53", "mock_route53")
mock_route53_deprecated = lazy_load(".route53", "mock_route53_deprecated")
mock_s3 = lazy_load(".s3", "mock_s3")
mock_s3_deprecated = lazy_load(".s3", "mock_s3_deprecated")
mock_sagemaker = lazy_load(".sagemaker", "mock_sagemaker")
mock_secretsmanager = lazy_load(".secretsmanager", "mock_secretsmanager")
mock_ses = lazy_load(".ses", "mock_ses")
mock_ses_deprecated = lazy_load(".ses", "mock_ses_deprecated")
mock_sns = lazy_load(".sns", "mock_sns")
mock_sns_deprecated = lazy_load(".sns", "mock_sns_deprecated")
mock_sqs = lazy_load(".sqs", "mock_sqs")
mock_sqs_deprecated = lazy_load(".sqs", "mock_sqs_deprecated")
mock_ssm = lazy_load(".ssm", "mock_ssm")
mock_stepfunctions = lazy_load(".stepfunctions", "mock_stepfunctions")
mock_sts = lazy_load(".sts", "mock_sts")
mock_sts_deprecated = lazy_load(".sts", "mock_sts_deprecated")
mock_swf = lazy_load(".swf", "mock_swf")
mock_swf_deprecated = lazy_load(".swf", "mock_swf_deprecated")
mock_transcribe = lazy_load(".transcribe", "mock_transcribe")
XRaySegment = lazy_load(".xray", "XRaySegment")
mock_xray = lazy_load(".xray", "mock_xray")
mock_xray_client = lazy_load(".xray", "mock_xray_client")
mock_kinesisvideo = lazy_load(".kinesisvideo", "mock_kinesisvideo")
mock_kinesisvideoarchivedmedia = lazy_load(
    ".kinesisvideoarchivedmedia", "mock_kinesisvideoarchivedmedia"
)
mock_medialive = lazy_load(".medialive", "mock_medialive")
mock_support = lazy_load(".support", "mock_support")
mock_mediaconnect = lazy_load(".mediaconnect", "mock_mediaconnect")
mock_mediapackage = lazy_load(".mediapackage", "mock_mediapackage")

# import logging
# logging.getLogger('boto').setLevel(logging.CRITICAL)

__title__ = "moto"
__version__ = "2.0.4.dev"

try:
    # Need to monkey-patch botocore requests back to underlying urllib3 classes
    from botocore.awsrequest import (
        HTTPSConnectionPool,
        HTTPConnectionPool,
        HTTPConnection,
        VerifiedHTTPSConnection,
    )
except ImportError:
    pass
else:
    HTTPSConnectionPool.ConnectionCls = VerifiedHTTPSConnection
    HTTPConnectionPool.ConnectionCls = HTTPConnection
<commit_msg>Increase version number post-release<commit_after>from __future__ import unicode_literals

import importlib

def lazy_load(module_name, element):
    def f(*args, **kwargs):
        module = importlib.import_module(module_name, "moto")
        return getattr(module, element)(*args, **kwargs)

    return f

mock_acm = lazy_load(".acm", "mock_acm")
mock_apigateway = lazy_load(".apigateway", "mock_apigateway")
mock_apigateway_deprecated = lazy_load(".apigateway", "mock_apigateway_deprecated")
mock_athena = lazy_load(".athena", "mock_athena")
mock_applicationautoscaling = lazy_load(
    ".applicationautoscaling", "mock_applicationautoscaling"
)
mock_autoscaling = lazy_load(".autoscaling", "mock_autoscaling")
mock_autoscaling_deprecated = lazy_load(".autoscaling", "mock_autoscaling_deprecated")
mock_lambda = lazy_load(".awslambda", "mock_lambda")
mock_lambda_deprecated = lazy_load(".awslambda", "mock_lambda_deprecated")
mock_batch = lazy_load(".batch", "mock_batch")
mock_batch = lazy_load(".batch", "mock_batch")
mock_cloudformation = lazy_load(".cloudformation", "mock_cloudformation")
mock_cloudformation_deprecated = lazy_load(
    ".cloudformation", "mock_cloudformation_deprecated"
)
mock_cloudwatch = lazy_load(".cloudwatch", "mock_cloudwatch")
mock_cloudwatch_deprecated = lazy_load(".cloudwatch", "mock_cloudwatch_deprecated")
mock_codecommit = lazy_load(".codecommit", "mock_codecommit")
mock_codepipeline = lazy_load(".codepipeline", "mock_codepipeline")
mock_cognitoidentity = lazy_load(".cognitoidentity", "mock_cognitoidentity")
mock_cognitoidentity_deprecated = lazy_load(
    ".cognitoidentity", "mock_cognitoidentity_deprecated"
)
mock_cognitoidp = lazy_load(".cognitoidp", "mock_cognitoidp")
mock_cognitoidp_deprecated = lazy_load(".cognitoidp", "mock_cognitoidp_deprecated")
mock_config = lazy_load(".config", "mock_config")
mock_datapipeline = lazy_load(".datapipeline", "mock_datapipeline")
mock_datapipeline_deprecated = lazy_load(
    ".datapipeline", "mock_datapipeline_deprecated"
)
mock_datasync = lazy_load(".datasync", "mock_datasync")
mock_dynamodb = lazy_load(".dynamodb", "mock_dynamodb")
mock_dynamodb_deprecated = lazy_load(".dynamodb", "mock_dynamodb_deprecated")
mock_dynamodb2 = lazy_load(".dynamodb2", "mock_dynamodb2")
mock_dynamodb2_deprecated = lazy_load(".dynamodb2", "mock_dynamodb2_deprecated")
mock_dynamodbstreams = lazy_load(".dynamodbstreams", "mock_dynamodbstreams")
mock_elasticbeanstalk = lazy_load(".elasticbeanstalk", "mock_elasticbeanstalk")
mock_ec2 = lazy_load(".ec2", "mock_ec2")
mock_ec2_deprecated = lazy_load(".ec2", "mock_ec2_deprecated")
mock_ec2instanceconnect = lazy_load(".ec2instanceconnect", "mock_ec2instanceconnect")
mock_ecr = lazy_load(".ecr", "mock_ecr")
mock_ecr_deprecated = lazy_load(".ecr", "mock_ecr_deprecated")
mock_ecs = lazy_load(".ecs", "mock_ecs")
mock_ecs_deprecated = lazy_load(".ecs", "mock_ecs_deprecated")
mock_elb = lazy_load(".elb", "mock_elb")
mock_elb_deprecated = lazy_load(".elb", "mock_elb_deprecated")
mock_elbv2 = lazy_load(".elbv2", "mock_elbv2")
mock_emr = lazy_load(".emr", "mock_emr")
mock_emr_deprecated = lazy_load(".emr", "mock_emr_deprecated")
mock_events = lazy_load(".events", "mock_events")
mock_forecast = lazy_load(".forecast", "mock_forecast")
mock_glacier = lazy_load(".glacier", "mock_glacier")
mock_glacier_deprecated = lazy_load(".glacier", "mock_glacier_deprecated")
mock_glue = lazy_load(".glue", "mock_glue")
mock_iam = lazy_load(".iam", "mock_iam")
mock_iam_deprecated = lazy_load(".iam", "mock_iam_deprecated")
mock_iot = lazy_load(".iot", "mock_iot")
mock_iotdata = lazy_load(".iotdata", "mock_iotdata")
mock_kinesis = lazy_load(".kinesis", "mock_kinesis")
mock_kinesis_deprecated = lazy_load(".kinesis", "mock_kinesis_deprecated")
mock_kms = lazy_load(".kms", "mock_kms")
mock_kms_deprecated = lazy_load(".kms", "mock_kms_deprecated")
mock_logs = lazy_load(".logs", "mock_logs")
mock_logs_deprecated = lazy_load(".logs", "mock_logs_deprecated")
mock_managedblockchain = lazy_load(".managedblockchain", "mock_managedblockchain")
mock_opsworks = lazy_load(".opsworks", "mock_opsworks")
mock_opsworks_deprecated = lazy_load(".opsworks", "mock_opsworks_deprecated")
mock_organizations = lazy_load(".organizations", "mock_organizations")
mock_polly = lazy_load(".polly", "mock_polly")
mock_ram = lazy_load(".ram", "mock_ram")
mock_rds = lazy_load(".rds", "mock_rds")
mock_rds_deprecated = lazy_load(".rds", "mock_rds_deprecated")
mock_rds2 = lazy_load(".rds2", "mock_rds2")
mock_rds2_deprecated = lazy_load(".rds2", "mock_rds2_deprecated")
mock_redshift = lazy_load(".redshift", "mock_redshift")
mock_redshift_deprecated = lazy_load(".redshift", "mock_redshift_deprecated")
mock_resourcegroups = lazy_load(".resourcegroups", "mock_resourcegroups")
mock_resourcegroupstaggingapi = lazy_load(
    ".resourcegroupstaggingapi", "mock_resourcegroupstaggingapi"
)
mock_route53 = lazy_load(".route53", "mock_route53")
mock_route53_deprecated = lazy_load(".route53", "mock_route53_deprecated")
mock_s3 = lazy_load(".s3", "mock_s3")
mock_s3_deprecated = lazy_load(".s3", "mock_s3_deprecated")
mock_sagemaker = lazy_load(".sagemaker", "mock_sagemaker")
mock_secretsmanager = lazy_load(".secretsmanager", "mock_secretsmanager")
mock_ses = lazy_load(".ses", "mock_ses")
mock_ses_deprecated = lazy_load(".ses", "mock_ses_deprecated")
mock_sns = lazy_load(".sns", "mock_sns")
mock_sns_deprecated = lazy_load(".sns", "mock_sns_deprecated")
mock_sqs = lazy_load(".sqs", "mock_sqs")
mock_sqs_deprecated = lazy_load(".sqs", "mock_sqs_deprecated")
mock_ssm = lazy_load(".ssm", "mock_ssm")
mock_stepfunctions = lazy_load(".stepfunctions", "mock_stepfunctions")
mock_sts = lazy_load(".sts", "mock_sts")
mock_sts_deprecated = lazy_load(".sts", "mock_sts_deprecated")
mock_swf = lazy_load(".swf", "mock_swf")
mock_swf_deprecated = lazy_load(".swf", "mock_swf_deprecated")
mock_transcribe = lazy_load(".transcribe", "mock_transcribe")
XRaySegment = lazy_load(".xray", "XRaySegment")
mock_xray = lazy_load(".xray", "mock_xray")
mock_xray_client = lazy_load(".xray", "mock_xray_client")
mock_kinesisvideo = lazy_load(".kinesisvideo", "mock_kinesisvideo")
mock_kinesisvideoarchivedmedia = lazy_load(
    ".kinesisvideoarchivedmedia", "mock_kinesisvideoarchivedmedia"
)
mock_medialive = lazy_load(".medialive", "mock_medialive")
mock_support = lazy_load(".support", "mock_support")
mock_mediaconnect = lazy_load(".mediaconnect", "mock_mediaconnect")
mock_mediapackage = lazy_load(".mediapackage", "mock_mediapackage")

# import logging
# logging.getLogger('boto').setLevel(logging.CRITICAL)

__title__ = "moto"
__version__ = "2.0.5.dev"

try:
    # Need to monkey-patch botocore requests back to underlying urllib3 classes
    from botocore.awsrequest import (
        HTTPSConnectionPool,
        HTTPConnectionPool,
        HTTPConnection,
        VerifiedHTTPSConnection,
    )
except ImportError:
    pass
else:
    HTTPSConnectionPool.ConnectionCls = VerifiedHTTPSConnection
    HTTPConnectionPool.ConnectionCls = HTTPConnection
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
###############################################################################
#
# ODOO (ex OpenERP)
# Open Source Management Solution
# Copyright (C) 2001-2015 Micronaet S.r.l. (<https://micronaet.com>)
# Developer: Nicola Riolini @thebrush (<https://it.linkedin.com/in/thebrush>)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################

import os
import logging
from openerp.osv import fields, osv, expression
from datetime import datetime, timedelta
from openerp.tools.translate import _
from openerp.tools import (DEFAULT_SERVER_DATE_FORMAT,
    DEFAULT_SERVER_DATETIME_FORMAT,
    DATETIME_FORMATS_MAP,
    float_compare)
try:
    import xlrd
except:
    pass
import pdb

_logger = logging.getLogger(__name__)

class MrpProductionWorkcenterLine(osv.osv):
    """ Model name: Job
    """

    _inherit = 'mrp.production.workcenter.line'

    def get_file(self, url, fullname):
        """ Extract from Dropbox
        """
        import requests
        _logger.info('Downloading file %s from Dropbox' % fullname)
        reply = requests.get(url)
        attachment_data = reply.content
        with open(fullname, 'wb') as f:
            f.write(attachment_data)

    # Override so update after update MRP:
    def update_product_level_from_production(self, cr, uid, ids, context=None):
        """ Update product level from production
        """
        def get_excel_date(value):
            """ Extract ISO date
            """
            months = {
                'GEN': '01',
                'FEB': '02',
                'MAR': '03',
                'APR': '04',
                'MAG': '05',
                'GIU': '06',
                'LUG': '07',
                'AGO': '08',
                'SET': '09',
                'OCT': '10',
                'NOV': '11',
                'DEC': '12',

                'DIC': '12',
            }
            if not value or type(value) in (float, ):
                res = ''
            else:
                value = value.strip()
                value_item = value.split('/')
                if len(value_item) == 3:
                    if len(value_item[2]) == 4:
                        pass  # correct
                    elif len(value_item[2]) == 2:
                        value_item[2] = '20%s' % value_item[2]
                    elif value_item == '209':
                        value_item[2] = '2019'
                    else:
                        _logger.error('Unmanaged error:')

                    if value_item[1].upper() in months:
                        value_item[1] = months[value_item[1].upper()]

                    res = '%s-%02d-%02d' % (
                        value_item[2],
                        int(value_item[1]),
                        int(value_item[0]),
                    )
                else:
                    res = value  # Nothing
            # _logger.warning(' ' * 70 + '>>>>>> From %s to %s' % (value, res))
            return res

        _logger.info('Update marketed product medium (from Excel)')
        product_pool = self.pool.get('product.product')
        company_pool = self.pool.get('res.company')

        # ---------------------------------------------------------------------
        # A. Update marketed product:
        # ---------------------------------------------------------------------
        # Fixed parameters:
        columns_position = {
            'date': 1,
            # 'invoice': 7,
            'default_code': 24,
            # 'name': 21,
            'qty': 26,
            # 'uom': 23,
            # 'price': 24,
        }
        start_test = 'INVOICE DATE'
        sheet_name = 'BASE'

        # Dynamic parameters:
        company_ids = company_pool.search(cr, uid, [], context=context)
        company = company_pool.browse(
           cr, uid, company_ids, context=context)[0]
        stock_level_days = company.stock_level_mm_days
        # TODO use it:
        stock_level_obsolete_days = company.stock_level_obsolete_days

        # Generate file:
        dropbox_link = context.get('dropbox_link')
        if not dropbox_link:
            _logger.error(
                'Setup the scheduled command with dropbox_link parameter')
        filename = '/tmp/account_status.xlsx'
        self.get_file(dropbox_link, filename)
        if not filename:
            raise osv.except_osv(
                _('Error stock management'),
                _('Setup the parameter in company form (input file)'),
                )

        now = datetime.now()
        from_date = now - timedelta(days=stock_level_days)
        from_obsolete_date = now - timedelta(days=stock_level_days)
        now_text = '%s 00:00:00' % now.strftime(
             DEFAULT_SERVER_DATE_FORMAT)
        from_text = '%s 00:00:00' % from_date.strftime(
             DEFAULT_SERVER_DATE_FORMAT)
        from_obsolete_text = '%s 00:00:00' % from_obsolete_date.strftime(
             DEFAULT_SERVER_DATE_FORMAT)

        # A1. Search product marketed:
        cr.execute('''
            SELECT id from product_product
            WHERE default_code is not null AND
                  SUBSTRING (default_code, 1, 1) IN (
                      'D', 'E', 'F', 'G', 'H', 'L', 'M',
                      'O', 'P', 'R', 'S', 'X') AND
                  SUBSTRING (default_code, 1, 3) NOT IN ('OLD', 'SER');
        ''')
        product_ids = [record[0] for record in cr.fetchall()]

        _logger.warning('Imported product #%s [%s - %s]' % (
            len(product_ids),
            from_text,
            now_text,
            ))

        # A2. Prepare dict for medium
        product_medium = {}
        product_obsolete = {}
        for product in product_pool.browse(
                cr, uid, product_ids, context=context):

            default_code = product.default_code
            if not default_code:
                _logger.error('Product %s has no code' % product.name)
                continue

            if default_code not in product_obsolete:
                product_obsolete[default_code] = True

            if default_code.endswith('X'):
                continue
            if default_code in product_medium:
                _logger.error('Product double: %s' % default_code)
            else:
                product_medium[default_code] = [0.0, product]

        # A3. Load data from Excel:
        try:
            wb = xlrd.open_workbook(filename)
        except:
            _logger.error(
                '[ERROR] Cannot read XLS file: %s' % filename
            )
            return False

        ws = wb.sheet_by_name(sheet_name)
        _logger.info('Read XLS file: %s' % filename)
        start = False
        for row in range(ws.nrows):
            date = get_excel_date(ws.cell(row, columns_position['date']).value)
            if not start and date == start_test:
                _logger.info('%s. Line not used: Start line' % (row + 1))
                start = True
                continue

            if date < from_text or date > now_text:  # Out of period range:
                _logger.info('%s. Line not used out of range %s' % (
                    row + 1, date))
                continue

            default_code = ws.cell(row, columns_position['default_code']).value
            if not(start and date and default_code in product_medium):
                _logger.info(
                    '%s. Line not used (no start or no product watched: %s' % (
                        row + 1,
                        default_code,
                    ))
                continue

            # Load data for medium
            qty = ws.cell(row, columns_position['qty']).value
            if type(qty) not in (float, int):
                _logger.error(
                    '%s. Line not used (qty not float: %s' % (
                        row + 1,
                        qty,
                    ))
                continue

            if date > from_obsolete_text:  # Not obsolete:
                product_obsolete[default_code] = False

            product_medium[default_code][0] += qty
            _logger.info('%s. Line used %s - %s' % (
                row + 1,
                default_code,
                qty,
            ))

        # A4. Update product medium
        _logger.warning('Product found: %s' % len(product_medium))

        for default_code in product_medium:
            total, product = product_medium[default_code]
            if product.manual_stock_level:
                continue

            if not total:
                medium_stock_qty = 0.0
                _logger.info(
                    'Update product without level: %s' % product.default_code)
            else:
                medium_stock_qty = total / stock_level_days
                _logger.info(
                    'Update product with level: %s' % product.default_code)

            """
            if default_code[0] in 'RP':
                day_min_level = 30
                day_max_level = 37
            else:
                day_min_level = 60
                day_max_level = 67
                """

            product_pool.write(cr, uid, [product.id], {
                'medium_origin': 'accounting',
                'medium_stock_qty': medium_stock_qty,
                'stock_obsolete':  product_obsolete[default_code],

                # TODO Force different values?
                # 'day_min_level': day_min_level,
                # 'day_max_level': day_max_level,
                'product_imported': True,

                'min_stock_level':
                    product.day_min_level * medium_stock_qty,
                'max_stock_level':
                    product.day_max_level * medium_stock_qty,
                'ready_stock_level':
                    product.day_max_ready_level * medium_stock_qty,
                }, context=context)
        _logger.info('Update marketed product: %s' % len(product_medium))
        # ---------------------------------------------------------------------
        # B. Update original procedure from MRP:
        # ---------------------------------------------------------------------
        return super(MrpProductionWorkcenterLine, self).\
            update_product_level_from_production(cr, uid, ids, context=context)

class ResCompany(osv.osv):
    """ Model name: Parameters
    """

    _inherit = 'res.company'

    _columns = {
        'stock_level_external_excel': fields.char(
            'Stock movement file', size=100,
            help='Path for external XLSX file for stock movement',
        ),

        # Days management for stats:
        'stock_level_mm_days': fields.integer(
            'Stock level MM days', required=True,
            help='Days for from data till today (period for stock movement'),
        'stock_level_obsolete_days': fields.integer(
            'Stock level obsolete days', required=True,
            help='Days for from data till today (> consider product obsolete'),
    }

    _defaults = {
        'stock_level_mm_days': lambda *x: 180,
        'stock_level_obsolete_days': lambda *x: 90,
    }

class ProductProduct(osv.osv):
    """ Model name: Parameters
    """

    _inherit = 'product.product'

    _columns = {
        'product_imported': fields.char('Imported product'),
    }
<commit_msg>generic bugfix (fast message)<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
###############################################################################
#
# ODOO (ex OpenERP)
# Open Source Management Solution
# Copyright (C) 2001-2015 Micronaet S.r.l. (<https://micronaet.com>)
# Developer: Nicola Riolini @thebrush (<https://it.linkedin.com/in/thebrush>)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################

import os
import logging
from openerp.osv import fields, osv, expression
from datetime import datetime, timedelta
from openerp.tools.translate import _
from openerp.tools import (DEFAULT_SERVER_DATE_FORMAT,
    DEFAULT_SERVER_DATETIME_FORMAT,
    DATETIME_FORMATS_MAP,
    float_compare)
try:
    import xlrd  # problem in pan!
except:
    pass
import pdb

_logger = logging.getLogger(__name__)

class MrpProductionWorkcenterLine(osv.osv):
    """ Model name: Job
    """

    _inherit = 'mrp.production.workcenter.line'

    def get_file(self, url, fullname):
        """ Extract from Dropbox
        """
        import requests
        _logger.info('Downloading file %s from Dropbox' % fullname)
        reply = requests.get(url)
        attachment_data = reply.content
        with open(fullname, 'wb') as f:
            f.write(attachment_data)

    # Override so update after update MRP:
    def update_product_level_from_production(self, cr, uid, ids, context=None):
        """ Update product level from production
        """
        def get_excel_date(value):
            """ Extract ISO date
            """
            months = {
                'GEN': '01',
                'FEB': '02',
                'MAR': '03',
                'APR': '04',
                'MAG': '05',
                'GIU': '06',
                'LUG': '07',
                'AGO': '08',
                'SET': '09',
                'OCT': '10',
                'NOV': '11',
                'DEC': '12',

                'DIC': '12',
            }
            if not value or type(value) in (float, ):
                res = ''
            else:
                value = value.strip()
                value_item = value.split('/')
                if len(value_item) == 3:
                    if len(value_item[2]) == 4:
                        pass  # correct
                    elif len(value_item[2]) == 2:
                        value_item[2] = '20%s' % value_item[2]
                    elif value_item == '209':
                        value_item[2] = '2019'
                    else:
                        _logger.error('Unmanaged error:')

                    if value_item[1].upper() in months:
                        value_item[1] = months[value_item[1].upper()]

                    res = '%s-%02d-%02d' % (
                        value_item[2],
                        int(value_item[1]),
                        int(value_item[0]),
                    )
                else:
                    res = value  # Nothing
            # _logger.warning(' ' * 70 + '>>>>>> From %s to %s' % (value, res))
            return res

        _logger.info('Update marketed product medium (from Excel)')
        product_pool = self.pool.get('product.product')
        company_pool = self.pool.get('res.company')

        # ---------------------------------------------------------------------
        # A. Update marketed product:
        # ---------------------------------------------------------------------
        # Fixed parameters:
        columns_position = {
            'date': 1,
            # 'invoice': 7,
            'default_code': 24,
            # 'name': 21,
            'qty': 26,
            # 'uom': 23,
            # 'price': 24,
        }
        start_test = 'INVOICE DATE'
        sheet_name = 'BASE'

        # Dynamic parameters:
        company_ids = company_pool.search(cr, uid, [], context=context)
        company = company_pool.browse(
           cr, uid, company_ids, context=context)[0]
        stock_level_days = company.stock_level_mm_days
        # TODO use it:
        stock_level_obsolete_days = company.stock_level_obsolete_days

        # Generate file:
        dropbox_link = context.get('dropbox_link')
        if not dropbox_link:
            _logger.error(
                'Setup the scheduled command with dropbox_link parameter')
        filename = '/tmp/account_status.xlsx'
        self.get_file(dropbox_link, filename)
        if not filename:
            raise osv.except_osv(
                _('Error stock management'),
                _('Setup the parameter in company form (input file)'),
                )

        now = datetime.now()
        from_date = now - timedelta(days=stock_level_days)
        from_obsolete_date = now - timedelta(days=stock_level_days)
        now_text = '%s 00:00:00' % now.strftime(
             DEFAULT_SERVER_DATE_FORMAT)
        from_text = '%s 00:00:00' % from_date.strftime(
             DEFAULT_SERVER_DATE_FORMAT)
        from_obsolete_text = '%s 00:00:00' % from_obsolete_date.strftime(
             DEFAULT_SERVER_DATE_FORMAT)

        # A1. Search product marketed:
        cr.execute('''
            SELECT id from product_product
            WHERE default_code is not null AND
                  SUBSTRING (default_code, 1, 1) IN (
                      'D', 'E', 'F', 'G', 'H', 'L', 'M',
                      'O', 'P', 'R', 'S', 'X') AND
                  SUBSTRING (default_code, 1, 3) NOT IN ('OLD', 'SER');
        ''')
        product_ids = [record[0] for record in cr.fetchall()]

        _logger.warning('Imported product #%s [%s - %s]' % (
            len(product_ids),
            from_text,
            now_text,
            ))

        # A2. Prepare dict for medium
        product_medium = {}
        product_obsolete = {}
        for product in product_pool.browse(
                cr, uid, product_ids, context=context):

            default_code = product.default_code
            if not default_code:
                _logger.error('Product %s has no code' % product.name)
                continue

            if default_code not in product_obsolete:
                product_obsolete[default_code] = True

            if default_code.endswith('X'):
                continue
            if default_code in product_medium:
                _logger.error('Product double: %s' % default_code)
            else:
                product_medium[default_code] = [0.0, product]

        # A3. Load data from Excel:
        try:
            wb = xlrd.open_workbook(filename)
        except:
            _logger.error(
                '[ERROR] Cannot read XLS file: %s' % filename
            )
            return False

        ws = wb.sheet_by_name(sheet_name)
        _logger.info('Read XLS file: %s' % filename)
        start = False
        for row in range(ws.nrows):
            date = get_excel_date(ws.cell(row, columns_position['date']).value)
            if not start and date == start_test:
                _logger.info('%s. Line not used: Start line' % (row + 1))
                start = True
                continue

            if date < from_text or date > now_text:  # Out of period range:
                _logger.info('%s. Line not used out of range %s' % (
                    row + 1, date))
                continue

            default_code = ws.cell(row, columns_position['default_code']).value
            if not(start and date and default_code in product_medium):
                _logger.info(
                    '%s. Line not used (no start or no product watched: %s' % (
                        row + 1,
                        default_code,
                    ))
                continue

            # Load data for medium
            qty = ws.cell(row, columns_position['qty']).value
            if type(qty) not in (float, int):
                _logger.error(
                    '%s. Line not used (qty not float: %s' % (
                        row + 1,
                        qty,
                    ))
                continue

            if date > from_obsolete_text:  # Not obsolete:
                product_obsolete[default_code] = False

            product_medium[default_code][0] += qty
            _logger.info('%s. Line used %s - %s' % (
                row + 1,
                default_code,
                qty,
            ))

        # A4. Update product medium
        _logger.warning('Product found: %s' % len(product_medium))

        for default_code in product_medium:
            total, product = product_medium[default_code]
            if product.manual_stock_level:
                continue

            if not total:
                medium_stock_qty = 0.0
                _logger.info(
                    'Update product without level: %s' % product.default_code)
            else:
                medium_stock_qty = total / stock_level_days
                _logger.info(
                    'Update product with level: %s' % product.default_code)

            """
            if default_code[0] in 'RP':
                day_min_level = 30
                day_max_level = 37
            else:
                day_min_level = 60
                day_max_level = 67
                """

            product_pool.write(cr, uid, [product.id], {
                'medium_origin': 'accounting',
                'medium_stock_qty': medium_stock_qty,
                'stock_obsolete':  product_obsolete[default_code],

                # TODO Force different values?
                # 'day_min_level': day_min_level,
                # 'day_max_level': day_max_level,
                'product_imported': True,

                'min_stock_level':
                    product.day_min_level * medium_stock_qty,
                'max_stock_level':
                    product.day_max_level * medium_stock_qty,
                'ready_stock_level':
                    product.day_max_ready_level * medium_stock_qty,
                }, context=context)
        _logger.info('Update marketed product: %s' % len(product_medium))
        # ---------------------------------------------------------------------
        # B. Update original procedure from MRP:
        # ---------------------------------------------------------------------
        return super(MrpProductionWorkcenterLine, self).\
            update_product_level_from_production(cr, uid, ids, context=context)

class ResCompany(osv.osv):
    """ Model name: Parameters
    """

    _inherit = 'res.company'

    _columns = {
        'stock_level_external_excel': fields.char(
            'Stock movement file', size=100,
            help='Path for external XLSX file for stock movement',
        ),

        # Days management for stats:
        'stock_level_mm_days': fields.integer(
            'Stock level MM days', required=True,
            help='Days for from data till today (period for stock movement'),
        'stock_level_obsolete_days': fields.integer(
            'Stock level obsolete days', required=True,
            help='Days for from data till today (> consider product obsolete'),
    }

    _defaults = {
        'stock_level_mm_days': lambda *x: 180,
        'stock_level_obsolete_days': lambda *x: 90,
    }

class ProductProduct(osv.osv):
    """ Model name: Parameters
    """

    _inherit = 'product.product'

    _columns = {
        'product_imported': fields.char('Imported product'),
    }
<|endoftext|>


<commit_before>'''
This files defines ETL objects that all have a common "load" function.
Each object is meant for transferring one type of data to another type.
'''
import os

from django.db import connections
from django.conf import settings
from django.template import Context, engines

from corehq.warehouse.utils import django_batch_records
from corehq.sql_db.routers import db_for_read_write

class BaseETLMixin(object):

    @classmethod
    def load(cls, start_datetime, end_datetime):
        raise NotImplementedError

class CustomSQLETLMixin(BaseETLMixin):
    '''
    Mixin for transferring data from a SQL store to another SQL store using
    a custom SQL script.
    '''

    @classmethod
    def load(cls, start_datetime, end_datetime):
        from corehq.warehouse.models.shared import WarehouseTable
        '''
        Bulk loads records for a dim or fact table from
        their corresponding dependencies
        '''

        assert issubclass(cls, WarehouseTable)
        database = db_for_read_write(cls)
        with connections[database].cursor() as cursor:
            cursor.execute(cls._sql_query_template(cls.slug, start_datetime, end_datetime))

    @classmethod
    def _table_context(cls, start_datetime, end_datetime):
        '''
        Get a dict of slugs to table name mapping
        :returns: Dict of slug to table_name
        {
            <slug>: <table_name>,
            ...
        }
        '''
        from corehq.warehouse.models import get_cls_by_slug

        context = {cls.slug: cls._meta.db_table}
        for dep in cls.dependencies():
            dep_cls = get_cls_by_slug(dep)
            context[dep] = dep_cls._meta.db_table
        context['start_datetime'] = start_datetime
        context['end_datetime'] = start_datetime
        return context

    @classmethod
    def _sql_query_template(cls, template_name, start_datetime, end_datetime):
        path = os.path.join(
            settings.BASE_DIR,
            'corehq',
            'warehouse',
            'transforms',
            'sql',
            '{}.sql'.format(template_name),
        )
        if not os.path.exists(path):
            raise NotImplementedError(
                'You must define {} in order to load data'.format(path)
            )

        return _render_template(path, cls._table_context(start_datetime, end_datetime))

class CouchToDjangoETLMixin(BaseETLMixin):
    '''
    Mixin for transferring docs from Couch to a Django model.
    '''

    @classmethod
    def field_mapping(cls):
        # Map source model fields to staging table fields
        # ( <source field>, <staging field> )
        raise NotImplementedError

    @classmethod
    def record_iter(cls, start_datetime, end_datetime):
        raise NotImplementedError

    @classmethod
    def load(cls, start_datetime, end_datetime):
        from corehq.warehouse.models.shared import WarehouseTable

        assert issubclass(cls, WarehouseTable)
        record_iter = cls.record_iter(start_datetime, end_datetime)

        django_batch_records(cls, record_iter, cls.field_mapping())

def _render_template(path, context):
    with open(path) as f:
        template_string = f.read()

    template = engines['django'].from_string(template_string)
    return template.render(Context(context))
<commit_msg>add the ability for additional table context<commit_after>'''
This files defines ETL objects that all have a common "load" function.
Each object is meant for transferring one type of data to another type.
'''
import os

from django.db import connections
from django.conf import settings
from django.template import Context, engines

from corehq.warehouse.utils import django_batch_records
from corehq.sql_db.routers import db_for_read_write

class BaseETLMixin(object):

    @classmethod
    def load(cls, start_datetime, end_datetime):
        raise NotImplementedError

class CustomSQLETLMixin(BaseETLMixin):
    '''
    Mixin for transferring data from a SQL store to another SQL store using
    a custom SQL script.
    '''

    @classmethod
    def additional_sql_context(cls):
        '''
        Override this method to provide additional context
        vars to the SQL script
        '''
        return {}

    @classmethod
    def load(cls, start_datetime, end_datetime):
        from corehq.warehouse.models.shared import WarehouseTable
        '''
        Bulk loads records for a dim or fact table from
        their corresponding dependencies
        '''

        assert issubclass(cls, WarehouseTable)
        database = db_for_read_write(cls)
        with connections[database].cursor() as cursor:
            cursor.execute(cls._sql_query_template(cls.slug, start_datetime, end_datetime))

    @classmethod
    def _table_context(cls, start_datetime, end_datetime):
        '''
        Get a dict of slugs to table name mapping
        :returns: Dict of slug to table_name
        {
            <slug>: <table_name>,
            ...
        }
        '''
        from corehq.warehouse.models import get_cls_by_slug

        context = {cls.slug: cls._meta.db_table}
        for dep in cls.dependencies():
            dep_cls = get_cls_by_slug(dep)
            context[dep] = dep_cls._meta.db_table
        context['start_datetime'] = start_datetime
        context['end_datetime'] = start_datetime
        context.update(cls.additional_sql_context())
        return context

    @classmethod
    def _sql_query_template(cls, template_name, start_datetime, end_datetime):
        path = os.path.join(
            settings.BASE_DIR,
            'corehq',
            'warehouse',
            'transforms',
            'sql',
            '{}.sql'.format(template_name),
        )
        if not os.path.exists(path):
            raise NotImplementedError(
                'You must define {} in order to load data'.format(path)
            )

        return _render_template(path, cls._table_context(start_datetime, end_datetime))

class CouchToDjangoETLMixin(BaseETLMixin):
    '''
    Mixin for transferring docs from Couch to a Django model.
    '''

    @classmethod
    def field_mapping(cls):
        # Map source model fields to staging table fields
        # ( <source field>, <staging field> )
        raise NotImplementedError

    @classmethod
    def record_iter(cls, start_datetime, end_datetime):
        raise NotImplementedError

    @classmethod
    def load(cls, start_datetime, end_datetime):
        from corehq.warehouse.models.shared import WarehouseTable

        assert issubclass(cls, WarehouseTable)
        record_iter = cls.record_iter(start_datetime, end_datetime)

        django_batch_records(cls, record_iter, cls.field_mapping())

def _render_template(path, context):
    with open(path) as f:
        template_string = f.read()

    template = engines['django'].from_string(template_string)
    return template.render(Context(context))
<|endoftext|>