<commit_before>import json
from datadog import api as datadog_api
import requests
from django.core.management import call_command
from corehq.apps.hqadmin.management.utils import get_deploy_email_message_body
from django.core.management.base import BaseCommand
from corehq.apps.hqadmin.models import HqDeploy
from datetime import datetime
from optparse import make_option
from django.conf import settings
from pillow_retry.models import PillowError

STYLE_MARKDOWN = 'markdown'
STYLE_SLACK = 'slack'
DASHBOARD_URL = 'https://p.datadoghq.com/sb/5c4af2ac8-1f739e93ef'
INTEGRATION_TEST_URL = 'https://jenkins.dimagi.com/job/integration-tests/'

def integration_tests_link(style, url):
    return make_link(style, 'tests', url)

def diff_link(style, url):
    return make_link(style, 'here', url)

def dashboard_link(style, url):
    return make_link(style, 'dashboard', url)

def make_link(style, label, url):
    if style == STYLE_MARKDOWN:
        return '[{label}]({url})'.format(label=label, url=url)
    elif style == STYLE_SLACK:
        return '<{url}|{label}>'.format(label=label, url=url)

class Command(BaseCommand):
    help = "Creates an HqDeploy document to record a successful deployment."
    args = "[user]"

    option_list = BaseCommand.option_list + (
        make_option('--user', help='User', default=False),
        make_option('--environment', help='Environment {production|staging etc...}', default=settings.SERVER_ENVIRONMENT),
        make_option('--mail_admins', help='Mail Admins', default=False, action='store_true'),
        make_option('--url', help='A link to a URL for the deploy', default=False),
        make_option(
            '--minutes',
            help='The number of minutes it took to deploy',
            type='int',
            default=None,
        ),
    )

    def handle(self, *args, **options):
        compare_url = options.get('url', None)
        minutes = options.get('minutes', None)

        deploy = HqDeploy(
            date=datetime.utcnow(),
            user=options['user'],
            environment=options['environment'],
            diff_url=compare_url
        )
        deploy.save()

        #  reset PillowTop errors in the hope that a fix has been deployed
        rows_updated = PillowError.bulk_reset_attempts(datetime.utcnow())
        if rows_updated:
            print "\n---------------- Pillow Errors Reset ----------------\n" \
                  "{} pillow errors queued for retry\n".format(rows_updated)

        deploy_notification_text = (
            "CommCareHQ has been successfully deployed to *{}* by *{}* in *{}* minutes. ".format(
                options['environment'],
                options['user'],
                minutes or '?',
            )
        )

        if options['environment'] == 'production':
            deploy_notification_text += "Monitor the {{dashboard_link}}. "

        if settings.MOBILE_INTEGRATION_TEST_TOKEN:
            deploy_notification_text += "Check the integration {{integration_tests_link}}. "
            requests.get(
                'https://jenkins.dimagi.com/job/integration-tests/build',
                params={'token': settings.MOBILE_INTEGRATION_TEST_TOKEN},
            )

        deploy_notification_text += "Find the diff {{diff_link}}"

        if hasattr(settings, 'MIA_THE_DEPLOY_BOT_API'):
            link = diff_link(STYLE_SLACK, compare_url)
            requests.post(settings.MIA_THE_DEPLOY_BOT_API, data=json.dumps({
                "username": "Igor the Iguana",
                "text": deploy_notification_text.format(
                    dashboard_link=dashboard_link(STYLE_SLACK, DASHBOARD_URL),
                    diff_link=link,
                    integration_tests_link=integration_tests_link(STYLE_SLACK, INTEGRATION_TEST_URL)
                ),
            }))

        if settings.DATADOG_API_KEY:
            tags = ['environment:{}'.format(options['environment'])]
            link = diff_link(STYLE_MARKDOWN, compare_url)
            datadog_api.Event.create(
                title="Deploy Success",
                text=deploy_notification_text.format(
                    dashboard_link=dashboard_link(STYLE_MARKDOWN, DASHBOARD_URL),
                    diff_link=link,
                    integration_tests_link=integration_tests_link(STYLE_MARKDOWN, INTEGRATION_TEST_URL)
                ),
                tags=tags,
                alert_type="success"
            )

            print "\n=============================================================\n" \
                  "Congratulations! Deploy Complete.\n\n" \
                  "Don't forget to keep an eye on the deploy dashboard to " \
                  "make sure everything is running smoothly.\n\n" \
                  "https://p.datadoghq.com/sb/5c4af2ac8-1f739e93ef" \
                  "\n=============================================================\n"

        if options['mail_admins']:
            message_body = get_deploy_email_message_body(
                environment=options['environment'], user=options['user'],
                compare_url=compare_url)
            call_command('mail_admins', message_body, **{'subject': 'Deploy successful', 'html': True})
<commit_msg>Don't escape brackets<commit_after>import json
from datadog import api as datadog_api
import requests
from django.core.management import call_command
from corehq.apps.hqadmin.management.utils import get_deploy_email_message_body
from django.core.management.base import BaseCommand
from corehq.apps.hqadmin.models import HqDeploy
from datetime import datetime
from optparse import make_option
from django.conf import settings
from pillow_retry.models import PillowError

STYLE_MARKDOWN = 'markdown'
STYLE_SLACK = 'slack'
DASHBOARD_URL = 'https://p.datadoghq.com/sb/5c4af2ac8-1f739e93ef'
INTEGRATION_TEST_URL = 'https://jenkins.dimagi.com/job/integration-tests/'

def integration_tests_link(style, url):
    return make_link(style, 'tests', url)

def diff_link(style, url):
    return make_link(style, 'here', url)

def dashboard_link(style, url):
    return make_link(style, 'dashboard', url)

def make_link(style, label, url):
    if style == STYLE_MARKDOWN:
        return '[{label}]({url})'.format(label=label, url=url)
    elif style == STYLE_SLACK:
        return '<{url}|{label}>'.format(label=label, url=url)

class Command(BaseCommand):
    help = "Creates an HqDeploy document to record a successful deployment."
    args = "[user]"

    option_list = BaseCommand.option_list + (
        make_option('--user', help='User', default=False),
        make_option('--environment', help='Environment {production|staging etc...}', default=settings.SERVER_ENVIRONMENT),
        make_option('--mail_admins', help='Mail Admins', default=False, action='store_true'),
        make_option('--url', help='A link to a URL for the deploy', default=False),
        make_option(
            '--minutes',
            help='The number of minutes it took to deploy',
            type='int',
            default=None,
        ),
    )

    def handle(self, *args, **options):
        compare_url = options.get('url', None)
        minutes = options.get('minutes', None)

        deploy = HqDeploy(
            date=datetime.utcnow(),
            user=options['user'],
            environment=options['environment'],
            diff_url=compare_url
        )
        deploy.save()

        #  reset PillowTop errors in the hope that a fix has been deployed
        rows_updated = PillowError.bulk_reset_attempts(datetime.utcnow())
        if rows_updated:
            print "\n---------------- Pillow Errors Reset ----------------\n" \
                  "{} pillow errors queued for retry\n".format(rows_updated)

        deploy_notification_text = (
            "CommCareHQ has been successfully deployed to *{}* by *{}* in *{}* minutes. ".format(
                options['environment'],
                options['user'],
                minutes or '?',
            )
        )

        if options['environment'] == 'production':
            deploy_notification_text += "Monitor the {dashboard_link}. "

        if settings.MOBILE_INTEGRATION_TEST_TOKEN:
            deploy_notification_text += "Check the integration {integration_tests_link}. "
            requests.get(
                'https://jenkins.dimagi.com/job/integration-tests/build',
                params={'token': settings.MOBILE_INTEGRATION_TEST_TOKEN},
            )

        deploy_notification_text += "Find the diff {diff_link}"

        if hasattr(settings, 'MIA_THE_DEPLOY_BOT_API'):
            link = diff_link(STYLE_SLACK, compare_url)
            requests.post(settings.MIA_THE_DEPLOY_BOT_API, data=json.dumps({
                "username": "Igor the Iguana",
                "text": deploy_notification_text.format(
                    dashboard_link=dashboard_link(STYLE_SLACK, DASHBOARD_URL),
                    diff_link=link,
                    integration_tests_link=integration_tests_link(STYLE_SLACK, INTEGRATION_TEST_URL)
                ),
            }))

        if settings.DATADOG_API_KEY:
            tags = ['environment:{}'.format(options['environment'])]
            link = diff_link(STYLE_MARKDOWN, compare_url)
            datadog_api.Event.create(
                title="Deploy Success",
                text=deploy_notification_text.format(
                    dashboard_link=dashboard_link(STYLE_MARKDOWN, DASHBOARD_URL),
                    diff_link=link,
                    integration_tests_link=integration_tests_link(STYLE_MARKDOWN, INTEGRATION_TEST_URL)
                ),
                tags=tags,
                alert_type="success"
            )

            print "\n=============================================================\n" \
                  "Congratulations! Deploy Complete.\n\n" \
                  "Don't forget to keep an eye on the deploy dashboard to " \
                  "make sure everything is running smoothly.\n\n" \
                  "https://p.datadoghq.com/sb/5c4af2ac8-1f739e93ef" \
                  "\n=============================================================\n"

        if options['mail_admins']:
            message_body = get_deploy_email_message_body(
                environment=options['environment'], user=options['user'],
                compare_url=compare_url)
            call_command('mail_admins', message_body, **{'subject': 'Deploy successful', 'html': True})
<|endoftext|>


<commit_before>from indy.error import IndyError
<commit_msg>IS-1115 - Adjust python wrapper namespacing<commit_after>from indy.error import IndyError
from indy import anoncreds
from indy import blob_storage
from indy import crypto
from indy import did
from indy import ledger
from indy import libindy
from indy import non_secrets
from indy import pairwise
from indy import payment
from indy import pool
from indy import wallet

__all__ = [
    'anoncreds',
    'blob_storage',
    'crypto',
    'did',
    'ledger',
    'libindy',
    'non_secrets',
    'pairwise',
    'payment',
    'pool',
    'wallet'
]
<|endoftext|>


<commit_before>def array(first_param):
    #? ['first_param']
    first_param
    return list()

#? []
array.first_param
#? []
array.first_param.
func = array
#? []
func.first_param

#? list()
array()

#? ['array']
arr

def inputs(param):
    return param

#? list()
inputs(list)

def variable_middle():
    var = 3
    return var

#? int()
variable_middle()

def variable_rename(param):
    var = param
    return var

#? int()
variable_rename(1)

# -----------------
# double execution
# -----------------
def double_exe(param):
    return param

#? str()
variable_rename(double_exe)("")

# -> shouldn't work (and throw no error)
#? []
variable_rename(list())().
#? []
variable_rename(1)().

# -----------------
# recursions (should ignore)
# -----------------
def recursion(a, b):
    if a:
        return b
    else:
        return recursion(a+".", b+1)

#? int() float()
recursion("a", 1.0)

def other(a):
    return recursion2(a)

def recursion2(a):
    if a:
        return other(a)
    else:
        return recursion2("")
    return a

#? int() str() str()
recursion2(1)

# -----------------
# ordering
# -----------------

#def b():
    #return ""

def a():
    #? int()
    b()
    return b()

def b():
    return 1

#? int()
a()

# -----------------
# keyword arguments
# -----------------

def func(a=1, b=''):
    return a, b

exe = func(b=list, a=tuple)
#? tuple()
exe[0]

#? list()
exe[1]

# -----------------
# default arguments
# -----------------

#? int()
func()[0]
#? str()
func()[1]
#? float()
func(1.0)[0]
#? str()
func(1.0)[1]

# -----------------
# closures
# -----------------
def a():
    l = 3
    def func_b():
        #? str()
        l = ''
    #? ['func_b']
    func_b
    #? int()
    l

# -----------------
# *args
# -----------------

def args_func(*args):
    return args

exe = args_func(1, "")
#? int()
exe[0]

#? str()
exe[1]

_list = [1,""]
exe2 = args_func(_list)[0]

#? str()
exe2[1]

exe3 = args_func([1,""])[0]

#? str()
exe3[1]

def args_func(arg1, *args):
    return arg1, args

exe = args_func(1, "", list)
#? int()
exe[0]
#? tuple()
exe[1]
#? list()
exe[1][1]

# -----------------
# ** kwargs
# -----------------
def kwargs_func(**kwargs):
    return kwargs

exe = kwargs_func(a=3,b=4)
#? dict()
exe

# -----------------
# *args / ** kwargs
# -----------------

def fu(a=1, b="", *args, **kwargs):
    return a, b, args, kwargs

exe = fu(list, 1, "", c=set, d="")

#? list()
exe[0]
#? int()
exe[1]
#? tuple()
exe[2]
#? str()
exe[2][0]
#? dict()
exe[3]
#? set()
exe[3]['c']

<commit_msg>new args/kwargs tests<commit_after>def array(first_param):
    #? ['first_param']
    first_param
    return list()

#? []
array.first_param
#? []
array.first_param.
func = array
#? []
func.first_param

#? list()
array()

#? ['array']
arr

def inputs(param):
    return param

#? list()
inputs(list)

def variable_middle():
    var = 3
    return var

#? int()
variable_middle()

def variable_rename(param):
    var = param
    return var

#? int()
variable_rename(1)

# -----------------
# double execution
# -----------------
def double_exe(param):
    return param

#? str()
variable_rename(double_exe)("")

# -> shouldn't work (and throw no error)
#? []
variable_rename(list())().
#? []
variable_rename(1)().

# -----------------
# recursions (should ignore)
# -----------------
def recursion(a, b):
    if a:
        return b
    else:
        return recursion(a+".", b+1)

#? int() float()
recursion("a", 1.0)

def other(a):
    return recursion2(a)

def recursion2(a):
    if a:
        return other(a)
    else:
        return recursion2("")
    return a

#? int() str() str()
recursion2(1)

# -----------------
# ordering
# -----------------

#def b():
    #return ""

def a():
    #? int()
    b()
    return b()

def b():
    return 1

#? int()
a()

# -----------------
# keyword arguments
# -----------------

def func(a=1, b=''):
    return a, b

exe = func(b=list, a=tuple)
#? tuple()
exe[0]

#? list()
exe[1]

# -----------------
# default arguments
# -----------------

#? int()
func()[0]
#? str()
func()[1]
#? float()
func(1.0)[0]
#? str()
func(1.0)[1]

# -----------------
# closures
# -----------------
def a():
    l = 3
    def func_b():
        #? str()
        l = ''
    #? ['func_b']
    func_b
    #? int()
    l

# -----------------
# *args
# -----------------

def args_func(*args):
    return args

exe = args_func(1, "")
#? int()
exe[0]

#? str()
exe[1]

_list = [1,""]
exe2 = args_func(_list)[0]

#? str()
exe2[1]

exe3 = args_func([1,""])[0]

#? str()
exe3[1]

def args_func(arg1, *args):
    return arg1, args

exe = args_func(1, "", list)
#? int()
exe[0]
#? tuple()
exe[1]
#? list()
exe[1][1]

# -----------------
# ** kwargs
# -----------------
def kwargs_func(**kwargs):
    return kwargs

exe = kwargs_func(a=3,b=4)
#? dict()
exe

# -----------------
# *args / ** kwargs
# -----------------

def fu(a=1, b="", *args, **kwargs):
    return a, b, args, kwargs

exe = fu(list, 1, "", c=set, d="")

#? list()
exe[0]
#? int()
exe[1]
#? tuple()
exe[2]
#? str()
exe[2][0]
#? dict()
exe[3]
#? set()
exe[3]['c']

# -----------------
# nested *args
# -----------------
def function_args(a, b, c):
    return b

def nested_args(*args):
    return function_args(*args)

def nested_args2(*args, **kwargs):
    return nested_args(*args)

#? int()
nested_args('', 1, 1.0, list)
#? []
nested_args('')

#? int()
nested_args2('', 1, 1.0)
#? []
nested_args2('')

# -----------------
# nested **kwargs
# -----------------
def nested_kw(**kwargs1):
    return function_args(**kwargs1)

def nested_kw2(**kwargs2):
    return nested_kw(**kwargs2)

#? int()
nested_kw(b=1, c=1.0, list)
#? int()
nested_kw('', b=1, c=1.0, list)
#? int()
nested_kw('', d=1.0, b=1, list)
#? int()
nested_kw(b=1)
#? int()
nested_kw(a=3.0, b=1)
#? int()
nested_kw(b=1, a=r"")
#? []
nested_kw('')
#? []
nested_kw(a='')

#? int()
nested_kw2(b=1)
#? int()
nested_kw2('', b=1, c=1.0)
#? int()
nested_kw2('', c=1.0, b=1)
#? []
nested_kw2('')
#? []
nested_kw2(a='')

# -----------------
# nested *args/**kwargs
# -----------------
def nested_both(*args, **kwargs):
    return function_args(*args, **kwargs)

def nested_both2(*args, **kwargs):
    return nested_both(*args, **kwargs)

#? int()
nested_both('', b=1, c=1.0, list)
#? int()
nested_both('', c=1.0, b=1, list)
#? []
nested_both('')

#? int()
nested_both2('', b=1, c=1.0)
#? int()
nested_both2('', c=1.0, b=1)
#? []
nested_both2('')
<|endoftext|>


<commit_before>from lxml import etree
import calendar
import time
import email.utils

import codecs
import sys
utf8_stdout = codecs.getwriter('utf8')(sys.stdout)

### FIXME: content encoding utf8

def _get_tag(data, tag_name):
    out = data.xpath('//' + tag_name)
    if out:
        return out[0].text
    return None

def split_headers_from_body():
    return '\r\n'

def get_date(data):
    w3cdate = _get_tag(data, 'w3cdate')
    assert (w3cdate is not None)

    # Silly hack.
    msg_time = time.strptime(w3cdate,
        "%Y-%m-%dT%H:%M:%SZ")
    out = email.utils.formatdate(calendar.timegm(msg_time))
    return ("Date: " + out + "\r\n")

def get_received(data):
    ipaddress = _get_tag(data, 'ipaddress')
    assert (ipaddress is not None)

    return "Received: from %s" % (ipaddress,) + "\r\n"

def get_from(data):
    from_data = []

    author = _get_tag(data, 'author')
    assert (author is not None)
    from_data.append(author)

    email_addr = _get_tag(data, 'email')
    if email_addr:
        from_data.append(email_addr)

    return "From: " + email.utils.formataddr(from_data) + "\r\n"

def get_subject(data):
    value = _get_tag(data, 'title')
    assert (value is not None)

    return "Subject: " + value + "\r\n"

def get_body(data):
    body = _get_tag(data, 'description')
    out = ''
    for line in body.split('\n'):
        if not line.endswith('\r'):
            line += '\r'
        out += line + '\n'
    return out + '\r\n'

def get_homepage(data):
    value = _get_tag(data, 'link')
    if value:
        return "Homepage: %s\r\n" % (
            value,)
    return ''

def main(filename):
    data = etree.fromstring(open(filename).read())

    out = ''
    out += get_date(data)
    out += get_received(data)
    out += get_from(data)
    out += get_subject(data)
    out += split_headers_from_body()
    out += get_body(data)
    out += get_homepage(data)
    return out

if __name__ == '__main__':
    utf8_stdout.write(main(sys.argv[1]))
<commit_msg>In the case of no email address, do not crash<commit_after>from lxml import etree
import calendar
import time
import email.utils

import codecs
import sys
utf8_stdout = codecs.getwriter('utf8')(sys.stdout)

### FIXME: content encoding utf8

def _get_tag(data, tag_name):
    out = data.xpath('//' + tag_name)
    if out:
        return out[0].text
    return None

def split_headers_from_body():
    return '\r\n'

def get_date(data):
    w3cdate = _get_tag(data, 'w3cdate')
    assert (w3cdate is not None)

    # Silly hack.
    msg_time = time.strptime(w3cdate,
        "%Y-%m-%dT%H:%M:%SZ")
    out = email.utils.formatdate(calendar.timegm(msg_time))
    return ("Date: " + out + "\r\n")

def get_received(data):
    ipaddress = _get_tag(data, 'ipaddress')
    assert (ipaddress is not None)

    return "Received: from %s" % (ipaddress,) + "\r\n"

def get_from(data):
    from_data = []

    author = _get_tag(data, 'author')
    assert (author is not None)
    from_data.append(author)

    email_addr = _get_tag(data, 'email')
    if email_addr:
        from_data.append(email_addr)

    if len(from_data) > 1:
        as_string = email.utils.formataddr(from_data)
    else:
        as_string = from_data[0]

    return "From: " + as_string + '\r\n'

def get_subject(data):
    value = _get_tag(data, 'title')
    assert (value is not None)

    return "Subject: " + value + "\r\n"

def get_body(data):
    body = _get_tag(data, 'description')
    out = ''
    for line in body.split('\n'):
        if not line.endswith('\r'):
            line += '\r'
        out += line + '\n'
    return out + '\r\n'

def get_homepage(data):
    value = _get_tag(data, 'link')
    if value:
        return "Homepage: %s\r\n" % (
            value,)
    return ''

def main(filename):
    data = etree.fromstring(open(filename).read())

    out = ''
    out += get_date(data)
    out += get_received(data)
    out += get_from(data)
    out += get_subject(data)
    out += split_headers_from_body()
    out += get_body(data)
    out += get_homepage(data)
    return out

if __name__ == '__main__':
    utf8_stdout.write(main(sys.argv[1]))
<|endoftext|>


<commit_before># Copyright 2017, Google LLC All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Base class for concurrency policy."""

from __future__ import absolute_import, division

import abc
import collections
import logging
import random
import time

from google.api_core import exceptions
import six

from google.cloud.pubsub_v1 import types
from google.cloud.pubsub_v1.subscriber import _consumer
from google.cloud.pubsub_v1.subscriber import _histogram

_LOGGER = logging.getLogger(__name__)

# Namedtuples for management requests. Used by the Message class to communicate
# items of work back to the policy.
AckRequest = collections.namedtuple(
    'AckRequest',
    ['ack_id', 'byte_size', 'time_to_ack'],
)

DropRequest = collections.namedtuple(
    'DropRequest',
    ['ack_id', 'byte_size'],
)

LeaseRequest = collections.namedtuple(
    'LeaseRequest',
    ['ack_id', 'byte_size'],
)

ModAckRequest = collections.namedtuple(
    'ModAckRequest',
    ['ack_id', 'seconds'],
)

NackRequest = collections.namedtuple(
    'NackRequest',
    ['ack_id', 'byte_size'],
)

@six.add_metaclass(abc.ABCMeta)
class BasePolicy(object):
    """Abstract class defining a subscription policy.

    Although the :class:`~.pubsub_v1.subscriber.policy.thread.Policy` class,
    based on :class:`threading.Thread`, is fine for most cases,
    advanced users may need to implement something based on a different
    concurrency model.

    This class defines the interface for the policy implementation;
    subclasses may be passed as the ``policy_class`` argument to
    :class:`~.pubsub_v1.client.SubscriberClient`.

    Args:
        client (google.cloud.pubsub_v1.subscriber.client.Client): The
            subscriber client used to create this instance.
        subscription (str): The name of the subscription. The canonical
            format for this is
            ``projects/{project}/subscriptions/{subscription}``.
        flow_control (google.cloud.pubsub_v1.types.FlowControl): The flow
            control settings.
        histogram_data (dict): Optional: A structure to store the histogram
            data for predicting appropriate ack times. If set, this should
            be a dictionary-like object.

            .. note::
                Additionally, the histogram relies on the assumption
                that the dictionary will properly sort keys provided
                that all keys are positive integers. If you are sending
                your own dictionary class, ensure this assumption holds
                or you will get strange behavior.
    """

    _managed_ack_ids = None
    _RETRYABLE_STREAM_ERRORS = (
        exceptions.DeadlineExceeded,
        exceptions.ServiceUnavailable,
    )

    def __init__(self, client, subscription,
                 flow_control=types.FlowControl(), histogram_data=None):
        self._client = client
        self._subscription = subscription
        self._consumer = _consumer.Consumer()
        self._ack_deadline = 10
        self._last_histogram_size = 0
        self._future = None
        self.flow_control = flow_control
        self.histogram = _histogram.Histogram(data=histogram_data)

        # These are for internal flow control tracking.
        # They should not need to be used by subclasses.
        self._bytes = 0
        self._ack_on_resume = set()

    @property
    def ack_deadline(self):
        """Return the appropriate ack deadline.

        This method is "sticky". It will only perform the computations to
        check on the right ack deadline if the histogram has gained a
        significant amount of new information.

        Returns:
            int: The correct ack deadline.
        """
        target = min([
            self._last_histogram_size * 2,
            self._last_histogram_size + 100,
        ])
        if len(self.histogram) > target:
            self._ack_deadline = self.histogram.percentile(percent=99)
        return self._ack_deadline

    @property
    def future(self):
        """Return the Future in use, if any.

        Returns:
            google.cloud.pubsub_v1.subscriber.futures.Future: A Future
            conforming to the :class:`~concurrent.futures.Future` interface.
        """
        return self._future

    @property
    def managed_ack_ids(self):
        """Return the ack IDs currently being managed by the policy.

        Returns:
            set: The set of ack IDs being managed.
        """
        if self._managed_ack_ids is None:
            self._managed_ack_ids = set()
        return self._managed_ack_ids

    @property
    def subscription(self):
        """Return the subscription.

        Returns:
            str: The subscription
        """
        return self._subscription

    @property
    def _load(self):
        """Return the current load.

        The load is represented as a float, where 1.0 represents having
        hit one of the flow control limits, and values between 0.0 and 1.0
        represent how close we are to them. (0.5 means we have exactly half
        of what the flow control setting allows, for example.)

        There are (currently) two flow control settings; this property
        computes how close the subscriber is to each of them, and returns
        whichever value is higher. (It does not matter that we have lots of
        running room on setting A if setting B is over.)

        Returns:
            float: The load value.
        """
        return max([
            len(self.managed_ack_ids) / self.flow_control.max_messages,
            self._bytes / self.flow_control.max_bytes,
            self._consumer.pending_requests / self.flow_control.max_requests
        ])

    def _maybe_resume_consumer(self):
        """Check the current load and resume the consumer if needed."""
        # If we have been paused by flow control, check and see if we are
        # back within our limits.
        #
        # In order to not thrash too much, require us to have passed below
        # the resume threshold (80% by default) of each flow control setting
        # before restarting.
        if not self._consumer.paused:
            return

        if self._load < self.flow_control.resume_threshold:
            self._consumer.resume()
        else:
            _LOGGER.debug('Did not resume, current load is %s', self._load)

    def ack(self, items):
        """Acknowledge the given messages.

        Args:
            items(Sequence[AckRequest]): The items to acknowledge.
        """
        # If we got timing information, add it to the histogram.
        for item in items:
            time_to_ack = item.time_to_ack
            if time_to_ack is not None:
                self.histogram.add(int(time_to_ack))

        ack_ids = [item.ack_id for item in items]
        if self._consumer.active:
            # Send the request to ack the message.
            request = types.StreamingPullRequest(ack_ids=ack_ids)
            self._consumer.send_request(request)
        else:
            # If the consumer is inactive, then queue the ack_ids here; it
            # will be acked as part of the initial request when the consumer
            # is started again.
            self._ack_on_resume.update(ack_ids)

        # Remove the message from lease management.
        self.drop(items)

    def call_rpc(self, request_generator):
        """Invoke the Pub/Sub streaming pull RPC.

        Args:
            request_generator (Generator): A generator that yields requests,
                and blocks if there are no outstanding requests (until such
                time as there are).

        Returns:
            Iterable[~google.cloud.pubsub_v1.types.StreamingPullResponse]: An
            iterable of pull responses.
        """
        return self._client.api.streaming_pull(request_generator)

    def drop(self, items):
        """Remove the given messages from lease management.

        Args:
            items(Sequence[DropRequest]): The items to drop.
        """
        # Remove the ack ID from lease management, and decrement the
        # byte counter.
        for item in items:
            if item.ack_id in self.managed_ack_ids:
                self.managed_ack_ids.remove(item.ack_id)
                self._bytes -= item.byte_size
            else:
                _LOGGER.debug('Item %s wasn\'t managed', item.ack_id)

        if self._bytes < 0:
            _LOGGER.debug(
                'Bytes was unexpectedly negative: %d', self._bytes)
            self._bytes = 0

        self._maybe_resume_consumer()

    def get_initial_request(self, ack_queue=False):
        """Return the initial request.

        This defines the initial request that must always be sent to Pub/Sub
        immediately upon opening the subscription.

        Args:
            ack_queue (bool): Whether to include any acks that were sent
                while the connection was paused.

        Returns:
            google.cloud.pubsub_v1.types.StreamingPullRequest: A request
            suitable for being the first request on the stream (and not
            suitable for any other purpose).

        .. note::
            If ``ack_queue`` is set to True, this includes the ack_ids, but
            also clears the internal set.

            This means that calls to :meth:`get_initial_request` with
            ``ack_queue`` set to True are not idempotent.
        """
        # Any ack IDs that are under lease management and not being acked
        # need to have their deadline extended immediately.
        ack_ids = set()
        lease_ids = self.managed_ack_ids
        if ack_queue:
            ack_ids = self._ack_on_resume
            lease_ids = lease_ids.difference(ack_ids)

        # Put the request together.
        request = types.StreamingPullRequest(
            ack_ids=list(ack_ids),
            modify_deadline_ack_ids=list(lease_ids),
            modify_deadline_seconds=[self.ack_deadline] * len(lease_ids),
            stream_ack_deadline_seconds=self.histogram.percentile(99),
            subscription=self.subscription,
        )

        # Clear the ack_ids set.
        # Note: If `ack_queue` is False, this just ends up being a no-op,
        # since the set is just an empty set.
        ack_ids.clear()

        # Return the initial request.
        return request

    def lease(self, items):
        """Add the given messages to lease management.

        Args:
            items(Sequence[LeaseRequest]): The items to lease.
        """
        for item in items:
            # Add the ack ID to the set of managed ack IDs, and increment
            # the size counter.
            if item.ack_id not in self.managed_ack_ids:
                self.managed_ack_ids.add(item.ack_id)
                self._bytes += item.byte_size
            else:
                _LOGGER.debug(
                    'Message %s is already lease managed', item.ack_id)

        # Sanity check: Do we have too many things in our inventory?
        # If we do, we need to stop the stream.
        if self._load >= 1.0:
            self._consumer.pause()

    def maintain_leases(self):
        """Maintain all of the leases being managed by the policy.

        This method modifies the ack deadline for all of the managed
        ack IDs, then waits for most of that time (but with jitter), and
        then calls itself.

        .. warning::
            This method blocks, and generally should be run in a separate
            thread or process.

            Additionally, you should not have to call this method yourself,
            unless you are implementing your own policy. If you are
            implementing your own policy, you _should_ call this method
            in an appropriate form of subprocess.
        """
        while True:
            # Sanity check: Should this infinite loop quit?
            if not self._consumer.active:
                _LOGGER.debug('Consumer inactive, ending lease maintenance.')
                return

            # Determine the appropriate duration for the lease. This is
            # based off of how long previous messages have taken to ack, with
            # a sensible default and within the ranges allowed by Pub/Sub.
            p99 = self.histogram.percentile(99)
            _LOGGER.debug('The current p99 value is %d seconds.', p99)

            # Create a streaming pull request.
            # We do not actually call `modify_ack_deadline` over and over
            # because it is more efficient to make a single request.
            ack_ids = list(self.managed_ack_ids)
            _LOGGER.debug('Renewing lease for %d ack IDs.', len(ack_ids))
            if ack_ids:
                request = types.StreamingPullRequest(
                    modify_deadline_ack_ids=ack_ids,
                    modify_deadline_seconds=[p99] * len(ack_ids),
                )
                # NOTE: This may not work as expected if ``consumer.active``
                #       has changed since we checked it. An implementation
                #       without any sort of race condition would require a
                #       way for ``send_request`` to fail when the consumer
                #       is inactive.
                self._consumer.send_request(request)

            # Now wait an appropriate period of time and do this again.
            #
            # We determine the appropriate period of time based on a random
            # period between 0 seconds and 90% of the lease. This use of
            # jitter (http://bit.ly/2s2ekL7) helps decrease contention in cases
            # where there are many clients.
            snooze = random.uniform(0.0, p99 * 0.9)
            _LOGGER.debug('Snoozing lease management for %f seconds.', snooze)
            time.sleep(snooze)

    def modify_ack_deadline(self, items):
        """Modify the ack deadline for the given messages.

        Args:
            items(Sequence[ModAckRequest]): The items to modify.
        """
        ack_ids = [item.ack_id for item in items]
        seconds = [item.seconds for item in items]

        request = types.StreamingPullRequest(
            modify_deadline_ack_ids=ack_ids,
            modify_deadline_seconds=seconds,
        )
        self._consumer.send_request(request)

    def nack(self, items):
        """Explicitly deny receipt of messages.

        Args:
            items(Sequence[NackRequest]): The items to deny.
        """
        self.modify_ack_deadline([
            ModAckRequest(ack_id=item.ack_id, seconds=0)
            for item in items])
        self.drop(
            [DropRequest(*item) for item in items])

    @abc.abstractmethod
    def close(self):
        """Close the existing connection.

        Raises:
            NotImplementedError: Always
        """
        raise NotImplementedError

    @abc.abstractmethod
    def on_exception(self, exception):
        """Called when a gRPC exception occurs.

        If this method does nothing, then the stream is re-started. If this
        raises an exception, it will stop the consumer thread. This is
        executed on the response consumer helper thread.

        Implementations should return :data:`True` if they want the consumer
        thread to remain active, otherwise they should return :data:`False`.

        Args:
            exception (Exception): The exception raised by the RPC.

        Raises:
            NotImplementedError: Always
        """
        raise NotImplementedError

    def on_request(self, request):
        """Called whenever a request has been sent to gRPC.

        This allows the policy to measure the rate of requests sent along the
        stream and apply backpressure by pausing or resuming the consumer
        if needed.

        Args:
            request (Any): The protobuf request that was sent to gRPC.
        """
        self._maybe_resume_consumer()

    @abc.abstractmethod
    def on_response(self, response):
        """Process a response from gRPC.

        This gives the consumer control over how responses are scheduled to
        be processed. This method is expected to not block and instead
        schedule the response to be consumed by some sort of concurrency.

        For example, if a the Policy implementation takes a callback in its
        constructor, you can schedule the callback using a
        :cls:`concurrent.futures.ThreadPoolExecutor`::

            self._pool.submit(self._callback, response)

        This is called from the response consumer helper thread.

        Args:
            response (Any): The protobuf response from the RPC.

        Raises:
            NotImplementedError: Always
        """
        raise NotImplementedError

    @abc.abstractmethod
    def open(self, callback):
        """Open a streaming pull connection and begin receiving messages.

        For each message received, the ``callback`` function is fired with
        a :class:`~.pubsub_v1.subscriber.message.Message` as its only
        argument.

        This method is virtual, but concrete implementations should return
        a :class:`~google.api_core.future.Future` that provides an interface
        to block on the subscription if desired, and handle errors.

        Args:
            callback (Callable[Message]): A callable that receives a
                Pub/Sub Message.

        Raises:
            NotImplementedError: Always
        """
        raise NotImplementedError
<commit_msg>Retry subscription stream on InternalServerError, Unknown, and GatewayTimeout (#5021)<commit_after># Copyright 2017, Google LLC All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Base class for concurrency policy."""

from __future__ import absolute_import, division

import abc
import collections
import logging
import random
import time

from google.api_core import exceptions
import six

from google.cloud.pubsub_v1 import types
from google.cloud.pubsub_v1.subscriber import _consumer
from google.cloud.pubsub_v1.subscriber import _histogram

_LOGGER = logging.getLogger(__name__)

# Namedtuples for management requests. Used by the Message class to communicate
# items of work back to the policy.
AckRequest = collections.namedtuple(
    'AckRequest',
    ['ack_id', 'byte_size', 'time_to_ack'],
)

DropRequest = collections.namedtuple(
    'DropRequest',
    ['ack_id', 'byte_size'],
)

LeaseRequest = collections.namedtuple(
    'LeaseRequest',
    ['ack_id', 'byte_size'],
)

ModAckRequest = collections.namedtuple(
    'ModAckRequest',
    ['ack_id', 'seconds'],
)

NackRequest = collections.namedtuple(
    'NackRequest',
    ['ack_id', 'byte_size'],
)

@six.add_metaclass(abc.ABCMeta)
class BasePolicy(object):
    """Abstract class defining a subscription policy.

    Although the :class:`~.pubsub_v1.subscriber.policy.thread.Policy` class,
    based on :class:`threading.Thread`, is fine for most cases,
    advanced users may need to implement something based on a different
    concurrency model.

    This class defines the interface for the policy implementation;
    subclasses may be passed as the ``policy_class`` argument to
    :class:`~.pubsub_v1.client.SubscriberClient`.

    Args:
        client (google.cloud.pubsub_v1.subscriber.client.Client): The
            subscriber client used to create this instance.
        subscription (str): The name of the subscription. The canonical
            format for this is
            ``projects/{project}/subscriptions/{subscription}``.
        flow_control (google.cloud.pubsub_v1.types.FlowControl): The flow
            control settings.
        histogram_data (dict): Optional: A structure to store the histogram
            data for predicting appropriate ack times. If set, this should
            be a dictionary-like object.

            .. note::
                Additionally, the histogram relies on the assumption
                that the dictionary will properly sort keys provided
                that all keys are positive integers. If you are sending
                your own dictionary class, ensure this assumption holds
                or you will get strange behavior.
    """

    _managed_ack_ids = None
    _RETRYABLE_STREAM_ERRORS = (
        exceptions.DeadlineExceeded,
        exceptions.ServiceUnavailable,
        exceptions.InternalServerError,
        exceptions.Unknown,
        exceptions.GatewayTimeout,
    )

    def __init__(self, client, subscription,
                 flow_control=types.FlowControl(), histogram_data=None):
        self._client = client
        self._subscription = subscription
        self._consumer = _consumer.Consumer()
        self._ack_deadline = 10
        self._last_histogram_size = 0
        self._future = None
        self.flow_control = flow_control
        self.histogram = _histogram.Histogram(data=histogram_data)

        # These are for internal flow control tracking.
        # They should not need to be used by subclasses.
        self._bytes = 0
        self._ack_on_resume = set()

    @property
    def ack_deadline(self):
        """Return the appropriate ack deadline.

        This method is "sticky". It will only perform the computations to
        check on the right ack deadline if the histogram has gained a
        significant amount of new information.

        Returns:
            int: The correct ack deadline.
        """
        target = min([
            self._last_histogram_size * 2,
            self._last_histogram_size + 100,
        ])
        if len(self.histogram) > target:
            self._ack_deadline = self.histogram.percentile(percent=99)
        return self._ack_deadline

    @property
    def future(self):
        """Return the Future in use, if any.

        Returns:
            google.cloud.pubsub_v1.subscriber.futures.Future: A Future
            conforming to the :class:`~concurrent.futures.Future` interface.
        """
        return self._future

    @property
    def managed_ack_ids(self):
        """Return the ack IDs currently being managed by the policy.

        Returns:
            set: The set of ack IDs being managed.
        """
        if self._managed_ack_ids is None:
            self._managed_ack_ids = set()
        return self._managed_ack_ids

    @property
    def subscription(self):
        """Return the subscription.

        Returns:
            str: The subscription
        """
        return self._subscription

    @property
    def _load(self):
        """Return the current load.

        The load is represented as a float, where 1.0 represents having
        hit one of the flow control limits, and values between 0.0 and 1.0
        represent how close we are to them. (0.5 means we have exactly half
        of what the flow control setting allows, for example.)

        There are (currently) two flow control settings; this property
        computes how close the subscriber is to each of them, and returns
        whichever value is higher. (It does not matter that we have lots of
        running room on setting A if setting B is over.)

        Returns:
            float: The load value.
        """
        return max([
            len(self.managed_ack_ids) / self.flow_control.max_messages,
            self._bytes / self.flow_control.max_bytes,
            self._consumer.pending_requests / self.flow_control.max_requests
        ])

    def _maybe_resume_consumer(self):
        """Check the current load and resume the consumer if needed."""
        # If we have been paused by flow control, check and see if we are
        # back within our limits.
        #
        # In order to not thrash too much, require us to have passed below
        # the resume threshold (80% by default) of each flow control setting
        # before restarting.
        if not self._consumer.paused:
            return

        if self._load < self.flow_control.resume_threshold:
            self._consumer.resume()
        else:
            _LOGGER.debug('Did not resume, current load is %s', self._load)

    def ack(self, items):
        """Acknowledge the given messages.

        Args:
            items(Sequence[AckRequest]): The items to acknowledge.
        """
        # If we got timing information, add it to the histogram.
        for item in items:
            time_to_ack = item.time_to_ack
            if time_to_ack is not None:
                self.histogram.add(int(time_to_ack))

        ack_ids = [item.ack_id for item in items]
        if self._consumer.active:
            # Send the request to ack the message.
            request = types.StreamingPullRequest(ack_ids=ack_ids)
            self._consumer.send_request(request)
        else:
            # If the consumer is inactive, then queue the ack_ids here; it
            # will be acked as part of the initial request when the consumer
            # is started again.
            self._ack_on_resume.update(ack_ids)

        # Remove the message from lease management.
        self.drop(items)

    def call_rpc(self, request_generator):
        """Invoke the Pub/Sub streaming pull RPC.

        Args:
            request_generator (Generator): A generator that yields requests,
                and blocks if there are no outstanding requests (until such
                time as there are).

        Returns:
            Iterable[~google.cloud.pubsub_v1.types.StreamingPullResponse]: An
            iterable of pull responses.
        """
        return self._client.api.streaming_pull(request_generator)

    def drop(self, items):
        """Remove the given messages from lease management.

        Args:
            items(Sequence[DropRequest]): The items to drop.
        """
        # Remove the ack ID from lease management, and decrement the
        # byte counter.
        for item in items:
            if item.ack_id in self.managed_ack_ids:
                self.managed_ack_ids.remove(item.ack_id)
                self._bytes -= item.byte_size
            else:
                _LOGGER.debug('Item %s wasn\'t managed', item.ack_id)

        if self._bytes < 0:
            _LOGGER.debug(
                'Bytes was unexpectedly negative: %d', self._bytes)
            self._bytes = 0

        self._maybe_resume_consumer()

    def get_initial_request(self, ack_queue=False):
        """Return the initial request.

        This defines the initial request that must always be sent to Pub/Sub
        immediately upon opening the subscription.

        Args:
            ack_queue (bool): Whether to include any acks that were sent
                while the connection was paused.

        Returns:
            google.cloud.pubsub_v1.types.StreamingPullRequest: A request
            suitable for being the first request on the stream (and not
            suitable for any other purpose).

        .. note::
            If ``ack_queue`` is set to True, this includes the ack_ids, but
            also clears the internal set.

            This means that calls to :meth:`get_initial_request` with
            ``ack_queue`` set to True are not idempotent.
        """
        # Any ack IDs that are under lease management and not being acked
        # need to have their deadline extended immediately.
        ack_ids = set()
        lease_ids = self.managed_ack_ids
        if ack_queue:
            ack_ids = self._ack_on_resume
            lease_ids = lease_ids.difference(ack_ids)

        # Put the request together.
        request = types.StreamingPullRequest(
            ack_ids=list(ack_ids),
            modify_deadline_ack_ids=list(lease_ids),
            modify_deadline_seconds=[self.ack_deadline] * len(lease_ids),
            stream_ack_deadline_seconds=self.histogram.percentile(99),
            subscription=self.subscription,
        )

        # Clear the ack_ids set.
        # Note: If `ack_queue` is False, this just ends up being a no-op,
        # since the set is just an empty set.
        ack_ids.clear()

        # Return the initial request.
        return request

    def lease(self, items):
        """Add the given messages to lease management.

        Args:
            items(Sequence[LeaseRequest]): The items to lease.
        """
        for item in items:
            # Add the ack ID to the set of managed ack IDs, and increment
            # the size counter.
            if item.ack_id not in self.managed_ack_ids:
                self.managed_ack_ids.add(item.ack_id)
                self._bytes += item.byte_size
            else:
                _LOGGER.debug(
                    'Message %s is already lease managed', item.ack_id)

        # Sanity check: Do we have too many things in our inventory?
        # If we do, we need to stop the stream.
        if self._load >= 1.0:
            self._consumer.pause()

    def maintain_leases(self):
        """Maintain all of the leases being managed by the policy.

        This method modifies the ack deadline for all of the managed
        ack IDs, then waits for most of that time (but with jitter), and
        then calls itself.

        .. warning::
            This method blocks, and generally should be run in a separate
            thread or process.

            Additionally, you should not have to call this method yourself,
            unless you are implementing your own policy. If you are
            implementing your own policy, you _should_ call this method
            in an appropriate form of subprocess.
        """
        while True:
            # Sanity check: Should this infinite loop quit?
            if not self._consumer.active:
                _LOGGER.debug('Consumer inactive, ending lease maintenance.')
                return

            # Determine the appropriate duration for the lease. This is
            # based off of how long previous messages have taken to ack, with
            # a sensible default and within the ranges allowed by Pub/Sub.
            p99 = self.histogram.percentile(99)
            _LOGGER.debug('The current p99 value is %d seconds.', p99)

            # Create a streaming pull request.
            # We do not actually call `modify_ack_deadline` over and over
            # because it is more efficient to make a single request.
            ack_ids = list(self.managed_ack_ids)
            _LOGGER.debug('Renewing lease for %d ack IDs.', len(ack_ids))
            if ack_ids:
                request = types.StreamingPullRequest(
                    modify_deadline_ack_ids=ack_ids,
                    modify_deadline_seconds=[p99] * len(ack_ids),
                )
                # NOTE: This may not work as expected if ``consumer.active``
                #       has changed since we checked it. An implementation
                #       without any sort of race condition would require a
                #       way for ``send_request`` to fail when the consumer
                #       is inactive.
                self._consumer.send_request(request)

            # Now wait an appropriate period of time and do this again.
            #
            # We determine the appropriate period of time based on a random
            # period between 0 seconds and 90% of the lease. This use of
            # jitter (http://bit.ly/2s2ekL7) helps decrease contention in cases
            # where there are many clients.
            snooze = random.uniform(0.0, p99 * 0.9)
            _LOGGER.debug('Snoozing lease management for %f seconds.', snooze)
            time.sleep(snooze)

    def modify_ack_deadline(self, items):
        """Modify the ack deadline for the given messages.

        Args:
            items(Sequence[ModAckRequest]): The items to modify.
        """
        ack_ids = [item.ack_id for item in items]
        seconds = [item.seconds for item in items]

        request = types.StreamingPullRequest(
            modify_deadline_ack_ids=ack_ids,
            modify_deadline_seconds=seconds,
        )
        self._consumer.send_request(request)

    def nack(self, items):
        """Explicitly deny receipt of messages.

        Args:
            items(Sequence[NackRequest]): The items to deny.
        """
        self.modify_ack_deadline([
            ModAckRequest(ack_id=item.ack_id, seconds=0)
            for item in items])
        self.drop(
            [DropRequest(*item) for item in items])

    @abc.abstractmethod
    def close(self):
        """Close the existing connection.

        Raises:
            NotImplementedError: Always
        """
        raise NotImplementedError

    @abc.abstractmethod
    def on_exception(self, exception):
        """Called when a gRPC exception occurs.

        If this method does nothing, then the stream is re-started. If this
        raises an exception, it will stop the consumer thread. This is
        executed on the response consumer helper thread.

        Implementations should return :data:`True` if they want the consumer
        thread to remain active, otherwise they should return :data:`False`.

        Args:
            exception (Exception): The exception raised by the RPC.

        Raises:
            NotImplementedError: Always
        """
        raise NotImplementedError

    def on_request(self, request):
        """Called whenever a request has been sent to gRPC.

        This allows the policy to measure the rate of requests sent along the
        stream and apply backpressure by pausing or resuming the consumer
        if needed.

        Args:
            request (Any): The protobuf request that was sent to gRPC.
        """
        self._maybe_resume_consumer()

    @abc.abstractmethod
    def on_response(self, response):
        """Process a response from gRPC.

        This gives the consumer control over how responses are scheduled to
        be processed. This method is expected to not block and instead
        schedule the response to be consumed by some sort of concurrency.

        For example, if a the Policy implementation takes a callback in its
        constructor, you can schedule the callback using a
        :cls:`concurrent.futures.ThreadPoolExecutor`::

            self._pool.submit(self._callback, response)

        This is called from the response consumer helper thread.

        Args:
            response (Any): The protobuf response from the RPC.

        Raises:
            NotImplementedError: Always
        """
        raise NotImplementedError

    @abc.abstractmethod
    def open(self, callback):
        """Open a streaming pull connection and begin receiving messages.

        For each message received, the ``callback`` function is fired with
        a :class:`~.pubsub_v1.subscriber.message.Message` as its only
        argument.

        This method is virtual, but concrete implementations should return
        a :class:`~google.api_core.future.Future` that provides an interface
        to block on the subscription if desired, and handle errors.

        Args:
            callback (Callable[Message]): A callable that receives a
                Pub/Sub Message.

        Raises:
            NotImplementedError: Always
        """
        raise NotImplementedError
<|endoftext|>


<commit_before>import logging
import msgpack
import msgpack.fallback as m_fallback

logger = logging.getLogger(__name__)

# Single Packer object which is reused for performance
pakr = msgpack.Packer(use_bin_type=True, encoding='utf8', unicode_errors='surrogatepass')
if isinstance(pakr, m_fallback.Packer):  # pragma: no cover
    logger.warning('msgpack is using the pure python fallback implementation. This will impact performance negatively.')
    pakr = None

def en(item):
    '''
    Use msgpack to serialize a compatible python object.

    Args:
        item (obj): The object to serialize

    Notes:
        String objects are encoded using utf8 encoding.  In order to handle
        potentially malformed input, ``unicode_errors='surrogatepass'`` is set
        to allow encoding bad input strings.

    Returns:
        bytes: The serialized bytes in msgpack format.
    '''
    if pakr is None:  # pragma: no cover
        return msgpack.packb(item, use_bin_type=True, encoding='utf8',
                             unicode_errors='surrogatepass')
    try:
        return pakr.pack(item)
    except Exception as e:
        pakr.reset()
        raise

def un(byts):
    '''
    Use msgpack to de-serialize a python object.

    Args:
        byts (bytes): The bytes to de-serialize

    Notes:
        String objects are decoded using utf8 encoding.  In order to handle
        potentially malformed input, ``unicode_errors='surrogatepass'`` is set
        to allow decoding bad input strings.

    Returns:
        obj: The de-serialized object
    '''
    return msgpack.loads(byts, use_list=False, encoding='utf8',
                         unicode_errors='surrogatepass')

def iterfd(fd):
    '''
    Generator which unpacks a file object of msgpacked content.

    Args:
        fd: File object to consume data from.

    Notes:
        String objects are decoded using utf8 encoding.  In order to handle
        potentially malformed input, ``unicode_errors='surrogatepass'`` is set
        to allow decoding bad input strings.

    Yields:
        Objects from a msgpack stream.
    '''
    unpk = msgpack.Unpacker(fd, use_list=False, encoding='utf8',
                            unicode_errors='surrogatepass')
    for mesg in unpk:
        yield mesg

class Unpk:
    '''
    An extension of the msgpack streaming Unpacker which reports sizes.

    Notes:
        String objects are decoded using utf8 encoding.  In order to handle
        potentially malformed input, ``unicode_errors='surrogatepass'`` is set
        to allow decoding bad input strings.
    '''
    def __init__(self):
        self.size = 0
        self.unpk = msgpack.Unpacker(use_list=False, encoding='utf8',
                                     unicode_errors='surrogatepass')

    def feed(self, byts):
        '''
        Feed bytes to the unpacker and return completed objects.

        Args:
            byts (bytes): Bytes to unpack.

        Notes:
            It is intended that this function is called multiple times with
            bytes from some sort of a stream, as it will unpack and return
            objects as they are available.

        Returns:
            list: List of tuples containing the item size and the unpacked item.
        '''
        self.unpk.feed(byts)

        retn = []

        while True:

            try:
                item = self.unpk.unpack()
                retn.append((self.unpk.tell() - self.size, item))
                self.size = self.unpk.tell()

            except msgpack.exceptions.OutOfData:
                break

        return retn
<commit_msg>Set tell to a local variable to avoid a second function call.:<commit_after>import logging
import msgpack
import msgpack.fallback as m_fallback

logger = logging.getLogger(__name__)

# Single Packer object which is reused for performance
pakr = msgpack.Packer(use_bin_type=True, encoding='utf8', unicode_errors='surrogatepass')
if isinstance(pakr, m_fallback.Packer):  # pragma: no cover
    logger.warning('msgpack is using the pure python fallback implementation. This will impact performance negatively.')
    pakr = None

def en(item):
    '''
    Use msgpack to serialize a compatible python object.

    Args:
        item (obj): The object to serialize

    Notes:
        String objects are encoded using utf8 encoding.  In order to handle
        potentially malformed input, ``unicode_errors='surrogatepass'`` is set
        to allow encoding bad input strings.

    Returns:
        bytes: The serialized bytes in msgpack format.
    '''
    if pakr is None:  # pragma: no cover
        return msgpack.packb(item, use_bin_type=True, encoding='utf8',
                             unicode_errors='surrogatepass')
    try:
        return pakr.pack(item)
    except Exception as e:
        pakr.reset()
        raise

def un(byts):
    '''
    Use msgpack to de-serialize a python object.

    Args:
        byts (bytes): The bytes to de-serialize

    Notes:
        String objects are decoded using utf8 encoding.  In order to handle
        potentially malformed input, ``unicode_errors='surrogatepass'`` is set
        to allow decoding bad input strings.

    Returns:
        obj: The de-serialized object
    '''
    return msgpack.loads(byts, use_list=False, encoding='utf8',
                         unicode_errors='surrogatepass')

def iterfd(fd):
    '''
    Generator which unpacks a file object of msgpacked content.

    Args:
        fd: File object to consume data from.

    Notes:
        String objects are decoded using utf8 encoding.  In order to handle
        potentially malformed input, ``unicode_errors='surrogatepass'`` is set
        to allow decoding bad input strings.

    Yields:
        Objects from a msgpack stream.
    '''
    unpk = msgpack.Unpacker(fd, use_list=False, encoding='utf8',
                            unicode_errors='surrogatepass')
    for mesg in unpk:
        yield mesg

class Unpk:
    '''
    An extension of the msgpack streaming Unpacker which reports sizes.

    Notes:
        String objects are decoded using utf8 encoding.  In order to handle
        potentially malformed input, ``unicode_errors='surrogatepass'`` is set
        to allow decoding bad input strings.
    '''
    def __init__(self):
        self.size = 0
        self.unpk = msgpack.Unpacker(use_list=False, encoding='utf8',
                                     unicode_errors='surrogatepass')

    def feed(self, byts):
        '''
        Feed bytes to the unpacker and return completed objects.

        Args:
            byts (bytes): Bytes to unpack.

        Notes:
            It is intended that this function is called multiple times with
            bytes from some sort of a stream, as it will unpack and return
            objects as they are available.

        Returns:
            list: List of tuples containing the item size and the unpacked item.
        '''
        self.unpk.feed(byts)

        retn = []

        while True:

            try:
                item = self.unpk.unpack()
                tell = self.unpk.tell()
                retn.append((tell - self.size, item))
                self.size = tell

            except msgpack.exceptions.OutOfData:
                break

        return retn
<|endoftext|>


<commit_before>import logging
from statistics import (mean,
                        stdev)
from typing import (Union,
                    Iterable,
                    Iterator,
                    Tuple,
                    List)

from alcor.models import (Group,
                          Star)
from alcor.models.velocities_vs_magnitudes import (Bin,
                                                   LepineCaseUBin,
                                                   LepineCaseVBin,
                                                   LepineCaseWBin,
                                                   LepineCaseUCloud,
                                                   LepineCaseVCloud,
                                                   LepineCaseWCloud)
from alcor.types import (StarsBinsType,
                         RowType)

MIN_BOLOMETRIC_MAGNITUDE = 6.0
MAX_BOLOMETRIC_MAGNITUDE = 30.0
BIN_SIZE = 0.5
BOLOMETRIC_MAGNITUDE_AMPLITUDE = (MAX_BOLOMETRIC_MAGNITUDE
                                  - MIN_BOLOMETRIC_MAGNITUDE)
BINS_COUNT = int(BOLOMETRIC_MAGNITUDE_AMPLITUDE / BIN_SIZE)
DEFAULT_VELOCITY_STD = 100.

logger = logging.getLogger(__name__)

def stars_clouds(stars: List[Star],
                 group: Group) -> Iterator[Union[LepineCaseUCloud,
                                                 LepineCaseVCloud,
                                                 LepineCaseWCloud]]:
    group_id = group.id
    for star in stars:
        max_coordinates_modulus = star.max_coordinates_modulus
        bolometric_magnitude = star.bolometric_magnitude

        if abs(star.x_coordinate) == max_coordinates_modulus:
            yield LepineCaseVCloud(
                    group_id=group_id,
                    v_velocity=star.v_velocity,
                    bolometric_magnitude=bolometric_magnitude)
            yield LepineCaseWCloud(
                    group_id=group_id,
                    w_velocity=star.w_velocity,
                    bolometric_magnitude=bolometric_magnitude)
        elif abs(star.y_coordinate) == max_coordinates_modulus:
            yield LepineCaseUCloud(
                    group_id=group_id,
                    u_velocity=star.u_velocity,
                    bolometric_magnitude=bolometric_magnitude)
            yield LepineCaseWCloud(
                    group_id=group_id,
                    w_velocity=star.w_velocity,
                    bolometric_magnitude=bolometric_magnitude)
        else:
            yield LepineCaseUCloud(
                    group_id=group_id,
                    u_velocity=star.u_velocity,
                    bolometric_magnitude=bolometric_magnitude)
            yield LepineCaseVCloud(
                    group_id=group_id,
                    v_velocity=star.v_velocity,
                    bolometric_magnitude=bolometric_magnitude)

def generate_u_bins(*,
                    stars_bins: StarsBinsType,
                    group: Group) -> Iterable[LepineCaseUBin]:
    for index, stars_bin in enumerate(stars_bins):
        if not stars_bin:
            continue

        avg_magnitude = (MIN_BOLOMETRIC_MAGNITUDE
                         + BIN_SIZE * (index + 0.5))
        avg_u_velocity = mean(star.u_velocity
                              for star in stars_bin)
        if len(stars_bin) == 1:
            u_velocity_std = DEFAULT_VELOCITY_STD
        else:
            u_velocity_std = stdev(star.u_velocity
                                   for star in stars_bin)
        yield LepineCaseUBin(group_id=group.id,
                             avg_magnitude=avg_magnitude,
                             avg_u_velocity=avg_u_velocity,
                             u_velocity_std=u_velocity_std)

def generate_v_bins(*,
                    stars_bins: StarsBinsType,
                    group: Group) -> Iterable[LepineCaseVBin]:
    for stars_bin_index, stars_bin in enumerate(stars_bins):
        if not stars_bin:
            continue

        avg_magnitude = (MIN_BOLOMETRIC_MAGNITUDE
                         + BIN_SIZE * (stars_bin_index + 0.5))
        avg_v_velocity = mean(star.v_velocity
                              for star in stars_bin)
        if len(stars_bin) == 1:
            v_velocity_std = DEFAULT_VELOCITY_STD
        else:
            v_velocity_std = stdev(star.v_velocity
                                   for star in stars_bin)
        yield LepineCaseVBin(group_id=group.id,
                             avg_magnitude=avg_magnitude,
                             avg_v_velocity=avg_v_velocity,
                             v_velocity_std=v_velocity_std)

def generate_w_bins(*,
                    stars_bins: StarsBinsType,
                    group: Group) -> Iterable[RowType]:
    for stars_bin_index, stars_bin in enumerate(stars_bins):
        if not stars_bin:
            continue

        avg_magnitude = (MIN_BOLOMETRIC_MAGNITUDE
                         + BIN_SIZE * (stars_bin_index + 0.5))
        avg_w_velocity = mean(star.w_velocity
                              for star in stars_bin)
        if len(stars_bin) == 1:
            w_velocity_std = DEFAULT_VELOCITY_STD
        else:
            w_velocity_std = stdev(star.w_velocity
                                   for star in stars_bin)
        yield LepineCaseWBin(group_id=group.id,
                             avg_magnitude=avg_magnitude,
                             avg_w_velocity=avg_w_velocity,
                             w_velocity_std=w_velocity_std)

def lepine_stars_bins(stars: List[Star]) -> Tuple[StarsBinsType,
                                                  StarsBinsType,
                                                  StarsBinsType]:
    u_stars_bins = [[] for _ in range(BINS_COUNT)]
    v_stars_bins = [[] for _ in range(BINS_COUNT)]
    w_stars_bins = [[] for _ in range(BINS_COUNT)]

    for star in stars:
        index = get_stars_bin_index(star)

        max_coordinates_modulus = star.max_coordinates_modulus

        if abs(star.x_coordinate) == max_coordinates_modulus:
            v_stars_bins[index].append(star)
            w_stars_bins[index].append(star)
        elif abs(star.y_coordinate) == max_coordinates_modulus:
            u_stars_bins[index].append(star)
            w_stars_bins[index].append(star)
        else:
            u_stars_bins[index].append(star)
            v_stars_bins[index].append(star)

    return u_stars_bins, v_stars_bins, w_stars_bins

def get_stars_bin_index(star: Star) -> int:
    return int((float(star.bolometric_magnitude)
                - MIN_BOLOMETRIC_MAGNITUDE) / BIN_SIZE)

def raw_stars_bins(stars: List[Star]) -> StarsBinsType:
    res = [[] for _ in range(BINS_COUNT)]
    for star in stars:
        index = get_stars_bin_index(star)
        if BINS_COUNT > index >= 0:
            res[index].append(star)
        else:
            logger.warning(f'Magnitude is out of bounds: '
                           f'{star.bolometric_magnitude}')
    return res

def generate_bins(*,
                  stars_bins: StarsBinsType,
                  group: Group) -> Iterable[Bin]:
    for index, stars_bin in enumerate(stars_bins):
        if not stars_bin:
            continue

        avg_magnitude = (MIN_BOLOMETRIC_MAGNITUDE
                         + BIN_SIZE * (index + 0.5))
        avg_u_velocity = mean(star.u_velocity
                              for star in stars_bin)
        avg_v_velocity = mean(star.v_velocity
                              for star in stars_bin)
        avg_w_velocity = mean(star.w_velocity
                              for star in stars_bin)
        if len(stars_bin) > 1:
            u_velocity_std = stdev(star.u_velocity
                                   for star in stars_bin)
            v_velocity_std = stdev(star.v_velocity
                                   for star in stars_bin)
            w_velocity_std = stdev(star.w_velocity
                                   for star in stars_bin)
        else:
            u_velocity_std = DEFAULT_VELOCITY_STD
            v_velocity_std = DEFAULT_VELOCITY_STD
            w_velocity_std = DEFAULT_VELOCITY_STD
        yield Bin(group_id=group.id,
                  avg_magnitude=avg_magnitude,
                  avg_u_velocity=avg_u_velocity,
                  avg_v_velocity=avg_v_velocity,
                  avg_w_velocity=avg_w_velocity,
                  u_velocity_std=u_velocity_std,
                  v_velocity_std=v_velocity_std,
                  w_velocity_std=w_velocity_std)
<commit_msg>Fixed indentation<commit_after>import logging
from statistics import (mean,
                        stdev)
from typing import (Union,
                    Iterable,
                    Iterator,
                    Tuple,
                    List)

from alcor.models import (Group,
                          Star)
from alcor.models.velocities_vs_magnitudes import (Bin,
                                                   LepineCaseUBin,
                                                   LepineCaseVBin,
                                                   LepineCaseWBin,
                                                   LepineCaseUCloud,
                                                   LepineCaseVCloud,
                                                   LepineCaseWCloud)
from alcor.types import (StarsBinsType,
                         RowType)

MIN_BOLOMETRIC_MAGNITUDE = 6.0
MAX_BOLOMETRIC_MAGNITUDE = 30.0
BIN_SIZE = 0.5
BOLOMETRIC_MAGNITUDE_AMPLITUDE = (MAX_BOLOMETRIC_MAGNITUDE
                                  - MIN_BOLOMETRIC_MAGNITUDE)
BINS_COUNT = int(BOLOMETRIC_MAGNITUDE_AMPLITUDE / BIN_SIZE)
DEFAULT_VELOCITY_STD = 100.

logger = logging.getLogger(__name__)

def stars_clouds(stars: List[Star],
                 group: Group) -> Iterator[Union[LepineCaseUCloud,
                                                 LepineCaseVCloud,
                                                 LepineCaseWCloud]]:
    group_id = group.id
    for star in stars:
        max_coordinates_modulus = star.max_coordinates_modulus
        bolometric_magnitude = star.bolometric_magnitude

        if abs(star.x_coordinate) == max_coordinates_modulus:
            yield LepineCaseVCloud(group_id=group_id,
                                   v_velocity=star.v_velocity,
                                   bolometric_magnitude=bolometric_magnitude)
            yield LepineCaseWCloud(group_id=group_id,
                                   w_velocity=star.w_velocity,
                                   bolometric_magnitude=bolometric_magnitude)
        elif abs(star.y_coordinate) == max_coordinates_modulus:
            yield LepineCaseUCloud(group_id=group_id,
                                   u_velocity=star.u_velocity,
                                   bolometric_magnitude=bolometric_magnitude)
            yield LepineCaseWCloud(group_id=group_id,
                                   w_velocity=star.w_velocity,
                                   bolometric_magnitude=bolometric_magnitude)
        else:
            yield LepineCaseUCloud(group_id=group_id,
                                   u_velocity=star.u_velocity,
                                   bolometric_magnitude=bolometric_magnitude)
            yield LepineCaseVCloud(group_id=group_id,
                                   v_velocity=star.v_velocity,
                                   bolometric_magnitude=bolometric_magnitude)

def generate_u_bins(*,
                    stars_bins: StarsBinsType,
                    group: Group) -> Iterable[LepineCaseUBin]:
    for index, stars_bin in enumerate(stars_bins):
        if not stars_bin:
            continue

        avg_magnitude = (MIN_BOLOMETRIC_MAGNITUDE
                         + BIN_SIZE * (index + 0.5))
        avg_u_velocity = mean(star.u_velocity
                              for star in stars_bin)
        if len(stars_bin) == 1:
            u_velocity_std = DEFAULT_VELOCITY_STD
        else:
            u_velocity_std = stdev(star.u_velocity
                                   for star in stars_bin)
        yield LepineCaseUBin(group_id=group.id,
                             avg_magnitude=avg_magnitude,
                             avg_u_velocity=avg_u_velocity,
                             u_velocity_std=u_velocity_std)

def generate_v_bins(*,
                    stars_bins: StarsBinsType,
                    group: Group) -> Iterable[LepineCaseVBin]:
    for stars_bin_index, stars_bin in enumerate(stars_bins):
        if not stars_bin:
            continue

        avg_magnitude = (MIN_BOLOMETRIC_MAGNITUDE
                         + BIN_SIZE * (stars_bin_index + 0.5))
        avg_v_velocity = mean(star.v_velocity
                              for star in stars_bin)
        if len(stars_bin) == 1:
            v_velocity_std = DEFAULT_VELOCITY_STD
        else:
            v_velocity_std = stdev(star.v_velocity
                                   for star in stars_bin)
        yield LepineCaseVBin(group_id=group.id,
                             avg_magnitude=avg_magnitude,
                             avg_v_velocity=avg_v_velocity,
                             v_velocity_std=v_velocity_std)

def generate_w_bins(*,
                    stars_bins: StarsBinsType,
                    group: Group) -> Iterable[RowType]:
    for stars_bin_index, stars_bin in enumerate(stars_bins):
        if not stars_bin:
            continue

        avg_magnitude = (MIN_BOLOMETRIC_MAGNITUDE
                         + BIN_SIZE * (stars_bin_index + 0.5))
        avg_w_velocity = mean(star.w_velocity
                              for star in stars_bin)
        if len(stars_bin) == 1:
            w_velocity_std = DEFAULT_VELOCITY_STD
        else:
            w_velocity_std = stdev(star.w_velocity
                                   for star in stars_bin)
        yield LepineCaseWBin(group_id=group.id,
                             avg_magnitude=avg_magnitude,
                             avg_w_velocity=avg_w_velocity,
                             w_velocity_std=w_velocity_std)

def lepine_stars_bins(stars: List[Star]) -> Tuple[StarsBinsType,
                                                  StarsBinsType,
                                                  StarsBinsType]:
    u_stars_bins = [[] for _ in range(BINS_COUNT)]
    v_stars_bins = [[] for _ in range(BINS_COUNT)]
    w_stars_bins = [[] for _ in range(BINS_COUNT)]

    for star in stars:
        index = get_stars_bin_index(star)

        max_coordinates_modulus = star.max_coordinates_modulus

        if abs(star.x_coordinate) == max_coordinates_modulus:
            v_stars_bins[index].append(star)
            w_stars_bins[index].append(star)
        elif abs(star.y_coordinate) == max_coordinates_modulus:
            u_stars_bins[index].append(star)
            w_stars_bins[index].append(star)
        else:
            u_stars_bins[index].append(star)
            v_stars_bins[index].append(star)

    return u_stars_bins, v_stars_bins, w_stars_bins

def get_stars_bin_index(star: Star) -> int:
    return int((float(star.bolometric_magnitude)
                - MIN_BOLOMETRIC_MAGNITUDE) / BIN_SIZE)

def raw_stars_bins(stars: List[Star]) -> StarsBinsType:
    res = [[] for _ in range(BINS_COUNT)]
    for star in stars:
        index = get_stars_bin_index(star)
        if BINS_COUNT > index >= 0:
            res[index].append(star)
        else:
            logger.warning(f'Magnitude is out of bounds: '
                           f'{star.bolometric_magnitude}')
    return res

def generate_bins(*,
                  stars_bins: StarsBinsType,
                  group: Group) -> Iterable[Bin]:
    for index, stars_bin in enumerate(stars_bins):
        if not stars_bin:
            continue

        avg_magnitude = (MIN_BOLOMETRIC_MAGNITUDE
                         + BIN_SIZE * (index + 0.5))
        avg_u_velocity = mean(star.u_velocity
                              for star in stars_bin)
        avg_v_velocity = mean(star.v_velocity
                              for star in stars_bin)
        avg_w_velocity = mean(star.w_velocity
                              for star in stars_bin)
        if len(stars_bin) > 1:
            u_velocity_std = stdev(star.u_velocity
                                   for star in stars_bin)
            v_velocity_std = stdev(star.v_velocity
                                   for star in stars_bin)
            w_velocity_std = stdev(star.w_velocity
                                   for star in stars_bin)
        else:
            u_velocity_std = DEFAULT_VELOCITY_STD
            v_velocity_std = DEFAULT_VELOCITY_STD
            w_velocity_std = DEFAULT_VELOCITY_STD
        yield Bin(group_id=group.id,
                  avg_magnitude=avg_magnitude,
                  avg_u_velocity=avg_u_velocity,
                  avg_v_velocity=avg_v_velocity,
                  avg_w_velocity=avg_w_velocity,
                  u_velocity_std=u_velocity_std,
                  v_velocity_std=v_velocity_std,
                  w_velocity_std=w_velocity_std)
<|endoftext|>


<commit_before>import csv
import itertools
from collections import namedtuple

from django.core.management import BaseCommand, CommandError

from corehq.apps.change_feed import data_sources, topics
from corehq.apps.change_feed.producer import producer
from corehq.apps.hqadmin.management.commands.stale_data_in_es import DataRow, HEADER_ROW, get_csv_args
from corehq.form_processor.utils import should_use_sql_backend
from corehq.toggles import DO_NOT_REPUBLISH_DOCS
from couchforms.models import XFormInstance
from dimagi.utils.chunked import chunked

from casexml.apps.case.models import CommCareCase
from corehq.util.couch import bulk_get_revs
from corehq.apps.hqcase.management.commands.backfill_couch_forms_and_cases import (
    create_case_change_meta,
    create_form_change_meta,
    publish_change,
)
from pillowtop.feed.interface import ChangeMeta

DocumentRecord = namedtuple('DocumentRecord', ['doc_id', 'doc_type', 'doc_subtype', 'domain'])

CASE_DOC_TYPES = {'CommCareCase'}
FORM_DOC_TYPES = {'XFormInstance', 'XFormArchived'}

ALL_DOC_TYPES = set.union(CASE_DOC_TYPES, FORM_DOC_TYPES)

class Command(BaseCommand):
    """
    Republish doc changes. Meant to be used in conjunction with stale_data_in_es command

        $ ./manage.py republish_doc_changes changes.tsv
    """

    def add_arguments(self, parser):
        parser.add_argument('stale_data_in_es_file')
        parser.add_argument('--delimiter', default='\t', choices=('\t', ','))
        parser.add_arument('--skip_domains', action='store_true')

    def handle(self, stale_data_in_es_file, delimiter, skip_domains, *args, **options):
        data_rows = _get_data_rows(stale_data_in_es_file, delimiter=delimiter)
        document_records = _get_document_records(data_rows)
        form_records = []
        case_records = []
        for record in document_records:
            if record.doc_type in CASE_DOC_TYPES:
                case_records.append(record)
            elif record.doc_type in FORM_DOC_TYPES:
                form_records.append(record)
            else:
                assert False, f'Bad doc type {record.doc_type} should have been caught already below.'
        _publish_cases(case_records, skip_domains=skip_domains)
        _publish_forms(form_records, skip_domains=skip_domains)

def _get_data_rows(stale_data_in_es_file, delimiter):
    with open(stale_data_in_es_file, 'r') as f:
        csv_reader = csv.reader(f, **get_csv_args(delimiter))
        for csv_row in csv_reader:
            data_row = DataRow(*csv_row)
            # Skip the header row anywhere in the file.
            # The "anywhere in the file" part is useful
            # if you cat multiple stale_data_in_es_file files together.
            if data_row != HEADER_ROW:
                yield data_row

def _get_document_records(data_rows):
    for data_row in data_rows:
        doc_id, doc_type, doc_subtype, domain = \
            data_row.doc_id, data_row.doc_type, data_row.doc_subtype, data_row.domain
        if doc_type not in ALL_DOC_TYPES:
            raise CommandError(f"Found bad doc type {doc_type}. "
                               "Did you use the right command to create the data?")
        yield DocumentRecord(doc_id, doc_type, doc_subtype, domain)

def _publish_cases(case_records, skip_domains):
    for domain, records in itertools.groupby(case_records, lambda r: r.domain):
        if skip_domains and DO_NOT_REPUBLISH_DOCS.enabled(domain):
            continue
        if should_use_sql_backend(domain):
            _publish_cases_for_sql(domain, list(records))
        else:
            _publish_cases_for_couch(domain, list(records))

def _publish_forms(form_records, skip_domains):
    for domain, records in itertools.groupby(form_records, lambda r: r.domain):
        if skip_domains and DO_NOT_REPUBLISH_DOCS.enabled(domain):
            continue
        if should_use_sql_backend(domain):
            _publish_forms_for_sql(domain, records)
        else:
            _publish_forms_for_couch(domain, records)

def _publish_cases_for_couch(domain, case_records):
    _publish_docs_for_couch(CommCareCase, create_case_change_meta, domain, case_records)

def _publish_cases_for_sql(domain, case_records):
    for record in case_records:
        producer.send_change(
            topics.CASE_SQL,
            _change_meta_for_sql_case(domain, record.doc_id, record.doc_subtype)
        )

def _change_meta_for_sql_case(domain, case_id, case_type):
    doc_type, = CASE_DOC_TYPES
    return ChangeMeta(
        document_id=case_id,
        data_source_type=data_sources.SOURCE_SQL,
        data_source_name=data_sources.CASE_SQL,
        document_type=doc_type,
        document_subtype=case_type,
        domain=domain,
        is_deletion=False,
    )

def _publish_forms_for_sql(domain, form_records):
    for record in form_records:
        producer.send_change(
            topics.FORM_SQL,
            _change_meta_for_sql_form_record(domain, record)
        )

def _publish_forms_for_couch(domain, form_records):
    _publish_docs_for_couch(XFormInstance, create_form_change_meta, domain, form_records)

def _change_meta_for_sql_form_record(domain, form_record):
    return ChangeMeta(
        document_id=form_record.doc_id,
        data_source_type=data_sources.SOURCE_SQL,
        data_source_name=data_sources.FORM_SQL,
        document_type=form_record.doc_type,
        document_subtype=form_record.doc_subtype,
        domain=domain,
        is_deletion=False,
    )

def _publish_docs_for_couch(doc_cls, get_meta, domain, records):
    doc_ids = [r.doc_id for r in records]
    for ids in chunked(doc_ids, 500):
        doc_id_rev_list = bulk_get_revs(doc_cls.get_db(), ids)
        for doc_id, doc_rev in doc_id_rev_list:
            publish_change(
                get_meta(domain, doc_id, doc_rev)
            )
<commit_msg>fix typo<commit_after>import csv
import itertools
from collections import namedtuple

from django.core.management import BaseCommand, CommandError

from corehq.apps.change_feed import data_sources, topics
from corehq.apps.change_feed.producer import producer
from corehq.apps.hqadmin.management.commands.stale_data_in_es import DataRow, HEADER_ROW, get_csv_args
from corehq.form_processor.utils import should_use_sql_backend
from corehq.toggles import DO_NOT_REPUBLISH_DOCS
from couchforms.models import XFormInstance
from dimagi.utils.chunked import chunked

from casexml.apps.case.models import CommCareCase
from corehq.util.couch import bulk_get_revs
from corehq.apps.hqcase.management.commands.backfill_couch_forms_and_cases import (
    create_case_change_meta,
    create_form_change_meta,
    publish_change,
)
from pillowtop.feed.interface import ChangeMeta

DocumentRecord = namedtuple('DocumentRecord', ['doc_id', 'doc_type', 'doc_subtype', 'domain'])

CASE_DOC_TYPES = {'CommCareCase'}
FORM_DOC_TYPES = {'XFormInstance', 'XFormArchived'}

ALL_DOC_TYPES = set.union(CASE_DOC_TYPES, FORM_DOC_TYPES)

class Command(BaseCommand):
    """
    Republish doc changes. Meant to be used in conjunction with stale_data_in_es command

        $ ./manage.py republish_doc_changes changes.tsv
    """

    def add_arguments(self, parser):
        parser.add_argument('stale_data_in_es_file')
        parser.add_argument('--delimiter', default='\t', choices=('\t', ','))
        parser.add_argument('--skip_domains', action='store_true')

    def handle(self, stale_data_in_es_file, delimiter, skip_domains, *args, **options):
        data_rows = _get_data_rows(stale_data_in_es_file, delimiter=delimiter)
        document_records = _get_document_records(data_rows)
        form_records = []
        case_records = []
        for record in document_records:
            if record.doc_type in CASE_DOC_TYPES:
                case_records.append(record)
            elif record.doc_type in FORM_DOC_TYPES:
                form_records.append(record)
            else:
                assert False, f'Bad doc type {record.doc_type} should have been caught already below.'
        _publish_cases(case_records, skip_domains=skip_domains)
        _publish_forms(form_records, skip_domains=skip_domains)

def _get_data_rows(stale_data_in_es_file, delimiter):
    with open(stale_data_in_es_file, 'r') as f:
        csv_reader = csv.reader(f, **get_csv_args(delimiter))
        for csv_row in csv_reader:
            data_row = DataRow(*csv_row)
            # Skip the header row anywhere in the file.
            # The "anywhere in the file" part is useful
            # if you cat multiple stale_data_in_es_file files together.
            if data_row != HEADER_ROW:
                yield data_row

def _get_document_records(data_rows):
    for data_row in data_rows:
        doc_id, doc_type, doc_subtype, domain = \
            data_row.doc_id, data_row.doc_type, data_row.doc_subtype, data_row.domain
        if doc_type not in ALL_DOC_TYPES:
            raise CommandError(f"Found bad doc type {doc_type}. "
                               "Did you use the right command to create the data?")
        yield DocumentRecord(doc_id, doc_type, doc_subtype, domain)

def _publish_cases(case_records, skip_domains):
    for domain, records in itertools.groupby(case_records, lambda r: r.domain):
        if skip_domains and DO_NOT_REPUBLISH_DOCS.enabled(domain):
            continue
        if should_use_sql_backend(domain):
            _publish_cases_for_sql(domain, list(records))
        else:
            _publish_cases_for_couch(domain, list(records))

def _publish_forms(form_records, skip_domains):
    for domain, records in itertools.groupby(form_records, lambda r: r.domain):
        if skip_domains and DO_NOT_REPUBLISH_DOCS.enabled(domain):
            continue
        if should_use_sql_backend(domain):
            _publish_forms_for_sql(domain, records)
        else:
            _publish_forms_for_couch(domain, records)

def _publish_cases_for_couch(domain, case_records):
    _publish_docs_for_couch(CommCareCase, create_case_change_meta, domain, case_records)

def _publish_cases_for_sql(domain, case_records):
    for record in case_records:
        producer.send_change(
            topics.CASE_SQL,
            _change_meta_for_sql_case(domain, record.doc_id, record.doc_subtype)
        )

def _change_meta_for_sql_case(domain, case_id, case_type):
    doc_type, = CASE_DOC_TYPES
    return ChangeMeta(
        document_id=case_id,
        data_source_type=data_sources.SOURCE_SQL,
        data_source_name=data_sources.CASE_SQL,
        document_type=doc_type,
        document_subtype=case_type,
        domain=domain,
        is_deletion=False,
    )

def _publish_forms_for_sql(domain, form_records):
    for record in form_records:
        producer.send_change(
            topics.FORM_SQL,
            _change_meta_for_sql_form_record(domain, record)
        )

def _publish_forms_for_couch(domain, form_records):
    _publish_docs_for_couch(XFormInstance, create_form_change_meta, domain, form_records)

def _change_meta_for_sql_form_record(domain, form_record):
    return ChangeMeta(
        document_id=form_record.doc_id,
        data_source_type=data_sources.SOURCE_SQL,
        data_source_name=data_sources.FORM_SQL,
        document_type=form_record.doc_type,
        document_subtype=form_record.doc_subtype,
        domain=domain,
        is_deletion=False,
    )

def _publish_docs_for_couch(doc_cls, get_meta, domain, records):
    doc_ids = [r.doc_id for r in records]
    for ids in chunked(doc_ids, 500):
        doc_id_rev_list = bulk_get_revs(doc_cls.get_db(), ids)
        for doc_id, doc_rev in doc_id_rev_list:
            publish_change(
                get_meta(domain, doc_id, doc_rev)
            )
<|endoftext|>


<commit_before>import time
import datetime
import subprocess
from collections import OrderedDict
from catnet.turning import stop, turn_clockwise, turn_c_clockwise
from catnet.compass import get_bearing
import RPi.GPIO as GPIO
import RPIO
import threading

GPIO_sensors = OrderedDict([
    (23, 0),
    (24, 117),
    (25, 180),
    (18, 278),
])

#Create a variable called daemon running, and a global set
angle_offset = None
daemon_running = False
motion_set = set()

#Function for reading motion_set and working out the correct angle to turn
def motion_bearing():
    motion_bearing = []
    for sensor, angle in GPIO_sensors.items():
        if sensor in motion_set:
            motion_bearing.append(angle)
    bearing = sum(motion_bearing)/len(motion_bearing)
    if bearing > 360:
        bearing = bearing - 360
    return bearing

#Function for activating turret
def turret(target_bearing):
    while True:
        current_bearing = get_bearing() - angle_offset
        if current_bearing < 0:
            current_bearing = current_bearing + 360
        print current_bearing
        if current_bearing - target_bearing > 10:
            turn_c_clockwise()
        elif current_bearing - target_bearing < -10:
            turn_clockwise()
        else:
            stop()
            break
        time.sleep(0.02)

#take picture function
def take_snap():
    time.sleep(1)
    file_name = (datetime.datetime.now()).strftime('%Y-%m-%d-%H-%M-%S')
    subprocess.call('fswebcam --device v4l2:/dev/video0 --input 0 --resolution 320x240 --skip 100 --jpeg 80 /home/jenny/test/image-{}.jpg'.format(file_name), shell=True)
    print "picture taken"

#Daemon thread
def activate_turret():
    global daemon_running
    time.sleep(0.5)
    print "daemon waking after sleep"
    print motion_set
    bearing = motion_bearing()
    print bearing
    turret(bearing)
    take_snap()
    daemon_running = False
    print "daemon died"

def motion_detected(gpio, x):
    global motion_set, daemon_running
    print "motion detected"
    if daemon_running == True:
        print "adding to daemon"
        motion_set.add(gpio)
    elif daemon_running == False:
        print "starting daemon"
        motion_set = set()
        motion_set.add(gpio)
        daemon_running = True
        d = threading.Thread(target=activate_turret)
        d.setDaemon(True)
        d.start()

def start_overwatch():
    global angle_offset
    try:
        #GPIO setup with relative headings

    	GPIO.setmode(GPIO.BCM)

    	for sensor in GPIO_sensors.keys():
    		GPIO.setup(sensor, GPIO.IN, pull_up_down=GPIO.PUD_DOWN)

        #Calibrate compuss heading to relative heading

    	angle_offset = get_bearing()
    	print angle_offset

    	for pin_number in GPIO_sensors.keys():
    		RPIO.add_interrupt_callback(pin_number, callback=motion_detected)

    	RPIO.wait_for_interrupts()

    finally:
    	stop()
    	GPIO.cleanup()

<commit_msg>More debug<commit_after>import time
import datetime
import subprocess
from collections import OrderedDict
from catnet.turning import stop, turn_clockwise, turn_c_clockwise
from catnet.compass import get_bearing
import RPi.GPIO as GPIO
import RPIO
import threading

GPIO_sensors = OrderedDict([
    (23, 0),
    (24, 117),
    (25, 180),
    (18, 278),
])

#Create a variable called daemon running, and a global set
angle_offset = None
daemon_running = False
motion_set = set()

#Function for reading motion_set and working out the correct angle to turn
def motion_bearing():
    motion_bearing = []
    for sensor, angle in GPIO_sensors.items():
        if sensor in motion_set:
            motion_bearing.append(angle)
    print("detected motion at", motion_bearing)
    bearing = sum(motion_bearing)/len(motion_bearing)
    if bearing > 360:
        bearing = bearing - 360
    return bearing

#Function for activating turret
def turret(target_bearing):
    print("turning to", target_bearing);
    while True:
        current_bearing = get_bearing() - angle_offset
        if current_bearing < 0:
            current_bearing = current_bearing + 360
        print current_bearing
        if current_bearing - target_bearing > 10:
            turn_c_clockwise()
        elif current_bearing - target_bearing < -10:
            turn_clockwise()
        else:
            stop()
            break
        time.sleep(0.02)

#take picture function
def take_snap():
    time.sleep(1)
    file_name = (datetime.datetime.now()).strftime('%Y-%m-%d-%H-%M-%S')
    subprocess.call('fswebcam --device v4l2:/dev/video0 --input 0 --resolution 320x240 --skip 100 --jpeg 80 /home/jenny/test/image-{}.jpg'.format(file_name), shell=True)
    print "picture taken"

#Daemon thread
def activate_turret():
    global daemon_running
    time.sleep(0.5)
    print "daemon waking after sleep"
    print motion_set
    bearing = motion_bearing()
    print bearing
    turret(bearing)
    take_snap()
    daemon_running = False
    print "daemon died"

def motion_detected(gpio, x):
    global motion_set, daemon_running
    print "motion detected"
    if daemon_running == True:
        print "adding to daemon"
        motion_set.add(gpio)
    elif daemon_running == False:
        print "starting daemon"
        motion_set = set()
        motion_set.add(gpio)
        daemon_running = True
        d = threading.Thread(target=activate_turret)
        d.setDaemon(True)
        d.start()

def start_overwatch():
    global angle_offset
    try:
        #GPIO setup with relative headings

    	GPIO.setmode(GPIO.BCM)

    	for sensor in GPIO_sensors.keys():
    		GPIO.setup(sensor, GPIO.IN, pull_up_down=GPIO.PUD_DOWN)

        #Calibrate compuss heading to relative heading

    	angle_offset = get_bearing()
    	print angle_offset

    	for pin_number in GPIO_sensors.keys():
    		RPIO.add_interrupt_callback(pin_number, callback=motion_detected)

    	RPIO.wait_for_interrupts()

    finally:
    	stop()
    	GPIO.cleanup()

<|endoftext|>