<commit_before>"""
soupselect.py - https://code.google.com/p/soupselect/

CSS selector support for BeautifulSoup.

soup = BeautifulSoup('<html>...')
select(soup, 'div')
- returns a list of div elements

select(soup, 'div#main ul a')
- returns a list of links inside a ul inside div#main

"""

import re
from BeautifulSoup import BeautifulSoup

tag_re = re.compile('^[a-z0-9]+$')

attribselect_re = re.compile(
    r'^(?P<tag>\w+)?\[(?P<attribute>\w+)(?P<operator>[=~\|\^\$\*]?)' +
    r'=?"?(?P<value>[^\]"]*)"?\]$'
)

# /^(\w+)\[(\w+)([=~\|\^\$\*]?)=?"?([^\]"]*)"?\]$/
#   \---/  \---/\-------------/    \-------/
#     |      |         |               |
#     |      |         |           The value
#     |      |    ~,|,^,$,* or =
#     |   Attribute
#    Tag

def attribute_checker(operator, attribute, value=''):
    """
    Takes an operator, attribute and optional value; returns a function that
    will return True for elements that match that combination.
    """
    return {
        '=': lambda el: el.get(attribute) == value,
        # attribute includes value as one of a set of space separated tokens
        '~': lambda el: value in el.get(attribute, '').split(),
        # attribute starts with value
        '^': lambda el: el.get(attribute, '').startswith(value),
        # attribute ends with value
        '$': lambda el: el.get(attribute, '').endswith(value),
        # attribute contains value
        '*': lambda el: value in el.get(attribute, ''),
        # attribute is either exactly value or starts with value-
        '|': lambda el: el.get(attribute, '') == value \
            or el.get(attribute, '').startswith('%s-' % value),
    }.get(operator, lambda el: el.has_key(attribute))

def select(soup, selector):
    """
    soup should be a BeautifulSoup instance; selector is a CSS selector
    specifying the elements you want to retrieve.
    """
    tokens = selector.split()
    current_context = [soup]
    for token in tokens:
        m = attribselect_re.match(token)
        if m:
            # Attribute selector
            tag, attribute, operator, value = m.groups()
            if not tag:
                tag = True
            checker = attribute_checker(operator, attribute, value)
            found = []
            for context in current_context:
                found.extend([el for el in context.findAll(tag) if checker(el)])
            current_context = found
            continue
        if '#' in token:
            # ID selector
            tag, id = token.split('#', 1)
            if not tag:
                tag = True
            el = current_context[0].find(tag, {'id': id})
            if not el:
                return [] # No match
            current_context = [el]
            continue
        if '.' in token:
            # Class selector
            tag, klass = token.split('.', 1)
            if not tag:
                tag = True
            found = []
            for context in current_context:
                found.extend(
                    context.findAll(tag,
                        {'class': lambda attr: attr and klass in attr.split()}
                    )
                )
            current_context = found
            continue
        if token == '*':
            # Star selector
            found = []
            for context in current_context:
                found.extend(context.findAll(True))
            current_context = found
            continue
        # Here we should just have a regular tag
        if not tag_re.match(token):
            return []
        found = []
        for context in current_context:
            found.extend(context.findAll(token))
        current_context = found
    return current_context

def monkeypatch(BeautifulSoupClass=None):
    """
    If you don't explicitly state the class to patch, defaults to the most
    common import location for BeautifulSoup.
    """
    if not BeautifulSoupClass:
        from BeautifulSoup import BeautifulSoup as BeautifulSoupClass
    BeautifulSoupClass.findSelect = select

def unmonkeypatch(BeautifulSoupClass=None):
    if not BeautifulSoupClass:
        from BeautifulSoup import BeautifulSoup as BeautifulSoupClass
    delattr(BeautifulSoupClass, 'findSelect')

def cssFind(html, selector):
    """
    Parse ``html`` with class:`BeautifulSoup.BeautifulSoup` and use
    :func:`.select` on the result.

    Added by Espen A. Kristiansen to make it even easier to use for testing.
    """
    soup = BeautifulSoup(html)
    return select(soup, selector)

def cssGet(html, selector):
    """
    Same as :func:`.cssFind`, but returns the first match.

    Added by Espen A. Kristiansen to make it even easier to use for testing.
    """
    try:
        return cssFind(html, selector)[0]
    except IndexError as e:
        raise IndexError('Could not find {}.'.format(selector))

def cssExists(html, selector):
    """
    Same as :func:`.cssFind`, but returns ``True`` if the selector matches at least one item.

    Added by Espen A. Kristiansen to make it even easier to use for testing.
    """
    matches = cssFind(html, selector)
    return bool(len(matches))

def prettyhtml(html):
    return BeautifulSoup(html).prettify()<commit_msg>devilry_developer.testhelpers.soupselect: Added normalize_whitespace()<commit_after>"""
soupselect.py - https://code.google.com/p/soupselect/

CSS selector support for BeautifulSoup.

soup = BeautifulSoup('<html>...')
select(soup, 'div')
- returns a list of div elements

select(soup, 'div#main ul a')
- returns a list of links inside a ul inside div#main

"""

import re
from BeautifulSoup import BeautifulSoup

tag_re = re.compile('^[a-z0-9]+$')

attribselect_re = re.compile(
    r'^(?P<tag>\w+)?\[(?P<attribute>\w+)(?P<operator>[=~\|\^\$\*]?)' +
    r'=?"?(?P<value>[^\]"]*)"?\]$'
)

# /^(\w+)\[(\w+)([=~\|\^\$\*]?)=?"?([^\]"]*)"?\]$/
#   \---/  \---/\-------------/    \-------/
#     |      |         |               |
#     |      |         |           The value
#     |      |    ~,|,^,$,* or =
#     |   Attribute
#    Tag

def attribute_checker(operator, attribute, value=''):
    """
    Takes an operator, attribute and optional value; returns a function that
    will return True for elements that match that combination.
    """
    return {
        '=': lambda el: el.get(attribute) == value,
        # attribute includes value as one of a set of space separated tokens
        '~': lambda el: value in el.get(attribute, '').split(),
        # attribute starts with value
        '^': lambda el: el.get(attribute, '').startswith(value),
        # attribute ends with value
        '$': lambda el: el.get(attribute, '').endswith(value),
        # attribute contains value
        '*': lambda el: value in el.get(attribute, ''),
        # attribute is either exactly value or starts with value-
        '|': lambda el: el.get(attribute, '') == value \
            or el.get(attribute, '').startswith('%s-' % value),
    }.get(operator, lambda el: el.has_key(attribute))

def select(soup, selector):
    """
    soup should be a BeautifulSoup instance; selector is a CSS selector
    specifying the elements you want to retrieve.
    """
    tokens = selector.split()
    current_context = [soup]
    for token in tokens:
        m = attribselect_re.match(token)
        if m:
            # Attribute selector
            tag, attribute, operator, value = m.groups()
            if not tag:
                tag = True
            checker = attribute_checker(operator, attribute, value)
            found = []
            for context in current_context:
                found.extend([el for el in context.findAll(tag) if checker(el)])
            current_context = found
            continue
        if '#' in token:
            # ID selector
            tag, id = token.split('#', 1)
            if not tag:
                tag = True
            el = current_context[0].find(tag, {'id': id})
            if not el:
                return [] # No match
            current_context = [el]
            continue
        if '.' in token:
            # Class selector
            tag, klass = token.split('.', 1)
            if not tag:
                tag = True
            found = []
            for context in current_context:
                found.extend(
                    context.findAll(tag,
                        {'class': lambda attr: attr and klass in attr.split()}
                    )
                )
            current_context = found
            continue
        if token == '*':
            # Star selector
            found = []
            for context in current_context:
                found.extend(context.findAll(True))
            current_context = found
            continue
        # Here we should just have a regular tag
        if not tag_re.match(token):
            return []
        found = []
        for context in current_context:
            found.extend(context.findAll(token))
        current_context = found
    return current_context

def monkeypatch(BeautifulSoupClass=None):
    """
    If you don't explicitly state the class to patch, defaults to the most
    common import location for BeautifulSoup.
    """
    if not BeautifulSoupClass:
        from BeautifulSoup import BeautifulSoup as BeautifulSoupClass
    BeautifulSoupClass.findSelect = select

def unmonkeypatch(BeautifulSoupClass=None):
    if not BeautifulSoupClass:
        from BeautifulSoup import BeautifulSoup as BeautifulSoupClass
    delattr(BeautifulSoupClass, 'findSelect')

def cssFind(html, selector):
    """
    Parse ``html`` with class:`BeautifulSoup.BeautifulSoup` and use
    :func:`.select` on the result.

    Added by Espen A. Kristiansen to make it even easier to use for testing.
    """
    soup = BeautifulSoup(html)
    return select(soup, selector)

def cssGet(html, selector):
    """
    Same as :func:`.cssFind`, but returns the first match.

    Added by Espen A. Kristiansen to make it even easier to use for testing.
    """
    try:
        return cssFind(html, selector)[0]
    except IndexError as e:
        raise IndexError('Could not find {}.'.format(selector))

def cssExists(html, selector):
    """
    Same as :func:`.cssFind`, but returns ``True`` if the selector matches at least one item.

    Added by Espen A. Kristiansen to make it even easier to use for testing.
    """
    matches = cssFind(html, selector)
    return bool(len(matches))

def prettyhtml(html):
    return BeautifulSoup(html).prettify()

def normalize_whitespace(html):
    return re.sub('\s+', ' ', html, re.MULTILINE).strip()<|endoftext|>


<commit_before>#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Optimizer Factory."""

__all__ = ["DistributedAdam"]
import paddle.fluid as fluid
from paddle.fluid.distribute_lookup_table import find_distributed_lookup_table
from paddle.fluid.distribute_lookup_table import find_distributed_lookup_table_inputs
from paddle.fluid.distribute_lookup_table import find_distributed_lookup_table_outputs
from google.protobuf import text_format
from collections import OrderedDict
from .node import DownpourWorker, DownpourServer
from . import ps_pb2 as pslib

class DistributedOptimizerImplBase(object):
    """
    DistributedOptimizerImplBase
    base class of optimizers
    """

    def __init__(self, optimizer):
        self._optimizer = optimizer
        self._learning_rate = optimizer._learning_rate
        self._regularization = optimizer.regularization

    def minimize(self,
                 losses,
                 startup_program=None,
                 parameter_list=None,
                 no_grad_set=None):
        """
        Args:
            losses(Variable): loss variable defined by user
            startup_program(Program): startup program that defined by user
            parameter_list(str list): parameter names defined by users
            no_grad_set(set): a set of variables that is defined by users
                so that these variables do not need gradient computation
        """
        pass

class DistributedAdam(DistributedOptimizerImplBase):
    """
    DistributedAdam
    adam optimizer in distributed training
    """

    def __init__(self, optimizer):
        # todo(guru4elephant): add more optimizers here as argument
        # todo(guru4elephant): make learning_rate as a variable
        super(DistributedAdam, self).__init__(optimizer)
        self._window = 1
        self.type = "downpour"
        self.data_norm_name = [
            ".batch_size", ".batch_square_sum", ".batch_sum",
            ".batch_size@GRAD", ".batch_square_sum@GRAD", ".batch_sum@GRAD"
        ]

    def _find_distributed_lookup_table_inputs(self, program, table_names):
        """
        Find input variable of distribute lookup table in program.
        We could support multi-distribute table now.
        Args:
            program(Program): given program, locate distributed lookup table
            table_name(str): given table names that is found beforehand
        Returns:
            inputs
        """
        local_vars = program.current_block().vars
        inputs_dict = dict()
        for table_name in table_names:
            inputs_dict[table_name] = []

        for op in program.global_block().ops:
            if op.type == "lookup_table":
                if op.input("W")[0] in table_names:
                    inputs_dict[op.input("W")[0]].extend(
                        [local_vars[name] for name in op.input("Ids")])
        return inputs_dict

    def _find_distributed_lookup_table_outputs(self, program, table_names):
        """
        Find output variable of distribute lookup table in program.
        We could support multi-distribute table now.
        Args:
            programs(Program): given program, locate distributed lookup table
            table_name(str): given table name that is found beforehand
        Returns:
            outputs
        """
        local_vars = program.current_block().vars
        outputs_dict = dict()
        for table_name in table_names:
            outputs_dict[table_name] = []

        for op in program.global_block().ops:
            if op.type == "lookup_table":
                if op.input("W")[0] in table_names:
                    outputs_dict[op.input("W")[0]].extend(
                        [local_vars[name] for name in op.output("Out")])
        return outputs_dict

    def _find_distributed_lookup_table_grads(self, program, table_names):
        local_vars = program.current_block().vars
        grads_dict = dict()
        for table_name in table_names:
            grads_dict[table_name] = []

        for op in program.global_block().ops:
            if op.type == "lookup_table_grad" and op.input("W")[
                    0] in table_names:
                grads_dict[op.input("W")[0]].extend(
                    [local_vars[name] for name in op.input("Out@GRAD")])
        return grads_dict

    def _find_multi_distributed_lookup_table(self, losses):
        """
        find multi-sparse-table
        """
        table_names = set()
        for loss in losses:
            for op in loss.block.program.global_block().ops:
                if op.type == "lookup_table":
                    if op.attr('is_distributed') is True:
                        table_name = op.input("W")[0]
                        table_names.add(table_name)
        return list(table_names)

    def _minimize(self,
                  losses,
                  startup_program=None,
                  parameter_list=None,
                  no_grad_set=None,
                  strategy={}):
        """
        DownpounSGD is a distributed optimizer so
        that user can call minimize to generate backward
        operators and optimization operators within minmize function
        Args:
            loss(Variable): loss variable defined by user
            startup_program(Program): startup program that defined by user
            parameter_list(str list): parameter names defined by users
            no_grad_set(set): a set of variables that is defined by users
            so that these variables do not need gradient computation
            strategy(dict): user-defined properties
        Returns:
            [optimize_ops, grads_and_weights]
        """
        # sparse table names of each program
        prog_id_to_sparse_table = OrderedDict()
        # inputs_dict and outputs_dict of sparse tables of each program
        prog_id_to_inputs_dict = OrderedDict()
        prog_id_to_outputs_dict = OrderedDict()
        # related to PSParameter
        ps_param = pslib.PSParameter()
        # related to ServerParameter
        server = DownpourServer()
        # program to worker (related to DownpourTrainerParameter)
        prog_id_to_worker = OrderedDict()
        # param_grads of each program
        prog_id_to_param_grads = OrderedDict()
        # sparse_grads of each program
        prog_id_to_sparse_grads = OrderedDict()

        sparse_table_to_index = OrderedDict()
        sparse_table_index = 0
        for loss in losses:
            sparse_table = self._find_multi_distributed_lookup_table([loss])
            prog_id = str(id(loss.block.program))
            prog_id_to_sparse_table[prog_id] = sparse_table

            # get sparse_table_to_index
            for tn in sparse_table:
                if sparse_table_to_index.get(tn) is None:
                    sparse_table_to_index[tn] = sparse_table_index
                    sparse_table_index += 1

            # get inputs_dict
            inputs_dict = self._find_distributed_lookup_table_inputs(
                loss.block.program, sparse_table)
            prog_id_to_inputs_dict[prog_id] = inputs_dict
            # get outputs_dict
            outputs_dict = self._find_distributed_lookup_table_outputs(
                loss.block.program, sparse_table)
            prog_id_to_outputs_dict[prog_id] = outputs_dict

            prog_id_to_worker[prog_id] = DownpourWorker(self._window)

            # param_grads of program
            params_grads = sorted(
                fluid.backward.append_backward(loss, parameter_list,
                                               no_grad_set),
                key=lambda x: x[0].name)
            prog_id_to_param_grads[prog_id] = params_grads

            grads_dict = self._find_distributed_lookup_table_grads(
                loss.block.program, sparse_table)
            prog_id_to_sparse_grads[prog_id] = grads_dict

        # if user specify a fleet_desc.prototxt file, then load the file
        # instead of creating default fleet_desc.prototxt.
        # user can specify server_param or trainer_param or fs_client_param.
        if strategy.get("fleet_desc_file") is not None:
            fleet_desc_file = strategy["fleet_desc_file"]
            with open(fleet_desc_file) as f:
                text_format.Merge(f.read(), ps_param)
            server.get_desc().CopyFrom(ps_param.server_param)
            if len(ps_param.trainer_param) == 1:
                for k in prog_id_to_worker:
                    prog_id_to_worker[k].get_desc().CopyFrom(
                        ps_param.trainer_param[0])
            else:
                if len(ps_param.trainer_param) != len(prog_id_to_worker):
                    raise ValueError(
                        "trainer param size != program size, %s vs %s" %
                        (len(ps_param.trainer_param), len(prog_id_to_worker)))
                idx = 0
                # prog_id_to_worker is OrderedDict
                for k in prog_id_to_worker:
                    prog_id_to_worker[k].get_desc().CopyFrom(
                        ps_param.trainer_param[idx])
                    idx += 1

        # ServerParameter add all sparse tables
        for tn in sparse_table_to_index:
            sparse_table_index = sparse_table_to_index[tn]
            if strategy.get(tn) is not None:
                server.add_sparse_table(sparse_table_index, strategy[tn])
            else:
                server.add_sparse_table(sparse_table_index, None)

        # each DownpourTrainerParameter add its own sparse tables
        for loss in losses:
            prog_id = str(id(loss.block.program))
            worker = prog_id_to_worker[prog_id]
            inputs_dict = prog_id_to_inputs_dict[prog_id]
            outputs_dict = prog_id_to_outputs_dict[prog_id]
            for tn in prog_id_to_sparse_table[prog_id]:
                sparse_table_index = sparse_table_to_index[tn]
                grads_dict = prog_id_to_sparse_grads[prog_id]
                worker.add_sparse_table(sparse_table_index, inputs_dict[tn],
                                        outputs_dict[tn], grads_dict[tn])

        dense_start_table_id = len(sparse_table_to_index)
        dense_table_index = len(sparse_table_to_index)
        program_configs = {}
        # ServerParameter add all dense tables
        # each DownpourTrainerParameter add its own dense tables
        for loss_index in range(len(losses)):
            program_id = str(id(losses[loss_index].block.program))
            worker = prog_id_to_worker[program_id]
            sparse_table_names = prog_id_to_sparse_table[program_id]
            sparse_table_index = \
                [sparse_table_to_index[i] for i in sparse_table_names]

            program_configs[program_id] = {
                "pull_sparse": [t_index for t_index in sparse_table_index],
                "push_sparse": [t_index for t_index in sparse_table_index]
            }

            params_grads = prog_id_to_param_grads[program_id]
            params = []
            grads = []
            data_norm_params = []
            data_norm_grads = []
            for i in params_grads:
                is_data_norm_data = False
                for data_norm_name in self.data_norm_name:
                    if i[0].name.endswith(data_norm_name):
                        is_data_norm_data = True
                        data_norm_params.append(i[0])
                if not is_data_norm_data:
                    params.append(i[0])

            for i in params_grads:
                is_data_norm_data = False
                for data_norm_grad in self.data_norm_name:
                    if i[0].name.endswith(data_norm_grad):
                        is_data_norm_data = True
                        data_norm_grads.append(i[1])
                if not is_data_norm_data:
                    grads.append(i[1])

            if strategy.get('dense_table') is not None:
                server.add_dense_table(dense_table_index, params, grads,
                                       strategy['dense_table'],
                                       sparse_table_names)
            else:
                server.add_dense_table(dense_table_index, params, grads, None,
                                       sparse_table_names)
            worker.add_dense_table(dense_table_index, self._learning_rate,
                                   params, grads, dense_start_table_id,
                                   sparse_table_names)
            program_configs[program_id]["pull_dense"] = [dense_table_index]
            program_configs[program_id]["push_dense"] = [dense_table_index]
            if len(data_norm_params) != 0 and len(data_norm_grads) != 0:
                dense_table_index += 1
                if strategy.get('datanorm_table') is not None:
                    server.add_data_norm_table(
                        dense_table_index, self._learning_rate,
                        data_norm_params, data_norm_grads,
                        strategy['datanorm_table'], sparse_table_names)
                else:
                    server.add_data_norm_table(
                        dense_table_index, self._learning_rate,
                        data_norm_params, data_norm_grads, None,
                        sparse_table_names)

                worker.add_dense_table(dense_table_index, self._learning_rate,
                                       data_norm_params, data_norm_grads,
                                       dense_start_table_id, sparse_table_names)
                program_configs[program_id]["pull_dense"].extend(
                    [dense_table_index])
                program_configs[program_id]["push_dense"].extend(
                    [dense_table_index])
            dense_table_index += 1

            # Todo(guru4elephant): figure out how to support more sparse parameters
            # currently only support lookup_table
            worker_skipped_ops = ["lookup_table", "lookup_table_grad"]
            if len(worker.get_desc().skip_op) == 0:
                worker.get_desc().skip_op.extend(worker_skipped_ops)

        ps_param.server_param.CopyFrom(server.get_desc())
        # prog_id_to_worker is OrderedDict
        if len(ps_param.trainer_param) == 0:
            for k in prog_id_to_worker:
                tp = ps_param.trainer_param.add()
                tp.CopyFrom(prog_id_to_worker[k].get_desc())

        opt_info = {}
        opt_info["program_id_to_worker"] = prog_id_to_worker
        opt_info["program_configs"] = program_configs
        opt_info["trainer"] = "DistMultiTrainer"
        opt_info["device_worker"] = "DownpourSGD"
        opt_info["optimizer"] = "DownpourSGD"
        opt_info["fleet_desc"] = ps_param
        opt_info["worker_skipped_ops"] = worker_skipped_ops
        opt_info["use_cvm"] = strategy.get("use_cvm", False)
        opt_info["stat_var_names"] = strategy.get("stat_var_names", [])
        opt_info["scale_datanorm"] = strategy.get("scale_datanorm", -1)
        opt_info["check_nan_var_names"] = strategy.get("check_nan_var_names",
                                                       [])
        opt_info["dump_slot"] = False
        opt_info["dump_converter"] = ""
        opt_info["dump_fields"] = strategy.get("dump_fields", [])
        opt_info["dump_file_num"] = strategy.get("dump_file_num", 16)
        opt_info["dump_fields_path"] = strategy.get("dump_fields_path", "")
        opt_info["dump_param"] = strategy.get("dump_param", [])
        if server._server.downpour_server_param.downpour_table_param[
                0].accessor.accessor_class == "DownpourCtrAccessor":
            opt_info["dump_slot"] = True
        opt_info["adjust_ins_weight"] = strategy.get("adjust_ins_weight", {})

        for loss in losses:
            loss.block.program._fleet_opt = opt_info

        param_grads_list = []
        for loss in losses:
            prog_id = str(id(loss.block.program))
            param_grads_list.append(prog_id_to_param_grads[prog_id])
        return None, param_grads_list, opt_info
<commit_msg>find lookup table in order (#20932)<commit_after>#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Optimizer Factory."""

__all__ = ["DistributedAdam"]
import paddle.fluid as fluid
from paddle.fluid.distribute_lookup_table import find_distributed_lookup_table
from paddle.fluid.distribute_lookup_table import find_distributed_lookup_table_inputs
from paddle.fluid.distribute_lookup_table import find_distributed_lookup_table_outputs
from google.protobuf import text_format
from collections import OrderedDict
from .node import DownpourWorker, DownpourServer
from . import ps_pb2 as pslib

class DistributedOptimizerImplBase(object):
    """
    DistributedOptimizerImplBase
    base class of optimizers
    """

    def __init__(self, optimizer):
        self._optimizer = optimizer
        self._learning_rate = optimizer._learning_rate
        self._regularization = optimizer.regularization

    def minimize(self,
                 losses,
                 startup_program=None,
                 parameter_list=None,
                 no_grad_set=None):
        """
        Args:
            losses(Variable): loss variable defined by user
            startup_program(Program): startup program that defined by user
            parameter_list(str list): parameter names defined by users
            no_grad_set(set): a set of variables that is defined by users
                so that these variables do not need gradient computation
        """
        pass

class DistributedAdam(DistributedOptimizerImplBase):
    """
    DistributedAdam
    adam optimizer in distributed training
    """

    def __init__(self, optimizer):
        # todo(guru4elephant): add more optimizers here as argument
        # todo(guru4elephant): make learning_rate as a variable
        super(DistributedAdam, self).__init__(optimizer)
        self._window = 1
        self.type = "downpour"
        self.data_norm_name = [
            ".batch_size", ".batch_square_sum", ".batch_sum",
            ".batch_size@GRAD", ".batch_square_sum@GRAD", ".batch_sum@GRAD"
        ]

    def _find_distributed_lookup_table_inputs(self, program, table_names):
        """
        Find input variable of distribute lookup table in program.
        We could support multi-distribute table now.
        Args:
            program(Program): given program, locate distributed lookup table
            table_name(str): given table names that is found beforehand
        Returns:
            inputs
        """
        local_vars = program.current_block().vars
        inputs_dict = dict()
        for table_name in table_names:
            inputs_dict[table_name] = []

        for op in program.global_block().ops:
            if op.type == "lookup_table":
                if op.input("W")[0] in table_names:
                    inputs_dict[op.input("W")[0]].extend(
                        [local_vars[name] for name in op.input("Ids")])
        return inputs_dict

    def _find_distributed_lookup_table_outputs(self, program, table_names):
        """
        Find output variable of distribute lookup table in program.
        We could support multi-distribute table now.
        Args:
            programs(Program): given program, locate distributed lookup table
            table_name(str): given table name that is found beforehand
        Returns:
            outputs
        """
        local_vars = program.current_block().vars
        outputs_dict = dict()
        for table_name in table_names:
            outputs_dict[table_name] = []

        for op in program.global_block().ops:
            if op.type == "lookup_table":
                if op.input("W")[0] in table_names:
                    outputs_dict[op.input("W")[0]].extend(
                        [local_vars[name] for name in op.output("Out")])
        return outputs_dict

    def _find_distributed_lookup_table_grads(self, program, table_names):
        local_vars = program.current_block().vars
        grads_dict = dict()
        for table_name in table_names:
            grads_dict[table_name] = []

        for op in program.global_block().ops:
            if op.type == "lookup_table_grad" and op.input("W")[
                    0] in table_names:
                grads_dict[op.input("W")[0]].extend(
                    [local_vars[name] for name in op.input("Out@GRAD")])
        return grads_dict

    def _find_multi_distributed_lookup_table(self, losses):
        """
        find multi-sparse-table
        """
        table_names = set()
        cnt = 0
        tmp_list = []
        ret_list = []
        for loss in losses:
            for op in loss.block.program.global_block().ops:
                if op.type == "lookup_table":
                    if op.attr('is_distributed') is True:
                        table_name = op.input("W")[0]
                        if table_name not in table_names:
                            table_names.add(table_name)
                            tmp_list.append([table_name, cnt])
                            cnt += 1
        tmp_list.sort(key=lambda k: k[1])
        for x in tmp_list:
            ret_list.append(x[0])
        return ret_list

    def _minimize(self,
                  losses,
                  startup_program=None,
                  parameter_list=None,
                  no_grad_set=None,
                  strategy={}):
        """
        DownpounSGD is a distributed optimizer so
        that user can call minimize to generate backward
        operators and optimization operators within minmize function
        Args:
            loss(Variable): loss variable defined by user
            startup_program(Program): startup program that defined by user
            parameter_list(str list): parameter names defined by users
            no_grad_set(set): a set of variables that is defined by users
            so that these variables do not need gradient computation
            strategy(dict): user-defined properties
        Returns:
            [optimize_ops, grads_and_weights]
        """
        # sparse table names of each program
        prog_id_to_sparse_table = OrderedDict()
        # inputs_dict and outputs_dict of sparse tables of each program
        prog_id_to_inputs_dict = OrderedDict()
        prog_id_to_outputs_dict = OrderedDict()
        # related to PSParameter
        ps_param = pslib.PSParameter()
        # related to ServerParameter
        server = DownpourServer()
        # program to worker (related to DownpourTrainerParameter)
        prog_id_to_worker = OrderedDict()
        # param_grads of each program
        prog_id_to_param_grads = OrderedDict()
        # sparse_grads of each program
        prog_id_to_sparse_grads = OrderedDict()

        sparse_table_to_index = OrderedDict()
        sparse_table_index = 0
        for loss in losses:
            sparse_table = self._find_multi_distributed_lookup_table([loss])
            prog_id = str(id(loss.block.program))
            prog_id_to_sparse_table[prog_id] = sparse_table

            # get sparse_table_to_index
            for tn in sparse_table:
                if sparse_table_to_index.get(tn) is None:
                    sparse_table_to_index[tn] = sparse_table_index
                    sparse_table_index += 1

            # get inputs_dict
            inputs_dict = self._find_distributed_lookup_table_inputs(
                loss.block.program, sparse_table)
            prog_id_to_inputs_dict[prog_id] = inputs_dict
            # get outputs_dict
            outputs_dict = self._find_distributed_lookup_table_outputs(
                loss.block.program, sparse_table)
            prog_id_to_outputs_dict[prog_id] = outputs_dict

            prog_id_to_worker[prog_id] = DownpourWorker(self._window)

            # param_grads of program
            params_grads = sorted(
                fluid.backward.append_backward(loss, parameter_list,
                                               no_grad_set),
                key=lambda x: x[0].name)
            prog_id_to_param_grads[prog_id] = params_grads

            grads_dict = self._find_distributed_lookup_table_grads(
                loss.block.program, sparse_table)
            prog_id_to_sparse_grads[prog_id] = grads_dict

        # if user specify a fleet_desc.prototxt file, then load the file
        # instead of creating default fleet_desc.prototxt.
        # user can specify server_param or trainer_param or fs_client_param.
        if strategy.get("fleet_desc_file") is not None:
            fleet_desc_file = strategy["fleet_desc_file"]
            with open(fleet_desc_file) as f:
                text_format.Merge(f.read(), ps_param)
            server.get_desc().CopyFrom(ps_param.server_param)
            if len(ps_param.trainer_param) == 1:
                for k in prog_id_to_worker:
                    prog_id_to_worker[k].get_desc().CopyFrom(
                        ps_param.trainer_param[0])
            else:
                if len(ps_param.trainer_param) != len(prog_id_to_worker):
                    raise ValueError(
                        "trainer param size != program size, %s vs %s" %
                        (len(ps_param.trainer_param), len(prog_id_to_worker)))
                idx = 0
                # prog_id_to_worker is OrderedDict
                for k in prog_id_to_worker:
                    prog_id_to_worker[k].get_desc().CopyFrom(
                        ps_param.trainer_param[idx])
                    idx += 1

        # ServerParameter add all sparse tables
        for tn in sparse_table_to_index:
            sparse_table_index = sparse_table_to_index[tn]
            if strategy.get(tn) is not None:
                server.add_sparse_table(sparse_table_index, strategy[tn])
            else:
                server.add_sparse_table(sparse_table_index, None)

        # each DownpourTrainerParameter add its own sparse tables
        for loss in losses:
            prog_id = str(id(loss.block.program))
            worker = prog_id_to_worker[prog_id]
            inputs_dict = prog_id_to_inputs_dict[prog_id]
            outputs_dict = prog_id_to_outputs_dict[prog_id]
            for tn in prog_id_to_sparse_table[prog_id]:
                sparse_table_index = sparse_table_to_index[tn]
                grads_dict = prog_id_to_sparse_grads[prog_id]
                worker.add_sparse_table(sparse_table_index, inputs_dict[tn],
                                        outputs_dict[tn], grads_dict[tn])

        dense_start_table_id = len(sparse_table_to_index)
        dense_table_index = len(sparse_table_to_index)
        program_configs = {}
        # ServerParameter add all dense tables
        # each DownpourTrainerParameter add its own dense tables
        for loss_index in range(len(losses)):
            program_id = str(id(losses[loss_index].block.program))
            worker = prog_id_to_worker[program_id]
            sparse_table_names = prog_id_to_sparse_table[program_id]
            sparse_table_index = \
                [sparse_table_to_index[i] for i in sparse_table_names]

            program_configs[program_id] = {
                "pull_sparse": [t_index for t_index in sparse_table_index],
                "push_sparse": [t_index for t_index in sparse_table_index]
            }

            params_grads = prog_id_to_param_grads[program_id]
            params = []
            grads = []
            data_norm_params = []
            data_norm_grads = []
            for i in params_grads:
                is_data_norm_data = False
                for data_norm_name in self.data_norm_name:
                    if i[0].name.endswith(data_norm_name):
                        is_data_norm_data = True
                        data_norm_params.append(i[0])
                if not is_data_norm_data:
                    params.append(i[0])

            for i in params_grads:
                is_data_norm_data = False
                for data_norm_grad in self.data_norm_name:
                    if i[0].name.endswith(data_norm_grad):
                        is_data_norm_data = True
                        data_norm_grads.append(i[1])
                if not is_data_norm_data:
                    grads.append(i[1])

            if strategy.get('dense_table') is not None:
                server.add_dense_table(dense_table_index, params, grads,
                                       strategy['dense_table'],
                                       sparse_table_names)
            else:
                server.add_dense_table(dense_table_index, params, grads, None,
                                       sparse_table_names)
            worker.add_dense_table(dense_table_index, self._learning_rate,
                                   params, grads, dense_start_table_id,
                                   sparse_table_names)
            program_configs[program_id]["pull_dense"] = [dense_table_index]
            program_configs[program_id]["push_dense"] = [dense_table_index]
            if len(data_norm_params) != 0 and len(data_norm_grads) != 0:
                dense_table_index += 1
                if strategy.get('datanorm_table') is not None:
                    server.add_data_norm_table(
                        dense_table_index, self._learning_rate,
                        data_norm_params, data_norm_grads,
                        strategy['datanorm_table'], sparse_table_names)
                else:
                    server.add_data_norm_table(
                        dense_table_index, self._learning_rate,
                        data_norm_params, data_norm_grads, None,
                        sparse_table_names)

                worker.add_dense_table(dense_table_index, self._learning_rate,
                                       data_norm_params, data_norm_grads,
                                       dense_start_table_id, sparse_table_names)
                program_configs[program_id]["pull_dense"].extend(
                    [dense_table_index])
                program_configs[program_id]["push_dense"].extend(
                    [dense_table_index])
            dense_table_index += 1

            # Todo(guru4elephant): figure out how to support more sparse parameters
            # currently only support lookup_table
            worker_skipped_ops = ["lookup_table", "lookup_table_grad"]
            if len(worker.get_desc().skip_op) == 0:
                worker.get_desc().skip_op.extend(worker_skipped_ops)

        ps_param.server_param.CopyFrom(server.get_desc())
        # prog_id_to_worker is OrderedDict
        if len(ps_param.trainer_param) == 0:
            for k in prog_id_to_worker:
                tp = ps_param.trainer_param.add()
                tp.CopyFrom(prog_id_to_worker[k].get_desc())

        opt_info = {}
        opt_info["program_id_to_worker"] = prog_id_to_worker
        opt_info["program_configs"] = program_configs
        opt_info["trainer"] = "DistMultiTrainer"
        opt_info["device_worker"] = "DownpourSGD"
        opt_info["optimizer"] = "DownpourSGD"
        opt_info["fleet_desc"] = ps_param
        opt_info["worker_skipped_ops"] = worker_skipped_ops
        opt_info["use_cvm"] = strategy.get("use_cvm", False)
        opt_info["stat_var_names"] = strategy.get("stat_var_names", [])
        opt_info["scale_datanorm"] = strategy.get("scale_datanorm", -1)
        opt_info["check_nan_var_names"] = strategy.get("check_nan_var_names",
                                                       [])
        opt_info["dump_slot"] = False
        opt_info["dump_converter"] = ""
        opt_info["dump_fields"] = strategy.get("dump_fields", [])
        opt_info["dump_file_num"] = strategy.get("dump_file_num", 16)
        opt_info["dump_fields_path"] = strategy.get("dump_fields_path", "")
        opt_info["dump_param"] = strategy.get("dump_param", [])
        if server._server.downpour_server_param.downpour_table_param[
                0].accessor.accessor_class == "DownpourCtrAccessor":
            opt_info["dump_slot"] = True
        opt_info["adjust_ins_weight"] = strategy.get("adjust_ins_weight", {})

        for loss in losses:
            loss.block.program._fleet_opt = opt_info

        param_grads_list = []
        for loss in losses:
            prog_id = str(id(loss.block.program))
            param_grads_list.append(prog_id_to_param_grads[prog_id])
        return None, param_grads_list, opt_info
<|endoftext|>


<commit_before>import pandas as pd
import numpy

from bioinput import Bioinput

class Biomeasure():
    """

    """

    def __init__(self, threshold=0.9):
        self.__hr_rawdata = pd.DataFrame(numpy.empty((10, 3))*numpy.nan, columns=['HeartRate', 'B/T', 'time'])
        self.__threshold = threshold
        input = Bioinput()
        self.ecg_file = input.file

    def thresholdhr(self):
        print(self.__threshold)
        print(self.__hr_rawdata)
        print(self.ecg_file)

    def change_threshold(self, threshold):
        self.__init__(threshold)

if __name__ == "__main__":
    hr_measure = Biomeasure()
    hr_measure.change_threshold(0.5)
    hr_measure.thresholdhr()

<commit_msg>condensed rhythm detection into single method in Biomeasure class; added methods to change brady & tachycardia thresholds<commit_after>import pandas as pd
import numpy

from bioinput import Bioinput

class Biomeasure():
    """

    """

    def __init__(self, threshold=0.9, thr_brady=50, thr_tachy=140):
        self.__hr_rawdata = pd.DataFrame(numpy.empty((10, 3))*numpy.nan, columns=['HeartRate', 'B/T', 'time'])
        self.__threshold = threshold
        self.__thr_brady = thr_brady
        self.__thr_tachy = thr_tachy
        self.hr = pd.DataFrame(numpy.empty((10, 3))*numpy.nan, columns=['HeartRate', 'B/T', 'time']) # need to create from others' work
        input = Bioinput()
        self.ecg_file = input.file

    def thresholdhr(self):
        print(self.__threshold)
        print(self.__hr_rawdata)
        print(self.ecg_file)

    def change_threshold(self, threshold):
        self.__init__(threshold)

    def change_brady_thresohld(self, brady_threshold):
        self.__init__(thr_brady=brady_threshold)

    def change_tachy_threshold(self, tachy_threshold):
        self.__init__(thr_tachy=tachy_threshold)

    def rhythmdetector(self):
        """Detects bradycardia & tachycardia based on threshold input and writes instances to hr DataFrame
        :param hr: DataFrame with interval heartrate, interval rhythm, and interval time demarcation
        """

        bradycount = 0
        tachycount = 0

        for x in range(0, len(self.hr)):
            if self.hr['HeartRate'][x] < self.__thr_brady:
                bradycount += 1
                self.hr.at[x, 'B/T'] = 'Bradycardia Detected'
            elif self.hr['HeartRate'][x] > self.__thr_tachy:
                tachycount += 1
                self.hr.at[x, 'B/T'] = 'Tachycardia Detected'
            else
                self.hr.at[x, 'B.T'] = 'Healthy... for now'

if __name__ == "__main__":
    hr_measure = Biomeasure()
    hr_measure.change_threshold(0.5)
    hr_measure.thresholdhr()

<|endoftext|>


<commit_before>
from colorama import Style
import dbt.events.functions as this  # don't worry I hate it too.
from dbt.events.types import Cli, Event, File, ShowException
import dbt.flags as flags
# TODO this will need to move eventually
from dbt.logger import SECRET_ENV_PREFIX, make_log_dir_if_missing
import json
import logging
from logging import Logger
from logging.handlers import WatchedFileHandler
import os
from typing import List

# create the global file logger with no configuration
global FILE_LOG
FILE_LOG = logging.getLogger('default_file')

# set up logger to go to stdout with defaults
# setup_event_logger will be called once args have been parsed
global STDOUT_LOG
STDOUT_LOG = logging.getLogger('default_stdout')
STDOUT_LOG.setLevel(logging.INFO)
stdout_handler = logging.StreamHandler()
stdout_handler.setLevel(logging.INFO)
STDOUT_LOG.addHandler(stdout_handler)
global color
format_color = True
global json
format_json = False

def setup_event_logger(log_path):
    make_log_dir_if_missing(log_path)
    this.format_json = flags.LOG_FORMAT == 'json'
    # USE_COLORS can be None if the app just started and the cli flags
    # havent been applied yet
    this.format_color = True if flags.USE_COLORS else False
    # TODO this default should live somewhere better
    log_dest = os.path.join('logs', 'dbt.log')
    level = logging.DEBUG if flags.DEBUG else logging.INFO

    # overwrite the STDOUT_LOG logger with the configured one
    this.STDOUT_LOG = logging.getLogger('configured_std_out')
    this.STDOUT_LOG.setLevel(level)

    FORMAT = "%(message)s"
    stdout_passthrough_formatter = logging.Formatter(fmt=FORMAT)

    stdout_handler = logging.StreamHandler()
    stdout_handler.setFormatter(stdout_passthrough_formatter)
    stdout_handler.setLevel(level)
    this.STDOUT_LOG.addHandler(stdout_handler)

    # overwrite the FILE_LOG logger with the configured one
    this.FILE_LOG = logging.getLogger('configured_file')
    this.FILE_LOG.setLevel(logging.DEBUG)  # always debug regardless of user input

    file_passthrough_formatter = logging.Formatter(fmt=FORMAT)

    # TODO log rotation is not handled by WatchedFileHandler
    file_handler = WatchedFileHandler(filename=log_dest, encoding='utf8')
    file_handler.setFormatter(file_passthrough_formatter)
    file_handler.setLevel(logging.DEBUG)  # always debug regardless of user input
    this.FILE_LOG.addHandler(file_handler)

def env_secrets() -> List[str]:
    return [
        v for k, v in os.environ.items()
        if k.startswith(SECRET_ENV_PREFIX)
    ]

def scrub_secrets(msg: str, secrets: List[str]) -> str:
    scrubbed = msg

    for secret in secrets:
        scrubbed = scrubbed.replace(secret, "*****")

    return scrubbed

# translates an Event to a completely formatted output log_line
# json=True -> json formatting
# json=False -> text formatting
# cli=True -> cli formatting
# cli=False -> file formatting
def create_log_line(e: Event, json_fmt: bool, cli_dest: bool) -> str:
    level = e.level_tag()
    values: dict = {
        'pid': e.pid,
        'msg': '',
        'level': level if len(level) == 5 else f"{level} "
    }
    if cli_dest and isinstance(e, Cli):
        values['msg'] = scrub_secrets(e.cli_msg(), env_secrets())
    elif not cli_dest and isinstance(e, File):
        values['msg'] = scrub_secrets(e.file_msg(), env_secrets())

    if json_fmt:
        values['ts'] = e.ts.isoformat()
        log_line = json.dumps(values, sort_keys=True)
    else:
        values['ts'] = e.ts.strftime("%H:%M:%S")
        color_tag = '' if this.format_color else Style.RESET_ALL
        log_line = f"{color_tag}{values['ts']} | [ {values['level']} ] | {values['msg']}"

    return log_line

# allows for resuse of this obnoxious if else tree.
# do not use for exceptions, it doesn't pass along exc_info, stack_info, or extra
def send_to_logger(l: Logger, level_tag: str, log_line: str):
    if level_tag == 'test':
        # TODO after implmenting #3977 send to new test level
        l.debug(log_line)
    elif level_tag == 'debug':
        l.debug(log_line)
    elif level_tag == 'info':
        l.info(log_line)
    elif level_tag == 'warn':
        l.warning(log_line)
    elif level_tag == 'error':
        l.error(log_line)
    else:
        raise AssertionError(
            f"While attempting to log {log_line}, encountered the unhandled level: {level_tag}"
        )

def send_exc_to_logger(
    l: Logger,
    level_tag: str,
    log_line: str,
    exc_info=True,
    stack_info=False,
    extra=False
):
    if level_tag == 'test':
        # TODO after implmenting #3977 send to new test level
        l.debug(
            log_line,
            exc_info=exc_info,
            stack_info=stack_info,
            extra=extra
        )
    elif level_tag == 'debug':
        l.debug(
            log_line,
            exc_info=exc_info,
            stack_info=stack_info,
            extra=extra
        )
    elif level_tag == 'info':
        l.info(
            log_line,
            exc_info=exc_info,
            stack_info=stack_info,
            extra=extra
        )
    elif level_tag == 'warn':
        l.warning(
            log_line,
            exc_info=exc_info,
            stack_info=stack_info,
            extra=extra
        )
    elif level_tag == 'error':
        l.error(
            log_line,
            exc_info=exc_info,
            stack_info=stack_info,
            extra=extra
        )
    else:
        raise AssertionError(
            f"While attempting to log {log_line}, encountered the unhandled level: {level_tag}"
        )

# top-level method for accessing the new eventing system
# this is where all the side effects happen branched by event type
# (i.e. - mutating the event history, printing to stdout, logging
# to files, etc.)
def fire_event(e: Event) -> None:
    # TODO manage history in phase 2: EVENT_HISTORY.append(e)
    # explicitly checking the debug flag here so that potentially expensive-to-construct
    # log messages are not constructed if debug messages are never shown.

    # always logs debug level regardless of user input
    if isinstance(e, File):
        log_line = create_log_line(e, json_fmt=this.format_json, cli_dest=False)
        # doesn't send exceptions to exception logger
        send_to_logger(FILE_LOG, level_tag=e.level_tag(), log_line=log_line)

    if isinstance(e, Cli):
        if e.level_tag() == 'debug' and not flags.DEBUG:
            return  # eat the message in case it was one of the expensive ones
        log_line = create_log_line(e, json_fmt=this.format_json, cli_dest=True)
        if not isinstance(e, ShowException):
            send_to_logger(STDOUT_LOG, level_tag=e.level_tag(), log_line=log_line)
        # CliEventABC and ShowException
        else:
            send_exc_to_logger(
                STDOUT_LOG,
                level_tag=e.level_tag(),
                log_line=log_line,
                exc_info=e.exc_info,
                stack_info=e.stack_info,
                extra=e.extra
            )
<commit_msg>whitespace change<commit_after>
from colorama import Style
import dbt.events.functions as this  # don't worry I hate it too.
from dbt.events.types import Cli, Event, File, ShowException
import dbt.flags as flags
# TODO this will need to move eventually
from dbt.logger import SECRET_ENV_PREFIX, make_log_dir_if_missing
import json
import logging
from logging import Logger
from logging.handlers import WatchedFileHandler
import os
from typing import List

# create the global file logger with no configuration
global FILE_LOG
FILE_LOG = logging.getLogger('default_file')

# set up logger to go to stdout with defaults
# setup_event_logger will be called once args have been parsed
global STDOUT_LOG
STDOUT_LOG = logging.getLogger('default_stdout')
STDOUT_LOG.setLevel(logging.INFO)
stdout_handler = logging.StreamHandler()
stdout_handler.setLevel(logging.INFO)
STDOUT_LOG.addHandler(stdout_handler)
global color
format_color = True
global json
format_json = False

def setup_event_logger(log_path):
    make_log_dir_if_missing(log_path)
    this.format_json = flags.LOG_FORMAT == 'json'
    # USE_COLORS can be None if the app just started and the cli flags
    # havent been applied yet
    this.format_color = True if flags.USE_COLORS else False
    # TODO this default should live somewhere better
    log_dest = os.path.join('logs', 'dbt.log')
    level = logging.DEBUG if flags.DEBUG else logging.INFO

    # overwrite the STDOUT_LOG logger with the configured one
    this.STDOUT_LOG = logging.getLogger('configured_std_out')
    this.STDOUT_LOG.setLevel(level)

    FORMAT = "%(message)s"
    stdout_passthrough_formatter = logging.Formatter(fmt=FORMAT)

    stdout_handler = logging.StreamHandler()
    stdout_handler.setFormatter(stdout_passthrough_formatter)
    stdout_handler.setLevel(level)
    this.STDOUT_LOG.addHandler(stdout_handler)

    # overwrite the FILE_LOG logger with the configured one
    this.FILE_LOG = logging.getLogger('configured_file')
    this.FILE_LOG.setLevel(logging.DEBUG)  # always debug regardless of user input

    file_passthrough_formatter = logging.Formatter(fmt=FORMAT)

    # TODO log rotation is not handled by WatchedFileHandler
    file_handler = WatchedFileHandler(filename=log_dest, encoding='utf8')
    file_handler.setFormatter(file_passthrough_formatter)
    file_handler.setLevel(logging.DEBUG)  # always debug regardless of user input
    this.FILE_LOG.addHandler(file_handler)

def env_secrets() -> List[str]:
    return [
        v for k, v in os.environ.items()
        if k.startswith(SECRET_ENV_PREFIX)
    ]

def scrub_secrets(msg: str, secrets: List[str]) -> str:
    scrubbed = msg

    for secret in secrets:
        scrubbed = scrubbed.replace(secret, "*****")

    return scrubbed

# translates an Event to a completely formatted output log_line
# json=True -> json formatting
# json=False -> text formatting
# cli=True -> cli formatting
# cli=False -> file formatting
def create_log_line(e: Event, json_fmt: bool, cli_dest: bool) -> str:
    level = e.level_tag()
    values: dict = {
        'pid': e.pid,
        'msg': '',
        'level': level if len(level) == 5 else f"{level} "
    }
    if cli_dest and isinstance(e, Cli):
        values['msg'] = scrub_secrets(e.cli_msg(), env_secrets())
    elif not cli_dest and isinstance(e, File):
        values['msg'] = scrub_secrets(e.file_msg(), env_secrets())

    if json_fmt:
        values['ts'] = e.ts.isoformat()
        log_line = json.dumps(values, sort_keys=True)
    else:
        values['ts'] = e.ts.strftime("%H:%M:%S")
        color_tag = '' if this.format_color else Style.RESET_ALL
        log_line = f"{color_tag}{values['ts']} | [ {values['level']} ] | {values['msg']}"

    return log_line

# allows for resuse of this obnoxious if else tree.
# do not use for exceptions, it doesn't pass along exc_info, stack_info, or extra
def send_to_logger(l: Logger, level_tag: str, log_line: str):
    if level_tag == 'test':
        # TODO after implmenting #3977 send to new test level
        l.debug(log_line)
    elif level_tag == 'debug':
        l.debug(log_line)
    elif level_tag == 'info':
        l.info(log_line)
    elif level_tag == 'warn':
        l.warning(log_line)
    elif level_tag == 'error':
        l.error(log_line)
    else:
        raise AssertionError(
            f"While attempting to log {log_line}, encountered the unhandled level: {level_tag}"
        )

def send_exc_to_logger(
    l: Logger,
    level_tag: str,
    log_line: str,
    exc_info=True,
    stack_info=False,
    extra=False
):
    if level_tag == 'test':
        # TODO after implmenting #3977 send to new test level
        l.debug(
            log_line,
            exc_info=exc_info,
            stack_info=stack_info,
            extra=extra
        )
    elif level_tag == 'debug':
        l.debug(
            log_line,
            exc_info=exc_info,
            stack_info=stack_info,
            extra=extra
        )
    elif level_tag == 'info':
        l.info(
            log_line,
            exc_info=exc_info,
            stack_info=stack_info,
            extra=extra
        )
    elif level_tag == 'warn':
        l.warning(
            log_line,
            exc_info=exc_info,
            stack_info=stack_info,
            extra=extra
        )
    elif level_tag == 'error':
        l.error(
            log_line,
            exc_info=exc_info,
            stack_info=stack_info,
            extra=extra
        )
    else:
        raise AssertionError(
            f"While attempting to log {log_line}, encountered the unhandled level: {level_tag}"
        )

# top-level method for accessing the new eventing system
# this is where all the side effects happen branched by event type
# (i.e. - mutating the event history, printing to stdout, logging
# to files, etc.)
def fire_event(e: Event) -> None:
    # TODO manage history in phase 2:  EVENT_HISTORY.append(e)
    # explicitly checking the debug flag here so that potentially expensive-to-construct
    # log messages are not constructed if debug messages are never shown.

    # always logs debug level regardless of user input
    if isinstance(e, File):
        log_line = create_log_line(e, json_fmt=this.format_json, cli_dest=False)
        # doesn't send exceptions to exception logger
        send_to_logger(FILE_LOG, level_tag=e.level_tag(), log_line=log_line)

    if isinstance(e, Cli):
        if e.level_tag() == 'debug' and not flags.DEBUG:
            return  # eat the message in case it was one of the expensive ones
        log_line = create_log_line(e, json_fmt=this.format_json, cli_dest=True)
        if not isinstance(e, ShowException):
            send_to_logger(STDOUT_LOG, level_tag=e.level_tag(), log_line=log_line)
        # CliEventABC and ShowException
        else:
            send_exc_to_logger(
                STDOUT_LOG,
                level_tag=e.level_tag(),
                log_line=log_line,
                exc_info=e.exc_info,
                stack_info=e.stack_info,
                extra=e.extra
            )
<|endoftext|>


<commit_before>#!/usr/bin/python3

import os
#import re
import sys
#import json
import pprint

import twitter

#import pudb
#pu.db

# ---- retweet

def retweet(topic):
    search_results = twit.search.tweets(q=topic, lang='en', result_type='popular')
    print('search_results')
    pp.pprint(search_results)

    texts = [{'id': tweet['id_str'], 'text': tweet['text'], 'retweets': tweet['retweet_count']} for tweet in search_results['statuses']]
    print(texts)
    pp.pprint(texts)

    if len(texts) > 0:
        # sort by retweet count
        texts = sorted(texts, key=lambda t: t['retweets'])
        pp.pprint(texts)

        #take most retweeted
        retweet_id = texts[0]['id']

        # retweet
        rtresult = twit.statuses.retweet(id=retweet_id)
        pp.pprint(rtresult)
    else:
        print('nothing to retweet')

#---- main

pp = pprint.PrettyPrinter(indent=4)

auth = twitter.oauth.OAuth(os.environ['tw_hmm_oauth_token'], os.environ['tw_hmm_oauth_token_secret'], os.environ['tw_hmm_consumer_key'], os.environ['tw_hmm_consumer_secret'])

twit = twitter.Twitter(auth=auth)

#---- start

retweet('#Appreciate5YearsOfZayn')
    sys.exit()

WORLD_WOE_ID = 1
AUSTRALIA_WOE_ID = 23424748

# Prefix ID with the underscore for query string parameterization.
# Without the underscore, the twitter package appends the ID value
# to the URL itself as a special case keyword argument.

world_trends = twit.trends.place(_id=WORLD_WOE_ID)
australia_trends = twit.trends.place(_id=AUSTRALIA_WOE_ID)

#print(json.dumps(world_trends, indent=4))
#print(json.dumps(australia_trends, indent=4))

world_trends_set = set([trend['name'] for trend in world_trends[0]['trends']])
australia_trends_set = set([trend['name'] for trend in australia_trends[0]['trends']])
common_trends = world_trends_set.intersection(australia_trends_set)

print('world')
pp.pprint(world_trends_set)

print('australia')
pp.pprint(australia_trends_set)

print('common')
pp.pprint(common_trends)

# tweet the first topic not in the last 24 topics tweeted
history_count = 24
history_filename = 'history.txt'

topic_history = []

if os.path.isfile(history_filename):
    with open(history_filename, 'rU') as in_file:
        topic_history = in_file.read().split('\n')

print('history')
pp.pprint(topic_history)

new_trends = common_trends.difference(set(topic_history))

print('new')
pp.pprint(new_trends)

if len(new_trends) > 0:
    # pick first
    trend = [t for t in new_trends][0]

    tweet = 'Trending in Australia and The World: ' + trend

    print(tweet)

    # add trend to history

    topic_history.append(trend)

    # save n last
    with open(history_filename, 'w') as out_file:
        out_file.write('\n'.join(topic_history[-history_count:]))

    #tweet
    result = twit.statuses.update(status=tweet)
    pp.pprint(result)

    # retweet the most popular
    retweet(trend)

<commit_msg>fix indent<commit_after>#!/usr/bin/python3

import os
#import re
import sys
#import json
import pprint

import twitter

#import pudb
#pu.db

# ---- retweet

def retweet(topic):
    search_results = twit.search.tweets(q=topic, lang='en', result_type='popular')
    print('search_results')
    pp.pprint(search_results)

    texts = [{'id': tweet['id_str'], 'text': tweet['text'], 'retweets': tweet['retweet_count']} for tweet in search_results['statuses']]
    print(texts)
    pp.pprint(texts)

    if len(texts) > 0:
        # sort by retweet count
        texts = sorted(texts, key=lambda t: t['retweets'])
        pp.pprint(texts)

        #take most retweeted
        retweet_id = texts[0]['id']

        # retweet
        rtresult = twit.statuses.retweet(id=retweet_id)
        pp.pprint(rtresult)
    else:
        print('nothing to retweet')

#---- main

pp = pprint.PrettyPrinter(indent=4)

auth = twitter.oauth.OAuth(os.environ['tw_hmm_oauth_token'], os.environ['tw_hmm_oauth_token_secret'], os.environ['tw_hmm_consumer_key'], os.environ['tw_hmm_consumer_secret'])

twit = twitter.Twitter(auth=auth)

#---- start

retweet('#Appreciate5YearsOfZayn')
sys.exit()

WORLD_WOE_ID = 1
AUSTRALIA_WOE_ID = 23424748

# Prefix ID with the underscore for query string parameterization.
# Without the underscore, the twitter package appends the ID value
# to the URL itself as a special case keyword argument.

world_trends = twit.trends.place(_id=WORLD_WOE_ID)
australia_trends = twit.trends.place(_id=AUSTRALIA_WOE_ID)

#print(json.dumps(world_trends, indent=4))
#print(json.dumps(australia_trends, indent=4))

world_trends_set = set([trend['name'] for trend in world_trends[0]['trends']])
australia_trends_set = set([trend['name'] for trend in australia_trends[0]['trends']])
common_trends = world_trends_set.intersection(australia_trends_set)

print('world')
pp.pprint(world_trends_set)

print('australia')
pp.pprint(australia_trends_set)

print('common')
pp.pprint(common_trends)

# tweet the first topic not in the last 24 topics tweeted
history_count = 24
history_filename = 'history.txt'

topic_history = []

if os.path.isfile(history_filename):
    with open(history_filename, 'rU') as in_file:
        topic_history = in_file.read().split('\n')

print('history')
pp.pprint(topic_history)

new_trends = common_trends.difference(set(topic_history))

print('new')
pp.pprint(new_trends)

if len(new_trends) > 0:
    # pick first
    trend = [t for t in new_trends][0]

    tweet = 'Trending in Australia and The World: ' + trend

    print(tweet)

    # add trend to history

    topic_history.append(trend)

    # save n last
    with open(history_filename, 'w') as out_file:
        out_file.write('\n'.join(topic_history[-history_count:]))

    #tweet
    result = twit.statuses.update(status=tweet)
    pp.pprint(result)

    # retweet the most popular
    retweet(trend)

<|endoftext|>


<commit_before># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import pytest

from wampy.errors import RemoteError
from wampy.peers.clients import Client
from wampy.roles.callee import callee

class UnreliableCallee(Client):

    @callee
    def get_foo(self, *args, **kwargs):
        raise ValueError(
            "i do not like any of your values: {}, {}".format(
                args, kwargs)
        )

@pytest.yield_fixture
def unreliable_callee(router, config_path):
    with UnreliableCallee(router=router):
        yield

def test_handle_value_error(unreliable_callee, router):
    with Client(router=router, name="caller") as client:

        with pytest.raises(RemoteError) as exc_info:
            client.rpc.get_foo(1, 2, three=3)

        exception = exc_info.value
        assert type(exception) is RemoteError

        message = str(exception)

        assert "ValueError" in message
        assert "i do not like any of your values" in message
        assert "(1, 2)" in message
        assert "{u'three': 3}" in message
<commit_msg>fix assertion across python versions<commit_after># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import pytest

from wampy.errors import RemoteError
from wampy.peers.clients import Client
from wampy.roles.callee import callee

class UnreliableCallee(Client):

    @callee
    def get_foo(self, *args, **kwargs):
        raise ValueError(
            "i do not like any of your values: {}, {}".format(
                args, kwargs)
        )

@pytest.yield_fixture
def unreliable_callee(router, config_path):
    with UnreliableCallee(router=router):
        yield

def test_handle_value_error(unreliable_callee, router):
    with Client(router=router, name="caller") as client:

        with pytest.raises(RemoteError) as exc_info:
            client.rpc.get_foo(1, 2, three=3)

        exception = exc_info.value
        assert type(exception) is RemoteError

        message = str(exception)

        assert "ValueError" in message
        assert "i do not like any of your values" in message
        assert "(1, 2)" in message
        assert "three" in message
<|endoftext|>


<commit_before># -*- coding: UTF-8 -*-
"""
:mod:`phi_pade` -- Phi Padé
============================

Computation of φ functions using Padé approximations and Scaling and Squaring.

.. module :: phi_pade
	:synopsis: Computation of φ functions.
.. moduleauthor :: Olivier Verdier <olivier.verdier@gmail.com>

Some formulae are taken from the `Expint documentation`_, of Håvard Berland, Bård Skaflestad and Will Wright.

.. _Expint project: http://www.math.ntnu.no/num/expint/
.. _Expint documentation: http://www.math.ntnu.no/preprint/numerics/2005/N4-2005.pdf

"""
from __future__ import division

import numpy as np
import numpy.linalg as lin
import math

import sys
sys.py3kwarning = True

class Polynomial(object):
	"""
Polynomial class used in the Padé approximation.
Usage is as follows::

	p = Polynomial([4.,2.,1.,...]) # polynomial 4 + 2x + x**2 + ...
	Z = Polynomial.exponents(z, 3) # compute z**2 and z**3
	p(Z) # value of the polynomial p(z)

The evaluation at a matrix value is computed using the Paterson and Stockmeyer method (see [Golub]_ § 11.2.4).
The polynomial is split into chunks of size :data:`s`.

.. [Golub] Golub, G.H.  and van Loan, C.F., *Matrix Computations*, 3rd ed. :isbn:`9780801854149`
	"""
	def __init__(self, coeffs):
		self.coeffs = coeffs

	@classmethod
	def exponents(self, z, s=1):
		"""
Compute the first s+1 exponents of z.

:Returns:
	[I,z,z**2,...,z**s]
		"""
		if np.isscalar(z):
			ident = 1
		else:
			ident = np.identity(len(z), dtype=z.dtype)
		Z = [ident]
		for i in range(s):
			Z.append(np.dot(Z[-1],z))
		return Z

	def __call__(self, Z):
		"""
Evaluate the polynomial on a matrix, using matrix multiplications (:func:`dot`).

:param list[s+1] Z:
		list of exponents of z up to s, so ``len(Z) == s+1``, where :data:`s` is the size of the chunks
		the various cases are the following:

		:s=1: Horner method
		:s ≥ d: naive polynomial evaluation.
		:s ≈ sqrt(d): optimal choice

		"""
		p = self.coeffs
		P = 0
		s = len(Z) - 1
		if s == 0: # ok only if the polynomial is constant
			if len(p) > 1:
				raise ValueError("Z must be provided in order to evaluate a non-constant polynomial.")
			return p[0]*Z[0]
		r = int(math.ceil(len(p)/s))
		# assert len(p) <= r*s # this should pass
		for k in reversed(range(r)):
			B = sum(b*Z[j] for j,b in enumerate(p[s*k:s*(k+1)]))
			P = np.dot(Z[s],P) + B
		return P

class RationalFraction(object):
	"""
Super simple Rational Function class, used to compute N(Z)/D(Z) for polynomials N and D.
	"""
	def __init__(self, numerator, denominator):
		"""
Initialize the object with numerator and denominator coefficients.
		"""
		self.numerator = Polynomial(numerator)
		self.denominator = Polynomial(denominator)

	def __call__(self, Z):
		try:
			return lin.solve(self.denominator(Z), self.numerator(Z))
		except lin.LinAlgError:
			return self.numerator(Z)/self.denominator(Z)

def ninf(M):
	if np.isscalar(M):
		return abs(M)
	return lin.norm(M,np.inf)

class Phi(object):
	r"""
Main class to compute the :math:`φ_l` functions.
The simplest way to define those functions is by the formula:

.. math::
	φ_{\ell}(x) = ∑_{k=0}^{∞} \frac{x^k}{(\ell+k)!}

Usage is as follows::

	phi = Phi(k,d)
	result = phi(M)

where :data:`M` is a square array.
The variable ``result`` is a list of all the values of :math:`φ_{k}(M)` for :math:`0≤k≤l`.
	"""

	def __init__(self, k, d=6):
		self.k = k
		self.d = d
		self.pade = list(self.compute_Pade())

	def compute_Pade(self):
		r"""
Compute the Padé approximations of order :math:`d` of :math:`φ_l`, for :math:`0 ≤ l ≤ k`.

The goal is to produce an approximation of the form:

.. math::
	φ_{\ell} = \frac{N}{D}

where :math:`N` and :math:`D` are polynomials.
The formula for :math:`D` is first computed recursively using the following recursion relations:

.. math::
	D_0^0 = 1\\
	D_{j+1}^0 = \frac{-(d-j)}{(2d -j)(j+1)} D_{j}^0\\
	D_{j}^{\ell+1} = (2d-\ell)(2d+\ell+1-j) D_{j}^{\ell}

Then, considering the inverse factorial series:

.. math::
	C_j := \frac{1}{j!}

The numerator :math:`N` is now computed by:

.. math::
	N = D*C

		"""
		d = self.d
		k = self.k
		J = np.arange(d+1)
		j = J[:-1]
		a = -(d-j)/(2*d-j)/(j+1)
		D = np.ones([k+1,d+1])
		D[0,1:] = np.cumprod(a)
		l = np.arange(k).reshape(-1,1)
		al = (2*d-l)*(2*d+l+1-J)
		D[1:,:] = np.cumprod(al,0) * D[0,:]
		C = np.empty(d+k+2)
		C[0] = 1.
		C[1:] = 1./np.cumprod(np.arange(d+k+1)+1)
		self.C = C # save for future use; C[j] == 1/j!
		for m,Dr in enumerate(D):
			yield RationalFraction(np.convolve(Dr, C[m:m+d+1])[:d+1], Dr)

	@classmethod
	def scaling(self, z, threshold=0):
		norm = ninf(z)
		e = max(threshold, np.log2(norm))
		return int(math.ceil(e))

	def eval_pade(self, z, s=None):
		"""
		Evaluate :math:`φ_l(z)` using the Padé approximation.
		"""
		if s is None:
			s = int(math.floor(math.sqrt(self.d)))
		Rs = self.pade
		Z = Polynomial.exponents(z,s)
		phi = [R(Z) for R in Rs]
		return phi

	def __call__(self, z):
		"""
The final call to compute the values of :math:`φ_k(z)`.
It proceeds in three steps:

1. figure out a scaling exponent :math:`s` such that :math:`z/2^s` is reasonably little
2. compute :math:`φ_k(z/2^s)` using the Padé approximation
3. scale back by repeatedly squaring
		"""
		scaling = self.scaling(z)
		phis = self.eval_pade(z/2**scaling)
		for s in range(scaling):
			phis = self.square(phis)
		return phis

	def phi_square(self, phis, l):
		"""
Formula for squaring phi_l from existing phi_k for k≤l, taken from the `Expint documentation`_.

.. _Expint documentation: http://www.math.ntnu.no/preprint/numerics/2005/N4-2005.pdf
		"""
		ifac = self.C
		odd = l % 2
		half = l//2
		next = half
		if odd:
			next += 1
		res = np.dot(phis[half], phis[next])
		res += sum(2*ifac[j]*phis[l-j] for j in xrange(half))
		if odd:
			res += ifac[half]*phis[half+1]
		res /= 2**l
		return res

	def square(self, phis):
		phis = [self.phi_square(phis, l) for l in range(self.k+1)]
		return phis

<commit_msg>one less argument in phi_square (now square_last)<commit_after># -*- coding: UTF-8 -*-
"""
:mod:`phi_pade` -- Phi Padé
============================

Computation of φ functions using Padé approximations and Scaling and Squaring.

.. module :: phi_pade
	:synopsis: Computation of φ functions.
.. moduleauthor :: Olivier Verdier <olivier.verdier@gmail.com>

Some formulae are taken from the `Expint documentation`_, of Håvard Berland, Bård Skaflestad and Will Wright.

.. _Expint project: http://www.math.ntnu.no/num/expint/
.. _Expint documentation: http://www.math.ntnu.no/preprint/numerics/2005/N4-2005.pdf

"""
from __future__ import division

import numpy as np
import numpy.linalg as lin
import math

import sys
sys.py3kwarning = True

class Polynomial(object):
	"""
Polynomial class used in the Padé approximation.
Usage is as follows::

	p = Polynomial([4.,2.,1.,...]) # polynomial 4 + 2x + x**2 + ...
	Z = Polynomial.exponents(z, 3) # compute z**2 and z**3
	p(Z) # value of the polynomial p(z)

The evaluation at a matrix value is computed using the Paterson and Stockmeyer method (see [Golub]_ § 11.2.4).
The polynomial is split into chunks of size :data:`s`.

.. [Golub] Golub, G.H.  and van Loan, C.F., *Matrix Computations*, 3rd ed. :isbn:`9780801854149`
	"""
	def __init__(self, coeffs):
		self.coeffs = coeffs

	@classmethod
	def exponents(self, z, s=1):
		"""
Compute the first s+1 exponents of z.

:Returns:
	[I,z,z**2,...,z**s]
		"""
		if np.isscalar(z):
			ident = 1
		else:
			ident = np.identity(len(z), dtype=z.dtype)
		Z = [ident]
		for i in range(s):
			Z.append(np.dot(Z[-1],z))
		return Z

	def __call__(self, Z):
		"""
Evaluate the polynomial on a matrix, using matrix multiplications (:func:`dot`).

:param list[s+1] Z:
		list of exponents of z up to s, so ``len(Z) == s+1``, where :data:`s` is the size of the chunks
		the various cases are the following:

		:s=1: Horner method
		:s ≥ d: naive polynomial evaluation.
		:s ≈ sqrt(d): optimal choice

		"""
		p = self.coeffs
		P = 0
		s = len(Z) - 1
		if s == 0: # ok only if the polynomial is constant
			if len(p) > 1:
				raise ValueError("Z must be provided in order to evaluate a non-constant polynomial.")
			return p[0]*Z[0]
		r = int(math.ceil(len(p)/s))
		# assert len(p) <= r*s # this should pass
		for k in reversed(range(r)):
			B = sum(b*Z[j] for j,b in enumerate(p[s*k:s*(k+1)]))
			P = np.dot(Z[s],P) + B
		return P

class RationalFraction(object):
	"""
Super simple Rational Function class, used to compute N(Z)/D(Z) for polynomials N and D.
	"""
	def __init__(self, numerator, denominator):
		"""
Initialize the object with numerator and denominator coefficients.
		"""
		self.numerator = Polynomial(numerator)
		self.denominator = Polynomial(denominator)

	def __call__(self, Z):
		try:
			return lin.solve(self.denominator(Z), self.numerator(Z))
		except lin.LinAlgError:
			return self.numerator(Z)/self.denominator(Z)

def ninf(M):
	if np.isscalar(M):
		return abs(M)
	return lin.norm(M,np.inf)

class Phi(object):
	r"""
Main class to compute the :math:`φ_l` functions.
The simplest way to define those functions is by the formula:

.. math::
	φ_{\ell}(x) = ∑_{k=0}^{∞} \frac{x^k}{(\ell+k)!}

Usage is as follows::

	phi = Phi(k,d)
	result = phi(M)

where :data:`M` is a square array.
The variable ``result`` is a list of all the values of :math:`φ_{k}(M)` for :math:`0≤k≤l`.
	"""

	def __init__(self, k, d=6):
		self.k = k
		self.d = d
		self.pade = list(self.compute_Pade())

	def compute_Pade(self):
		r"""
Compute the Padé approximations of order :math:`d` of :math:`φ_l`, for :math:`0 ≤ l ≤ k`.

The goal is to produce an approximation of the form:

.. math::
	φ_{\ell} = \frac{N}{D}

where :math:`N` and :math:`D` are polynomials.
The formula for :math:`D` is first computed recursively using the following recursion relations:

.. math::
	D_0^0 = 1\\
	D_{j+1}^0 = \frac{-(d-j)}{(2d -j)(j+1)} D_{j}^0\\
	D_{j}^{\ell+1} = (2d-\ell)(2d+\ell+1-j) D_{j}^{\ell}

Then, considering the inverse factorial series:

.. math::
	C_j := \frac{1}{j!}

The numerator :math:`N` is now computed by:

.. math::
	N = D*C

		"""
		d = self.d
		k = self.k
		J = np.arange(d+1)
		j = J[:-1]
		a = -(d-j)/(2*d-j)/(j+1)
		D = np.ones([k+1,d+1])
		D[0,1:] = np.cumprod(a)
		l = np.arange(k).reshape(-1,1)
		al = (2*d-l)*(2*d+l+1-J)
		D[1:,:] = np.cumprod(al,0) * D[0,:]
		C = np.empty(d+k+2)
		C[0] = 1.
		C[1:] = 1./np.cumprod(np.arange(d+k+1)+1)
		self.C = C # save for future use; C[j] == 1/j!
		for m,Dr in enumerate(D):
			yield RationalFraction(np.convolve(Dr, C[m:m+d+1])[:d+1], Dr)

	@classmethod
	def scaling(self, z, threshold=0):
		norm = ninf(z)
		e = max(threshold, np.log2(norm))
		return int(math.ceil(e))

	def eval_pade(self, z, s=None):
		"""
		Evaluate :math:`φ_l(z)` using the Padé approximation.
		"""
		if s is None:
			s = int(math.floor(math.sqrt(self.d)))
		Rs = self.pade
		Z = Polynomial.exponents(z,s)
		phi = [R(Z) for R in Rs]
		return phi

	def __call__(self, z):
		"""
The final call to compute the values of :math:`φ_k(z)`.
It proceeds in three steps:

1. figure out a scaling exponent :math:`s` such that :math:`z/2^s` is reasonably little
2. compute :math:`φ_k(z/2^s)` using the Padé approximation
3. scale back by repeatedly squaring
		"""
		scaling = self.scaling(z)
		phis = self.eval_pade(z/2**scaling)
		for s in range(scaling):
			phis = self.square(phis)
		return phis

	def square_last(self, phis):
		"""
Formula for squaring phi_l from existing phi_k for k≤l, taken from the `Expint documentation`_.

The argument is an array containing [phi_0,...,phi_l].

.. _Expint documentation: http://www.math.ntnu.no/preprint/numerics/2005/N4-2005.pdf
		"""
		l = len(phis) - 1
		ifac = self.C
		odd = l % 2
		half = l//2
		next = half
		if odd:
			next += 1
		res = np.dot(phis[half], phis[next])
		res += sum(2*ifac[j]*phis[l-j] for j in xrange(half))
		if odd:
			res += ifac[half]*phis[half+1]
		res /= 2**l
		return res

	def square(self, phis):
		phis = [self.square_last(phis[:l+1]) for l in range(self.k+1)]
		return phis

<|endoftext|>


<commit_before>import os
import pika
import sys
import pyowm
import json
from pika import PlainCredentials

OWM_API_KEY = os.getenv("OWM_API_KEY")
RAW_GEOLOCATION_QUEUE = "rawGeoLocations"
WETTER_QUEUE = "gefundenesWetter"

owm = pyowm.OWM(OWM_API_KEY)

connection = pika.BlockingConnection(
    pika.ConnectionParameters('192.168.0.151', credentials=PlainCredentials("gast", "gast")))

locationChannel = connection.channel()
locationChannel.queue_declare(RAW_GEOLOCATION_QUEUE, durable=True)

wetterChannel = connection.channel()
wetterChannel.queue_declare(WETTER_QUEUE, durable=True)

class Location:
    def __init__(self, latitude, longitude):
        self.latitude = latitude
        self.longitude = longitude

class Wetter:
    def __init__(self, status):
        self.status = status

def location_decoder(obj):
    return Location(obj['latitude'], obj['longitude'])

class WetterFinder:
    def __init__(self, owm, publishChannel, publishQueueName):
        self.owm = owm
        self.publishChannel = publishChannel
        self.publishQueueName = publishQueueName
        self.gefundeneWetter = 0

    def get_wetter(self, wetterresponse):
        return wetterresponse.get_weather().get_status()

    def tweet_erhalten(self, channel, method, properties, body):
        location = json.loads(body, object_hook=location_decoder)
        if location.latitude < -90 or location.latitude > 90 or location.longitude < -90 or location.longitude > 90:
            return

        wetterresponse = self.owm.weather_at_coords(location.longitude, location.latitude)
        daswetter = self.get_wetter(wetterresponse)
        channel.basic_ack(delivery_tag=method.delivery_tag)
        if daswetter is None:
            return
        self.gefundeneWetter += 1
        sys.stdout.write("\rGefundene Wetter: " + str(self.gefundeneWetter))
        sys.stdout.flush()
        wetter = Wetter(daswetter)
        self.publishChannel.basic_publish(exchange='',
                                          routing_key=self.publishQueueName,
                                          body=json.dumps(wetter.__dict__))

wetterfinder = WetterFinder(owm, wetterChannel, WETTER_QUEUE)

locationChannel.basic_consume(wetterfinder.tweet_erhalten,
                              queue=RAW_GEOLOCATION_QUEUE)

locationChannel.start_consuming()
<commit_msg>API Key ueber Kommandozeile<commit_after>import os
import pika
import sys
import pyowm
import json
from pika import PlainCredentials

OWM_API_KEY = str(sys.argv[0])
RAW_GEOLOCATION_QUEUE = "rawGeoLocations"
WETTER_QUEUE = "gefundenesWetter"

owm = pyowm.OWM(OWM_API_KEY)

connection = pika.BlockingConnection(
    pika.ConnectionParameters('192.168.1.1', credentials=PlainCredentials("gast", "gast")))

locationChannel = connection.channel()
locationChannel.queue_declare(RAW_GEOLOCATION_QUEUE, durable=True)

wetterChannel = connection.channel()
wetterChannel.queue_declare(WETTER_QUEUE, durable=True)

class Location:
    def __init__(self, latitude, longitude):
        self.latitude = latitude
        self.longitude = longitude

class Wetter:
    def __init__(self, status):
        self.status = status

def location_decoder(obj):
    return Location(obj['latitude'], obj['longitude'])

class WetterFinder:
    def __init__(self, owm, publishChannel, publishQueueName):
        self.owm = owm
        self.publishChannel = publishChannel
        self.publishQueueName = publishQueueName
        self.gefundeneWetter = 0

    def get_wetter(self, wetterresponse):
        return wetterresponse.get_weather().get_status()

    def tweet_erhalten(self, channel, method, properties, body):
        location = json.loads(body, object_hook=location_decoder)
        if location.latitude < -90 or location.latitude > 90 or location.longitude < -90 or location.longitude > 90:
            return

        wetterresponse = self.owm.weather_at_coords(location.longitude, location.latitude)
        daswetter = self.get_wetter(wetterresponse)
        channel.basic_ack(delivery_tag=method.delivery_tag)
        if daswetter is None:
            return
        self.gefundeneWetter += 1
        sys.stdout.write("\rGefundene Wetter: " + str(self.gefundeneWetter))
        sys.stdout.flush()
        wetter = Wetter(daswetter)
        self.publishChannel.basic_publish(exchange='',
                                          routing_key=self.publishQueueName,
                                          body=json.dumps(wetter.__dict__))

wetterfinder = WetterFinder(owm, wetterChannel, WETTER_QUEUE)

locationChannel.basic_consume(wetterfinder.tweet_erhalten,
                              queue=RAW_GEOLOCATION_QUEUE)

locationChannel.start_consuming()
<|endoftext|>


<commit_before>import rospy
from tf2_geometry_msgs import PoseStamped

from dynamic_stack_decider.abstract_action_element import AbstractActionElement

class SendStop(AbstractActionElement):
    """ This stopps the robots walking and directly pops itself  """

    def perform(self, reevaluate=False):
        stand_pose = PoseStamped()
        stand_pose.header.stamp = rospy.Time.now()
        stand_pose.header.frame_id = 'base_footprint'
        stand_pose.pose.orientation.w = 1
        self.blackboard.pathfinding.publish(stand_pose)
        self.pop()

class Stop(AbstractActionElement):
    """ This stopps the robots walking and pops itself after stop has completed """

    def perform(self, reevaluate=False):
        stand_pose = PoseStamped()
        stand_pose.header.stamp = rospy.Time.now()
        stand_pose.header.frame_id = 'base_footprint'
        stand_pose.pose.orientation.w = 1
        self.blackboard.pathfinding.publish(stand_pose)
        #TODO we should wait until the robot really stopped, by looking at the odometry
        self.pop()<commit_msg>implemented stand action as dummy<commit_after>import rospy
from tf2_geometry_msgs import PoseStamped

from dynamic_stack_decider.abstract_action_element import AbstractActionElement

class SendStop(AbstractActionElement):
    """ This stopps the robots walking and directly pops itself  """

    def perform(self, reevaluate=False):
        stand_pose = PoseStamped()
        stand_pose.header.stamp = rospy.Time.now()
        stand_pose.header.frame_id = 'base_footprint'
        stand_pose.pose.orientation.w = 1
        self.blackboard.pathfinding.publish(stand_pose)
        self.pop()

class Stop(AbstractActionElement):
    """ This stops the robots walking and pops itself after stop has completed """

    def perform(self, reevaluate=False):
        stand_pose = PoseStamped()
        stand_pose.header.stamp = rospy.Time.now()
        stand_pose.header.frame_id = 'base_footprint'
        stand_pose.pose.orientation.w = 1
        self.blackboard.pathfinding.publish(stand_pose)
        #TODO we should wait until the robot really stopped, by looking at the odometry
        self.pop()

class Stand(AbstractActionElement):
    """ This stops the robots walking and pops itself after stop has completed """

    def perform(self, reevaluate=False):
        stand_pose = PoseStamped()
        stand_pose.header.stamp = rospy.Time.now()
        stand_pose.header.frame_id = 'base_footprint'
        stand_pose.pose.orientation.w = 1
        self.blackboard.pathfinding.publish(stand_pose)
        #TODO we should wait until the robot really stopped, by looking at the odometry
        self.pop()<|endoftext|>


<commit_before>"""

@author: JD Chodera
@author: JH Prinz
"""

from openpathsampling.engines import BaseSnapshot, SnapshotFactory
import openpathsampling.engines.features as feats
from . import features as toy_feats

@feats.attach_features([
    toy_feats.velocities,
    toy_feats.coordinates,
    toy_feats.instantaneous_temperature,
    toy_feats.engine
])
class ToySnapshot(BaseSnapshot):
    """
    Simulation snapshot. Only references to coordinates and velocities
    """

    @property
    def topology(self):
        return self.engine.topology

    @property
    def masses(self):
        return self.topology.masses

# The following code does almost the same as above

# ToySnapshot = SnapshotFactory(
#     name='ToySnapshot',
#     features=[
#         features.velocities,
#         features.coordinates,
#         features.engine
#     ],
#     description="Simulation snapshot. Only references to coordinates and "
#                 "velocities",
#     base_class=BaseSnapshot
# )
<commit_msg>Fix for bad merge decision<commit_after>"""

@author: JD Chodera
@author: JH Prinz
"""

from openpathsampling.engines import BaseSnapshot, SnapshotFactory
from openpathsampling.engines import features as feats
from . import features as toy_feats

@feats.attach_features([
    toy_feats.velocities,
    toy_feats.coordinates,
    toy_feats.instantaneous_temperature,
    toy_feats.engine
])
class ToySnapshot(BaseSnapshot):
    """
    Simulation snapshot. Only references to coordinates and velocities
    """

    @property
    def topology(self):
        return self.engine.topology

    @property
    def masses(self):
        return self.topology.masses

# The following code does almost the same as above

# ToySnapshot = SnapshotFactory(
#     name='ToySnapshot',
#     features=[
#         features.velocities,
#         features.coordinates,
#         features.engine
#     ],
#     description="Simulation snapshot. Only references to coordinates and "
#                 "velocities",
#     base_class=BaseSnapshot
# )
<|endoftext|>


<commit_before>"""
WSGI config for Bitket project.

This module contains the WSGI application used by Django's development server
and any production WSGI deployments. It should expose a module-level variable
named ``application``. Django's ``runserver`` and ``runfcgi`` commands discover
this application via the ``WSGI_APPLICATION`` setting.

Usually you will have the standard Django WSGI application here, but it also
might make sense to replace the whole Django WSGI application with a custom one
that later delegates to the Django one. For example, you could introduce WSGI
middleware here, or combine a Django application with an application of another
framework.

"""
from django.core.wsgi import get_wsgi_application

# This application object is used by any WSGI server configured to use this
# file. This includes Django's development server, if the WSGI_APPLICATION
# setting points here.
application = get_wsgi_application()
<commit_msg>Removed unneeded WSGI file<commit_after><|endoftext|>