<commit_before>#
# Copyright (c) 2015, Arista Networks, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#   Redistributions of source code must retain the above copyright notice,
#   this list of conditions and the following disclaimer.
#
#   Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in the
#   documentation and/or other materials provided with the distribution.
#
#   Neither the name of Arista Networks nor the names of its
#   contributors may be used to endorse or promote products derived from
#   this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL ARISTA NETWORKS
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
# IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
"""Module for working with EOS MLAG resources

The Mlag resource provides configuration management of global and interface
MLAG settings on an EOS node.

Parameters:

    config (dict): The global MLAG configuration values

    interfaces (dict): The configured MLAG interfaces

Config Parameters:
    domain_id (str): The domain_id parameter  is parsed from the nodes
        mlag configuration.  The domain id is an alphanumeric string that
        names the MLAG domain

    local_interface (str): The local VLAN interface used as the control
        plane endpoint between MLAG peers.  Valid values include any
        VLAN SVI

    peer_address (str): The IP address of the MLAG peer used to send MLAG
        control traffic.  The peer address must be reachable from the
        local interface.  Valid values include any IPv4 unicast address

    peer_link (str): The physical link that connects the node to its MLAG
        peer.  Valid values for the peer link include layer 2 Ethernet or
        Port-Channel interfaces

    shutdown (bool): The administrative state of the global MLAG process.

Interface Parameters:
    mlag_id (str): The interface mlag parameter parsed from the nodes
        interface configuration.  Valid values for the mlag id are in
        the range of 1 to 2000

"""

import re

from pyeapi.api import Entity

class Routemaps(Entity):
    """The Routemaps class provides management of the routemaps configuration

    The Routemaps class is derived from Entity and provides an API for working
    with the nodes routemaps configuraiton.
    """

    def get(self, name, action, seqno):
        return self._get_instance(name, action, seqno)

    def getall(self):
        resources = dict()
        routemaps_re = re.compile(r'^route-map\s(\w+)\s(\w+)\s(\d+)$', re.M)
        for entry in routemaps_re.findall(self.config):
            name = entry[0]
            action = entry[1]
            seqno = int(entry[2])

            routemap = self.get(name, action, seqno)
            if routemap:
                key = (name, action, seqno)
                resources[key] = routemap
        return resources

    def _get_instance(self, name, action, seqno):
        routemap_re = r'route-map\s%s\s%s\s%s' % (name, action, seqno)
        routemap = self.get_block(routemap_re)

        if not routemap:
            return None

        resource = dict(name=name, action=action, seqno=int(seqno))
        resource.update(self._parse_match_statements(routemap))
        resource.update(self._parse_set_statements(routemap))
        resource.update(self._parse_continue_statement(routemap))
        return resource

    def _parse_match_statements(self, config):
        match_re = re.compile(r'^\s+match\s(.*)$', re.M)
        return dict(match=match_re.findall(config))

    def _parse_set_statements(self, config):
        set_re = re.compile(r'^\s+set\s(.*)$', re.M)
        return dict(set=set_re.findall(config))

    def _parse_continue_statement(self, config):
        continue_re = re.compile(r'^\s+continue\s(\d+)$', re.M)
        match = continue_re.search(config)
        value = int(match.group(1)) if match else None
        return {'continue': value}

    def create(self, name, action, seqno):
        """Creates a new routemap on the node

        Note:
            This method will attempt to create the routemap regardless
            if the routemap exists or not.  If the routemap already exists
            then this method will still return True.

        Args:
            name (string): The full name of the routemap.
            action (choices: permit,deny): The action to take for this routemap
                                           clause.
            seqno (integer): The sequence number for the routemap clause.

        Returns:
            True if the routemap could be created otherwise False (see Note)

        """
        if action not in ['permit', 'deny']:
            raise ValueError('action must be permit or deny')
        if seqno < 1 or seqno > 16777215:
            raise ValueError('seqno must be an integer between 1 and 16777215')
        return self.configure('route-map %s %s %s' % (name, action, seqno))

    def delete(self, name, action, seqno):
        """Deletes the routemap from the node

        Note:
            This method will attempt to delete the routemap from the nodes
            operational config.  If the routemap does not exist then this
            method will not perform any changes but still return True

        Args:
            name (string): The full name of the routemap.
            action (choices: permit,deny): The action to take for this routemap
                                           clause.
            seqno (integer): The sequence number for the routemap clause.

        Returns:
            True if the routemap could be deleted otherwise False (see Node)

        """
        return self.configure('no route-map %s %s %s' % (name, action, seqno))

    def default(self, name, action, seqno):
        """Defaults the routemap on the node

        Note:
            This method will attempt to default the routemap from the nodes
            operational config. Since routemaps do not exist by default,
            the default action is essentially a negation and the result will
            be the removal of the routemap clause.
            If the routemap does not exist then this
            method will not perform any changes but still return True

        Args:
            name (string): The full name of the routemap.
            action (choices: permit,deny): The action to take for this routemap
                                           clause.
            seqno (integer): The sequence number for the routemap clause.

        Returns:
            True if the routemap could be deleted otherwise False (see Node)

        """
        return self.configure('default route-map %s %s %s'
                              % (name, action, seqno))

    def set_match_statements(self, name, action, seqno, statements):
        """Configures the match statements that are found within the
        routemap clause. Match statements found in the routemap that
        are not specified in the \'statements\' list will be removed.

        Args:
            name (string): The full name of the routemap.
            action (choices: permit,deny): The action to take for this routemap
                                           clause.
            seqno (integer): The sequence number for the routemap clause.
            statements (list): A list of the match-related statements. Note
                               that the statements should omit the leading
                               \'match\'.

        Returns:
            True if the operation succeeds otherwise False
        """
        try:
            current_statements = self._get_instance(name, action, seqno)['match']
        except:
            current_statements = []

        commands = list()

        # remove set statements from current routemap
        for entry in set(current_statements).difference(statements):
            commands.append('route-map %s %s %s' % (name, action, seqno))
            commands.append('no match %s' % entry)

        # add new set statements to the routemap
        for entry in set(statements).difference(current_statements):
            commands.append('route-map %s %s %s' % (name, action, seqno))
            commands.append('match %s' % entry)

        return self.configure(commands) if commands else True

    def set_set_statements(self, name, action, seqno, statements):
        """Configures the set statements that are found within the
        routemap clause. Set statements found in the routemap that
        are not specified in the \'statements\' list will be removed.

        Args:
            name (string): The full name of the routemap.
            action (choices: permit,deny): The action to take for this routemap
                                           clause.
            seqno (integer): The sequence number for the routemap clause.
            statements (list): A list of the set-related statements. Note that
                               the statements should omit the leading \'set\'.

        Returns:
            True if the operation succeeds otherwise False
        """
        try:
            current_statements = self._get_instance(name, action, seqno)['set']
        except:
            current_statements = []

        commands = list()

        # remove set statements from current routemap
        for entry in set(current_statements).difference(statements):
            commands.append('route-map %s %s %s' % (name, action, seqno))
            commands.append('no set %s' % entry)

        # add new set statements to the routemap
        for entry in set(statements).difference(current_statements):
            commands.append('route-map %s %s %s' % (name, action, seqno))
            commands.append('set %s' % entry)

        return self.configure(commands) if commands else True

    def set_continue(self, name, action, seqno, value=None, default=False):
        """Configures the routemap \'continue\' value

        Args:
            name (string): The full name of the routemap.
            action (choices: permit,deny): The action to take for this routemap
                                           clause.
            seqno (integer): The sequence number for the routemap clause.
            value (integer): The value to configure for the routemap continue
            default (bool): Specifies to default the routemap continue value

        Returns:
            True if the operation succeeds otherwise False is returned
        """
        commands = ['route-map %s %s %s' % (name, action, seqno)]
        if default:
            commands.append('default continue')
        elif value is not None:
            if value < 1 or value > 16777215:
                raise ValueError('seqno must be an integer between '
                                 '1 and 16777215')
            commands.append('continue %s' % value)
        else:
            commands.append('no continue')

        return self.configure(commands)

def instance(node):
    """Returns an instance of Routemaps

    Args:
        node (Node): The node argument passes an instance of Node to the
            resource

    Returns:
        object: An instance of Routemaps
    """
    return Routemaps(node)
<commit_msg>Update docstring<commit_after>#
# Copyright (c) 2015, Arista Networks, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#   Redistributions of source code must retain the above copyright notice,
#   this list of conditions and the following disclaimer.
#
#   Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in the
#   documentation and/or other materials provided with the distribution.
#
#   Neither the name of Arista Networks nor the names of its
#   contributors may be used to endorse or promote products derived from
#   this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL ARISTA NETWORKS
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
# IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
"""Module for working with EOS routemap resources

The Routemap resource provides configuration management of global route-map
resources on an EOS node. It provides the following class implementations:

    * Routemaps - Configures routemaps in EOS

Routemaps Attributes:
    name (string): The name given to the routemap clause
    action (choices: permit/deny): How the clause will filter the route
    seqno (integer): The sequence number of this clause
    set (list): The list of set statements present in this clause
    match (list): The list of match statements present in this clause.
    continue (integer): The next sequence number to evaluate if the criteria
                        in this clause are met.

Notes:
    The \'set\' and \'match\' attributes produce a list of strings with The
    corresponding configuration. These strings will omit the preceeding
    \'set\' or \'match\' words, respectively.
"""

import re

from pyeapi.api import Entity

class Routemaps(Entity):
    """The Routemaps class provides management of the routemaps configuration

    The Routemaps class is derived from Entity and provides an API for working
    with the nodes routemaps configuraiton.
    """

    def get(self, name, action, seqno):
        return self._get_instance(name, action, seqno)

    def getall(self):
        resources = dict()
        routemaps_re = re.compile(r'^route-map\s(\w+)\s(\w+)\s(\d+)$', re.M)
        for entry in routemaps_re.findall(self.config):
            name = entry[0]
            action = entry[1]
            seqno = int(entry[2])

            routemap = self.get(name, action, seqno)
            if routemap:
                key = (name, action, seqno)
                resources[key] = routemap
        return resources

    def _get_instance(self, name, action, seqno):
        routemap_re = r'route-map\s%s\s%s\s%s' % (name, action, seqno)
        routemap = self.get_block(routemap_re)

        if not routemap:
            return None

        resource = dict(name=name, action=action, seqno=int(seqno))
        resource.update(self._parse_match_statements(routemap))
        resource.update(self._parse_set_statements(routemap))
        resource.update(self._parse_continue_statement(routemap))
        return resource

    def _parse_match_statements(self, config):
        match_re = re.compile(r'^\s+match\s(.*)$', re.M)
        return dict(match=match_re.findall(config))

    def _parse_set_statements(self, config):
        set_re = re.compile(r'^\s+set\s(.*)$', re.M)
        return dict(set=set_re.findall(config))

    def _parse_continue_statement(self, config):
        continue_re = re.compile(r'^\s+continue\s(\d+)$', re.M)
        match = continue_re.search(config)
        value = int(match.group(1)) if match else None
        return {'continue': value}

    def create(self, name, action, seqno):
        """Creates a new routemap on the node

        Note:
            This method will attempt to create the routemap regardless
            if the routemap exists or not.  If the routemap already exists
            then this method will still return True.

        Args:
            name (string): The full name of the routemap.
            action (choices: permit,deny): The action to take for this routemap
                                           clause.
            seqno (integer): The sequence number for the routemap clause.

        Returns:
            True if the routemap could be created otherwise False (see Note)

        """
        if action not in ['permit', 'deny']:
            raise ValueError('action must be permit or deny')
        if seqno < 1 or seqno > 16777215:
            raise ValueError('seqno must be an integer between 1 and 16777215')
        return self.configure('route-map %s %s %s' % (name, action, seqno))

    def delete(self, name, action, seqno):
        """Deletes the routemap from the node

        Note:
            This method will attempt to delete the routemap from the nodes
            operational config.  If the routemap does not exist then this
            method will not perform any changes but still return True

        Args:
            name (string): The full name of the routemap.
            action (choices: permit,deny): The action to take for this routemap
                                           clause.
            seqno (integer): The sequence number for the routemap clause.

        Returns:
            True if the routemap could be deleted otherwise False (see Node)

        """
        return self.configure('no route-map %s %s %s' % (name, action, seqno))

    def default(self, name, action, seqno):
        """Defaults the routemap on the node

        Note:
            This method will attempt to default the routemap from the nodes
            operational config. Since routemaps do not exist by default,
            the default action is essentially a negation and the result will
            be the removal of the routemap clause.
            If the routemap does not exist then this
            method will not perform any changes but still return True

        Args:
            name (string): The full name of the routemap.
            action (choices: permit,deny): The action to take for this routemap
                                           clause.
            seqno (integer): The sequence number for the routemap clause.

        Returns:
            True if the routemap could be deleted otherwise False (see Node)

        """
        return self.configure('default route-map %s %s %s'
                              % (name, action, seqno))

    def set_match_statements(self, name, action, seqno, statements):
        """Configures the match statements that are found within the
        routemap clause. Match statements found in the routemap that
        are not specified in the \'statements\' list will be removed.

        Args:
            name (string): The full name of the routemap.
            action (choices: permit,deny): The action to take for this routemap
                                           clause.
            seqno (integer): The sequence number for the routemap clause.
            statements (list): A list of the match-related statements. Note
                               that the statements should omit the leading
                               \'match\'.

        Returns:
            True if the operation succeeds otherwise False
        """
        try:
            current_statements = self._get_instance(name, action, seqno)['match']
        except:
            current_statements = []

        commands = list()

        # remove set statements from current routemap
        for entry in set(current_statements).difference(statements):
            commands.append('route-map %s %s %s' % (name, action, seqno))
            commands.append('no match %s' % entry)

        # add new set statements to the routemap
        for entry in set(statements).difference(current_statements):
            commands.append('route-map %s %s %s' % (name, action, seqno))
            commands.append('match %s' % entry)

        return self.configure(commands) if commands else True

    def set_set_statements(self, name, action, seqno, statements):
        """Configures the set statements that are found within the
        routemap clause. Set statements found in the routemap that
        are not specified in the \'statements\' list will be removed.

        Args:
            name (string): The full name of the routemap.
            action (choices: permit,deny): The action to take for this routemap
                                           clause.
            seqno (integer): The sequence number for the routemap clause.
            statements (list): A list of the set-related statements. Note that
                               the statements should omit the leading \'set\'.

        Returns:
            True if the operation succeeds otherwise False
        """
        try:
            current_statements = self._get_instance(name, action, seqno)['set']
        except:
            current_statements = []

        commands = list()

        # remove set statements from current routemap
        for entry in set(current_statements).difference(statements):
            commands.append('route-map %s %s %s' % (name, action, seqno))
            commands.append('no set %s' % entry)

        # add new set statements to the routemap
        for entry in set(statements).difference(current_statements):
            commands.append('route-map %s %s %s' % (name, action, seqno))
            commands.append('set %s' % entry)

        return self.configure(commands) if commands else True

    def set_continue(self, name, action, seqno, value=None, default=False):
        """Configures the routemap \'continue\' value

        Args:
            name (string): The full name of the routemap.
            action (choices: permit,deny): The action to take for this routemap
                                           clause.
            seqno (integer): The sequence number for the routemap clause.
            value (integer): The value to configure for the routemap continue
            default (bool): Specifies to default the routemap continue value

        Returns:
            True if the operation succeeds otherwise False is returned
        """
        commands = ['route-map %s %s %s' % (name, action, seqno)]
        if default:
            commands.append('default continue')
        elif value is not None:
            if value < 1 or value > 16777215:
                raise ValueError('seqno must be an integer between '
                                 '1 and 16777215')
            commands.append('continue %s' % value)
        else:
            commands.append('no continue')

        return self.configure(commands)

def instance(node):
    """Returns an instance of Routemaps

    Args:
        node (Node): The node argument passes an instance of Node to the
            resource

    Returns:
        object: An instance of Routemaps
    """
    return Routemaps(node)
<|endoftext|>


<commit_before>#!/usr/bin/env python-sirius
"""."""

import time as _time
from epics import PV as _PV
import numpy as np
from optimization import PSO

class PSOInjection(PSO):

    def initialization(self):
        self._upper_limits = np.array([2e-3, 2e-3, 2e-3, 2e-3, 1e-3])
        self._lower_limits = - self._upper_limits
        self._sum_lim = 0
        self._nswarm = 10 + 2 * int(np.sqrt(len(self._upper_limits)))
        self._p_bpm = 1.0
        self._p_orb = 1.0e3

        # Single Pass acquisition
        _pv_sp_orbx = _PV('BO-Glob:AP-SOFB:SPassOrbX-Mon')  # um
        _pv_sp_orby = _PV('BO-Glob:AP-SOFB:SPassOrbY-Mon')  # um
        _pv_sp_sum = _PV('BO-Glob:AP-SOFB:SPassSum-Mon')    # counts

        sp_diag = [_pv_sp_orbx,
                   _pv_sp_orby,
                   _pv_sp_sum]

        # # Multi Turn acquisition
        # _pv_mt_orbx = _PV('BO-Glob:AP-SOFB:MTurnIdxOrbX-Mon')  # um
        # _pv_mt_orby = _PV('BO-Glob:AP-SOFB:MTurnIdxOrbY-Mon')  # um
        # _pv_mt_sum = _PV('BO-Glob:AP-SOFB:MTurnSum-Mon')    # counts

        # mt_diag = [_pv_mt_orbx,
        #            _pv_mt_orby,
        #            _pv_mt_sum]

        self.diag = sp_diag

        _pv_dx = _PV('TB-Glob:AP-PosAng:DeltaPosX-SP')   # mm
        _pv_dxl = _PV('TB-Glob:AP-PosAng:DeltaAngX-SP')  # mrad
        _pv_dy = _PV('TB-Glob:AP-PosAng:DeltaPosY-SP')   # mm
        _pv_dyl = _PV('TB-Glob:AP-PosAng:DeltaAngY-SP')  # mrad
        _pv_kckr = _PV('BO-01D:PM-InjKckr:Kick-SP')      # mrad
        self.corrs = [_pv_dx,
                      _pv_dxl,
                      _pv_dy,
                      _pv_dyl,
                      _pv_kckr]

        _pv_resetposang = _PV('TB-Glob:AP-PosAng:SetNewRefKick-Cmd')
        _pv_resetposang.value = 1

        self.reference = np.array([cval.value for cval in self.corrs])

        _pv_buffer = _PV('BO-Glob:AP-SOFB:SmoothNrPts-RB')
        self._wait = _pv_buffer.value

    def calc_merit_function(self):
        f_out = np.zeros(self._nswarm)
        ind_bpm = np.arange(1, len(self.diag[-1].value) + 1)

        for i in range(self._nswarm):
            change = self.reference + self._position[i, :]

            for k in range(len(self.corrs)):
                self.corrs[k].value = change[k]

            _time.sleep(self._wait * 0.5 + 1)
            f_bpm = np.dot(ind_bpm, self.diag[-1].value)
            bpm_sel = self.diag[-1].value < self._sum_lim
            ind_sel = np.where(bpm_sel)
            orbx = self.diag[0].value
            orby = self.diag[1].value
            sum_quad = np.sum(
                orbx[:ind_sel[0]] ** 2 +
                orby[:ind_sel[0]] ** 2)
            f_orb = np.sqrt(sum_quad)
            if f_orb.size > 0 and f_orb != 0:
                f_out[i] = self._p_bpm * f_bpm + self._p_orb / f_orb
            else:
                f_out[i] = self._p_bpm * f_bpm
        return - f_out

# if __name__ == "__main__":
#     import argparse as _argparse

#     parser = _argparse.ArgumentParser(
#         description="PSO script for Booster Injection Optimization.")

#     parser.add_argument(
#         '-sum', '--sum_lim', type=float, default=1e3,
#         help='Minimum BPM Sum Signal to calculate merit function (1 kcount).')
#     parser.add_argument(
#         '-niter', '--niter', type=int, default=30,
#         help='Number of Iteractions (30).')
#     parser.add_argument(
#         '-lim', '--limits', nargs='+', type=float,
#         help='Upper limits [XPos, XAng, YPos, YAng, Kckr]')

#     args = parser.parse_args()
#     Pso = PSOInjection(limits=args.limits, sum_lim=args.sum_lim)
#     Pso._start_optimization(niter=args.niter)
<commit_msg>Delete old script<commit_after><|endoftext|>


<commit_before># -*- Python -*-

# Common configuration for running leak detection tests under LSan/ASan.

import os
import re

import lit.util

def get_required_attr(config, attr_name):
  attr_value = getattr(config, attr_name, None)
  if attr_value == None:
    lit_config.fatal(
      "No attribute %r in test configuration! You may need to run "
      "tests from your build directory or add this attribute "
      "to lit.site.cfg.py " % attr_name)
  return attr_value

# Setup source root.
config.test_source_root = os.path.dirname(__file__)

# Choose between standalone and LSan+ASan modes.
lsan_lit_test_mode = get_required_attr(config, 'lsan_lit_test_mode')
if lsan_lit_test_mode == "Standalone":
  config.name = "LeakSanitizer-Standalone"
  lsan_cflags = ["-fsanitize=leak"]
elif lsan_lit_test_mode == "AddressSanitizer":
  config.name = "LeakSanitizer-AddressSanitizer"
  lsan_cflags = ["-fsanitize=address"]
  config.available_features.add('asan')
else:
  lit_config.fatal("Unknown LSan test mode: %r" % lsan_lit_test_mode)
config.name += config.name_suffix

# Platform-specific default LSAN_OPTIONS for lit tests.
default_lsan_opts = 'detect_leaks=1'
if config.host_os == 'Darwin':
  # On Darwin, we default to `abort_on_error=1`, which would make tests run
  # much slower. Let's override this and run lit tests with 'abort_on_error=0'.
  # Also, make sure we do not overwhelm the syslog while testing.
  default_lsan_opts += ':abort_on_error=0'
  default_lsan_opts += ':log_to_syslog=0'

if default_lsan_opts:
  config.environment['LSAN_OPTIONS'] = default_lsan_opts
  default_lsan_opts += ':'
config.substitutions.append(('%env_lsan_opts=',
                             'env LSAN_OPTIONS=' + default_lsan_opts))

if lit.util.which('strace'):
  config.available_features.add('strace')

clang_cflags = ["-O0", config.target_cflags] + config.debug_info_flags
clang_cxxflags = config.cxx_mode_flags + clang_cflags
lsan_incdir = config.test_source_root + "/../"
clang_lsan_cflags = clang_cflags + lsan_cflags + ["-I%s" % lsan_incdir]
clang_lsan_cxxflags = clang_cxxflags + lsan_cflags + ["-I%s" % lsan_incdir]

config.clang_cflags = clang_cflags
config.clang_cxxflags = clang_cxxflags

def build_invocation(compile_flags):
  return " " + " ".join([config.clang] + compile_flags) + " "

config.substitutions.append( ("%clang ", build_invocation(clang_cflags)) )
config.substitutions.append( ("%clangxx ", build_invocation(clang_cxxflags)) )
config.substitutions.append( ("%clang_lsan ", build_invocation(clang_lsan_cflags)) )
config.substitutions.append( ("%clangxx_lsan ", build_invocation(clang_lsan_cxxflags)) )

# LeakSanitizer tests are currently supported on x86-64 Linux, PowerPC64 Linux, arm Linux, mips64 Linux, and x86_64 Darwin.
supported_linux = config.host_os is 'Linux' and config.host_arch in ['x86_64', 'ppc64', 'ppc64le', 'mips64', 'arm', 'armhf', 'armv7l']
supported_darwin = config.host_os is 'Darwin' and config.target_arch is 'x86_64'
supported_netbsd = config.host_os is 'NetBSD' and config.target_arch is 'x86_64'
if not (supported_linux or supported_darwin or supported_netbsd):
  config.unsupported = True

# Don't support Thumb due to broken fast unwinder
if re.search('mthumb', config.target_cflags) is not None:
  config.unsupported = True

config.suffixes = ['.c', '.cpp', '.mm']
<commit_msg>Enable LSan tests for NetBSD/i386<commit_after># -*- Python -*-

# Common configuration for running leak detection tests under LSan/ASan.

import os
import re

import lit.util

def get_required_attr(config, attr_name):
  attr_value = getattr(config, attr_name, None)
  if attr_value == None:
    lit_config.fatal(
      "No attribute %r in test configuration! You may need to run "
      "tests from your build directory or add this attribute "
      "to lit.site.cfg.py " % attr_name)
  return attr_value

# Setup source root.
config.test_source_root = os.path.dirname(__file__)

# Choose between standalone and LSan+ASan modes.
lsan_lit_test_mode = get_required_attr(config, 'lsan_lit_test_mode')
if lsan_lit_test_mode == "Standalone":
  config.name = "LeakSanitizer-Standalone"
  lsan_cflags = ["-fsanitize=leak"]
elif lsan_lit_test_mode == "AddressSanitizer":
  config.name = "LeakSanitizer-AddressSanitizer"
  lsan_cflags = ["-fsanitize=address"]
  config.available_features.add('asan')
else:
  lit_config.fatal("Unknown LSan test mode: %r" % lsan_lit_test_mode)
config.name += config.name_suffix

# Platform-specific default LSAN_OPTIONS for lit tests.
default_lsan_opts = 'detect_leaks=1'
if config.host_os == 'Darwin':
  # On Darwin, we default to `abort_on_error=1`, which would make tests run
  # much slower. Let's override this and run lit tests with 'abort_on_error=0'.
  # Also, make sure we do not overwhelm the syslog while testing.
  default_lsan_opts += ':abort_on_error=0'
  default_lsan_opts += ':log_to_syslog=0'

if default_lsan_opts:
  config.environment['LSAN_OPTIONS'] = default_lsan_opts
  default_lsan_opts += ':'
config.substitutions.append(('%env_lsan_opts=',
                             'env LSAN_OPTIONS=' + default_lsan_opts))

if lit.util.which('strace'):
  config.available_features.add('strace')

clang_cflags = ["-O0", config.target_cflags] + config.debug_info_flags
clang_cxxflags = config.cxx_mode_flags + clang_cflags
lsan_incdir = config.test_source_root + "/../"
clang_lsan_cflags = clang_cflags + lsan_cflags + ["-I%s" % lsan_incdir]
clang_lsan_cxxflags = clang_cxxflags + lsan_cflags + ["-I%s" % lsan_incdir]

config.clang_cflags = clang_cflags
config.clang_cxxflags = clang_cxxflags

def build_invocation(compile_flags):
  return " " + " ".join([config.clang] + compile_flags) + " "

config.substitutions.append( ("%clang ", build_invocation(clang_cflags)) )
config.substitutions.append( ("%clangxx ", build_invocation(clang_cxxflags)) )
config.substitutions.append( ("%clang_lsan ", build_invocation(clang_lsan_cflags)) )
config.substitutions.append( ("%clangxx_lsan ", build_invocation(clang_lsan_cxxflags)) )

# LeakSanitizer tests are currently supported on x86-64 Linux, PowerPC64 Linux, arm Linux, mips64 Linux, and x86_64 Darwin.
supported_linux = config.host_os is 'Linux' and config.host_arch in ['x86_64', 'ppc64', 'ppc64le', 'mips64', 'arm', 'armhf', 'armv7l']
supported_darwin = config.host_os is 'Darwin' and config.target_arch is 'x86_64'
supported_netbsd = config.host_os is 'NetBSD' and config.target_arch in ['x86_64', 'i386']
if not (supported_linux or supported_darwin or supported_netbsd):
  config.unsupported = True

# Don't support Thumb due to broken fast unwinder
if re.search('mthumb', config.target_cflags) is not None:
  config.unsupported = True

config.suffixes = ['.c', '.cpp', '.mm']
<|endoftext|>


<commit_before>def write_line(header, num_indent, code):
    ''' Write the line of code with num_indent number of indents. '''
    header.write(' ' * 4 * num_indent + code + '\n')

def write_subblocks(header, mat_name, dim1, dim2):
    '''
    Write subblocks of matrix of dimensions dim1 x dim2.
    mat_name is the name of the matrix, e.g., 'A' or 'B'
    header is file to write to.
    '''
    x_step = '%s_x_step' % mat_name
    y_step = '%s_y_step' % mat_name
    write_line(header, 1, 'int %s = %s.m() / %d;' % (x_step, mat_name, dim1))
    write_line(header, 1, 'int %s = %s.n() / %d;' % (y_step, mat_name, dim2))
    for i in xrange(dim1):
        for j in xrange(dim2):
            write_line(header, 1, 'Matrix<Scalar> %s%d%d(%s.data() + %d * %s + %d * %s * %s.stride(), %s.stride(), %s, %s);' % (
                    mat_name, i + 1, j + 1, mat_name, i, x_step, j, y_step, mat_name, mat_name, x_step, y_step))

def read_coeffs(filename):
    '''
    Read the coefficient file.  There is one group of coefficients for each
    of the three matrices.
    '''
    coeffs = []
    with open(filename, 'r') as coeff_file:
        curr_group = []
        for line in coeff_file:
            if line[0] == '#':
                coeffs.append(curr_group)
                curr_group = []
            else:
                curr_group.append([float(val) for val in line.split()])
    coeffs.append(curr_group)
    # There should be three sets of coefficients: one for each matrix.
    if (len(coeffs) != 3):
        raise Exception('Expected three sets of coefficients!')
    return coeffs

def write_matmul(header, ind, a_coeffs, b_coeffs):
    ''' Write a matrix multiplication call.

    header is the file to which we are writing
    ind is which multiplication this is (1, 2, 3, ...)
    a_coeffs are the coefficients for the A matrix
    b_coeffs are the coefficients for the B matrix
    '''
    comment = '// M%d = (' % (ind + 1)
    comment += ' + '.join(['%d * A%d' % (c, i + 1) for i, c in enumerate(a_coeffs) if c != 0])
    comment += ') * ('
    comment += ' + '.join(['%d * B%d' % (c, i + 1) for i, c in enumerate(b_coeffs) if c != 0])
    comment += ')'
    write_line(header, 1, comment)

    # Cilk wrapper (start)
    write_line(header, 0, '#ifdef _CILK_')
    write_line(header, 1, 'cilk_spawn [&]{')
    write_line(header, 0, '#endif')

    def linear2cart(ind, rows, cols):
        ''' Given a linear index ind from [0, 1, 2, ..., rows * cols - 1],
        return the corresponding (row, col) index from
        [1, 2, ..., row] x [1, 2, ..., col].
        Ordering of the linear index is assumed to be _row_ major.
        '''
        return ((ind / rows) + 1, (ind % cols) + 1)

    def num_nonzero(arr):
        ''' Returns number of non-zero entries in the array arr. '''
        return len(filter(lambda x: x != 0, arr))

    def write_addition(coeffs, mat_name, mat_dims):
        if num_nonzero(coeffs) > 1:
            tmp_mat = 'M%d%s' % (ind + 1, mat_name)
            write_line(header, 1, 'Matrix<Scalar> %s(%s11.n());' % (
                    tmp_mat, mat_name))
            add = 'Add('
            for i, coeff in enumerate(coeffs):
                if coeff != 0:
                    add += mat_name + '%d%d, ' % linear2cart(i, mat_dims[0], mat_dims[1])
            for i, coeff in enumerate(coeffs):
                if coeff != 0:
                    add += 'Scalar(%g), ' % coeff
            add += tmp_mat + ');'
            write_line(header, 1, add)

    write_addition(a_coeffs, 'A', (dims[0], dims[1]))
    write_addition(b_coeffs, 'B', (dims[1], dims[2]))

    def subblock_name(coeffs, mat_name):
        if (num_nonzero(coeffs) > 1):
            name = 'M%d%s' % (ind + 1, mat_name)
        else:
            loc = [i for i, c in enumerate(coeffs) if c != 0]
            name = '%s%d' % (mat_name, loc[0] + 1)
        return name

    write_line(header, 1, 'FastMatmul(%s, %s, M%d, numsteps - 1);' % (
            subblock_name(a_coeffs, 'A'), subblock_name(b_coeffs, 'B'), ind + 1))

    if (num_nonzero(a_coeffs) > 1):
        write_line(header, 1, 'M%dA.deallocate();' % (ind + 1))
    if (num_nonzero(b_coeffs) > 1):
        write_line(header, 1, 'M%dB.deallocate();' % (ind + 1))

    # Cilk wrapper (end)
    write_line(header, 0, '#ifdef _CILK_')
    write_line(header, 1, '}();')
    write_line(header, 0, '#endif\n')

outfile = 'output/fast.hpp'
coeffs = read_coeffs('grey-strassen')
dims = (2, 2, 2)
print 'Generating code for %d x %d x %d' % dims

with open(outfile, 'w') as header:
    # header information
    write_line(header, 0, '#ifndef _FAST_HPP_')
    write_line(header, 0, '#define _FAST_HPP_')
    write_line(header, 0, '\n')
    write_line(header, 0, '#include "linalg.hpp"')
    write_line(header, 0, '#ifdef _CILK_')
    write_line(header, 0, '# include <cilk/cilk.h>')
    write_line(header, 0, '#endif')
    write_line(header, 0, '\n')

    write_line(header, 0, 'template <typename Scalar>')
    write_line(header, 0, 'void FastMatmul(Matrix<Scalar>& A, Matrix<Scalar>& B, Matrix<Scalar>& C, int numsteps) {')
    write_line(header, 1, '// Base case for recursion')
    write_line(header, 1, 'if (numsteps == 0) {')
    write_line(header, 2, 'Gemm(A, B, C);')
    write_line(header, 2, 'return;')
    write_line(header, 1, '}')

    # Generate sub-blocks
    write_subblocks(header, 'A', dims[0], dims[1])
    write_subblocks(header, 'B', dims[1], dims[2])
    write_subblocks(header, 'C', dims[0], dims[2])

    num_multiplies = len(coeffs[0][0])
    print '%d matrix multiplications...' % num_multiplies

    # Declaration of all of the intermediate multiplication results.
    write_line(header, 0, '\n')
    write_line(header, 1, '// These are the intermediate matrices.')
    write_line(header, 1, '// We define them here so that they can be used')
    write_line(header, 1, '// inside the lambda functions for Cilk.')
    for i in xrange(num_multiplies):
        write_line(header, 1, 'Matrix<Scalar> M%d(C_x_step, C_y_step);' % (i + 1))
    write_line(header, 0, '\n')

    # Write the mutliplication blocks.
    for i in xrange(num_multiplies):
        a_coeffs = [c[i] for c in coeffs[0]]
        b_coeffs = [c[i] for c in coeffs[1]]
        write_matmul(header, i, a_coeffs, b_coeffs)

    # end of function
    write_line(header, 0, '}\n')

    write_line(header, 0, '#endif  // _FAST_HPP_')
<commit_msg>Additions for the output matrix C.<commit_after>def write_line(header, num_indent, code):
    ''' Write the line of code with num_indent number of indents. '''
    header.write(' ' * 4 * num_indent + code + '\n')

def write_subblocks(header, mat_name, dim1, dim2):
    '''
    Write subblocks of matrix of dimensions dim1 x dim2.
    mat_name is the name of the matrix, e.g., 'A' or 'B'
    header is file to write to.
    '''
    x_step = '%s_x_step' % mat_name
    y_step = '%s_y_step' % mat_name
    write_line(header, 1, 'int %s = %s.m() / %d;' % (x_step, mat_name, dim1))
    write_line(header, 1, 'int %s = %s.n() / %d;' % (y_step, mat_name, dim2))
    for i in xrange(dim1):
        for j in xrange(dim2):
            write_line(header, 1, 'Matrix<Scalar> %s%d%d(%s.data() + %d * %s + %d * %s * %s.stride(), %s.stride(), %s, %s);' % (
                    mat_name, i + 1, j + 1, mat_name, i, x_step, j, y_step, mat_name, mat_name, x_step, y_step))

def read_coeffs(filename):
    ''' Read the coefficient file.  There is one group of coefficients for each
    of the three matrices.  '''
    coeffs = []
    with open(filename, 'r') as coeff_file:
        curr_group = []
        for line in coeff_file:
            if line[0] == '#':
                coeffs.append(curr_group)
                curr_group = []
            else:
                curr_group.append([float(val) for val in line.split()])
    coeffs.append(curr_group)
    # There should be three sets of coefficients: one for each matrix.
    if (len(coeffs) != 3):
        raise Exception('Expected three sets of coefficients!')
    return coeffs

def addition(ind, coeffs, mat_name, mat_dims):
    if num_nonzero(coeffs) > 1:
        tmp_mat = 'M%d%s' % (ind + 1, mat_name)
        write_line(header, 1, 'Matrix<Scalar> %s(%s11.n());' % (
                tmp_mat, mat_name))
        add = 'Add('
        for i, coeff in enumerate(coeffs):
            if coeff != 0:
                add += mat_name + '%d%d, ' % linear2cart(i, mat_dims[0], mat_dims[1])
        for i, coeff in enumerate(coeffs):
            if coeff != 0:
                add += 'Scalar(%g), ' % coeff
        return add + tmp_mat + ');'
    return ''

def num_nonzero(arr):
    ''' Returns number of non-zero entries in the array arr. '''
    return len(filter(lambda x: x != 0, arr))

def linear2cart(ind, rows, cols):
    ''' Given a linear index ind from [0, 1, 2, ..., rows * cols - 1],
    return the corresponding (row, col) index from
    [1, 2, ..., row] x [1, 2, ..., col].
    Ordering of the linear index is assumed to be _row_ major.
    '''
    return ((ind / rows) + 1, (ind % cols) + 1)

def write_matmul(header, ind, a_coeffs, b_coeffs):
    ''' Write a matrix multiplication call.

    header is the file to which we are writing
    ind is which multiplication this is (1, 2, 3, ...)
    a_coeffs are the coefficients for the A matrix
    b_coeffs are the coefficients for the B matrix
    '''
    comment = '// M%d = (' % (ind + 1)
    comment += ' + '.join([str(c) + ' * A%d%d' % linear2cart(i, dims[0], dims[1]) \
                               for i, c in enumerate(a_coeffs) if c != 0])
    comment += ') * ('
    comment += ' + '.join([str(c) + ' * B%d%d' % linear2cart(i, dims[1], dims[2]) \
                               for i, c in enumerate(b_coeffs) if c != 0])
    comment += ')'
    write_line(header, 1, comment)

    # Cilk wrapper (start)
    write_line(header, 0, '#ifdef _CILK_')
    write_line(header, 1, 'cilk_spawn [&]{')
    write_line(header, 0, '#endif')

    write_line(header, 1, addition(ind, a_coeffs, 'A', (dims[0], dims[1])))
    write_line(header, 1, addition(ind, b_coeffs, 'B', (dims[1], dims[2])))

    def subblock_name(coeffs, mat_name):
        if (num_nonzero(coeffs) > 1):
            name = 'M%d%s' % (ind + 1, mat_name)
        else:
            loc = [i for i, c in enumerate(coeffs) if c != 0]
            name = '%s%d' % (mat_name, loc[0] + 1)
        return name

    write_line(header, 1, 'FastMatmul(%s, %s, M%d, numsteps - 1);' % (
            subblock_name(a_coeffs, 'A'), subblock_name(b_coeffs, 'B'), ind + 1))

    if (num_nonzero(a_coeffs) > 1):
        write_line(header, 1, 'M%dA.deallocate();' % (ind + 1))
    if (num_nonzero(b_coeffs) > 1):
        write_line(header, 1, 'M%dB.deallocate();' % (ind + 1))

    # Cilk wrapper (end)
    write_line(header, 0, '#ifdef _CILK_')
    write_line(header, 1, '}();')
    write_line(header, 0, '#endif\n')

def write_output(header, ind, coeffs, mat_dims):
    add = 'Add('
    for i, coeff in enumerate(coeffs):
        if coeff != 0:
            add += 'M%d, ' % (i + 1)
    for i, coeff in enumerate(coeffs):
        if coeff != 0:
            add += 'Scalar(%g), ' % coeff
    add += 'C%d%d);' % linear2cart(ind, mat_dims[0], mat_dims[1])
    write_line(header, 1, add)

outfile = 'output/fast.hpp'
coeffs = read_coeffs('grey-strassen')
dims = (2, 2, 2)
print 'Generating code for %d x %d x %d' % dims

with open(outfile, 'w') as header:
    # header information
    write_line(header, 0, '#ifndef _FAST_HPP_')
    write_line(header, 0, '#define _FAST_HPP_')
    write_line(header, 0, '\n')
    write_line(header, 0, '#include "linalg.hpp"')
    write_line(header, 0, '#ifdef _CILK_')
    write_line(header, 0, '# include <cilk/cilk.h>')
    write_line(header, 0, '#endif')
    write_line(header, 0, '\n')

    write_line(header, 0, 'template <typename Scalar>')
    write_line(header, 0, 'void FastMatmul(Matrix<Scalar>& A, Matrix<Scalar>& B, ' +
                          'Matrix<Scalar>& C, int numsteps) {')
    write_line(header, 1, '// Base case for recursion')
    write_line(header, 1, 'if (numsteps == 0) {')
    write_line(header, 2, 'Gemm(A, B, C);')
    write_line(header, 2, 'return;')
    write_line(header, 1, '}\n')

    # Generate sub-blocks
    write_subblocks(header, 'A', dims[0], dims[1])
    write_subblocks(header, 'B', dims[1], dims[2])
    write_subblocks(header, 'C', dims[0], dims[2])

    num_multiplies = len(coeffs[0][0])
    print '%d matrix multiplications...' % num_multiplies

    # Declaration of all of the intermediate multiplication results.
    write_line(header, 0, '\n')
    write_line(header, 1, '// These are the intermediate matrices.')
    write_line(header, 1, '// We define them here so that they can be used')
    write_line(header, 1, '// inside the lambda functions for Cilk.')
    for i in xrange(num_multiplies):
        write_line(header, 1, 'Matrix<Scalar> M%d(C_x_step, C_y_step);' % (i + 1))
    write_line(header, 0, '\n')

    # Write the mutliplication blocks.
    for i in xrange(num_multiplies):
        a_coeffs = [c[i] for c in coeffs[0]]
        b_coeffs = [c[i] for c in coeffs[1]]
        write_matmul(header, i, a_coeffs, b_coeffs)

    for ind, row in enumerate(coeffs[2]):
        write_output(header, ind, row, (dims[0], dims[2]))

    # end of function
    write_line(header, 0, '}\n')

    # end of file
    write_line(header, 0, '#endif  // _FAST_HPP_')
<|endoftext|>


<commit_before>import os
import re
from calendar import timegm
from collections import defaultdict
from itertools import cycle
from multiprocessing import Pool, cpu_count

import matplotlib
matplotlib.use("Agg")
matplotlib.rcParams.update({"font.size": 5})
matplotlib.rcParams.update({"lines.linewidth": 0.5})
matplotlib.rcParams.update({"lines.marker": "."})
matplotlib.rcParams.update({"lines.markersize": 3})
matplotlib.rcParams.update({"lines.linestyle": 'None'})
matplotlib.rcParams.update({"axes.linewidth": 0.5})
matplotlib.rcParams.update({"axes.grid": True})
matplotlib.rcParams.update({"axes.formatter.limits": (-6, 6)})
matplotlib.rcParams.update({"legend.numpoints": 1})
matplotlib.rcParams.update({"legend.fancybox": True})
matplotlib.rcParams.update({"legend.markerscale": 1.5})
matplotlib.rcParams.update({"legend.loc": 0})
matplotlib.rcParams.update({"legend.frameon": True})
import matplotlib.pyplot as plt

import numpy as np
import pandas as pd
import seriesly
from django.conf import settings
from eventlet import GreenPool
from scipy import stats

from cbmonitor import constants, models

class Colors(object):

    def __init__(self):
        self.cycle = cycle(constants.PALETTE)

    def next(self):
        return self.cycle.next()

# Defined externally in order to be pickled
def save_png(filename, series, ylabel, labels, chart_id, rebalances):
    fig = plt.figure(figsize=(4.66, 2.625))

    colors = Colors()

    ax = fig.add_subplot(1, 1, 1)
    ax.ticklabel_format(useOffset=False)

    if chart_id in ("_lt90", "_gt90", "_histo"):
        ax.set_ylabel(ylabel)
        ax.set_xlabel("Percentile")
        width = cycle((0.6, 0.4))
        align = cycle(("edge", "center"))
        if chart_id == "_lt90":
            percentiles = range(1, 90)
            x = percentiles
            plt.xlim(0, 90)
        elif chart_id == "_gt90":
            percentiles = (90, 95, 97.5, 99, 99.9, 99.99, 99.999)
            x = range(len(percentiles))
            plt.xticks(x, percentiles)
        else:
            percentiles = range(1, 100)
            x = percentiles
            plt.xlim(0, 100)
        for i, s in enumerate(series):
            y = np.percentile(s.values, percentiles)
            ax.bar(x, y, linewidth=0.0, label=labels[i],
                   width=width.next(), align=align.next(), color=colors.next())
    elif chart_id == "_subplot":
        ax.set_ylabel(ylabel)
        map(lambda s: s.set_color('none'), ax.spines.values())
        ax.tick_params(top='off', bottom='off', left='off', right='off',
                       labelcolor='w')
        ax.grid(None)

        for i, s in enumerate(series):
            color = colors.next()

            ax = fig.add_subplot(2, 1, 1)
            ax.plot(s.index, s.values, label=labels[i], color=color)
            plt.setp(ax.get_xticklabels(), visible=False)
            ymin, ymax = ax.get_ylim()
            plt.ylim(ymin=0, ymax=max(1, ymax * 1.05))

            ax = fig.add_subplot(2, 1, 2)
            rolling_median = pd.rolling_median(s, window=5)
            ax.plot(s.index, rolling_median, label=labels[i], color=color)
            ax.set_xlabel("Time elapsed, sec")
            ymin, ymax = ax.get_ylim()
            plt.ylim(ymin=0, ymax=max(1, ymax * 1.05))
    elif chart_id == "_kde":
        ax.set_ylabel("Kernel density estimation")
        ax.set_xlabel(ylabel)
        for i, s in enumerate(series):
            x = np.linspace(0, int(s.quantile(0.99)), 200)
            kde = stats.kde.gaussian_kde(s.values)
            ax.plot(x, kde(x), label=labels[i], color=colors.next())
    else:
        ax.set_ylabel(ylabel)
        ax.set_xlabel("Time elapsed, sec")
        for i, s in enumerate(series):
            ax.plot(s.index, s.values, label=labels[i], color=colors.next())
        ymin, ymax = ax.get_ylim()
        plt.ylim(ymin=0, ymax=max(1, ymax * 1.05))

        colors = Colors()
        for rebalance_start, rebalance_end in rebalances:
            plt.axvspan(rebalance_start, rebalance_end,
                        facecolor=colors.next(), alpha=0.1, linewidth=0.5)

    legend = ax.legend()
    legend.get_frame().set_linewidth(0.5)

    fig.tight_layout()
    fig.savefig(filename, dpi=200)
    plt.close()

class Plotter(object):

    def __init__(self):
        self.db = seriesly.Seriesly()
        self.all_dbs = self.db.list_dbs()

        self.urls = list()
        self.images = list()

        self.eventlet_pool = GreenPool()
        self.mp_pool = Pool(cpu_count())

    def __del__(self):
        self.mp_pool.close()

    @staticmethod
    def build_dbname(cluster, server, bucket, collector):
        db_name = (collector or "") + cluster + (bucket or "") + (server or "")
        for char in "[]/\;.,><&*:%=+@!#^()|?^'\"":
            db_name = db_name.replace(char, "")
        return db_name

    def query_data(self, snapshot, cluster, server, bucket, metric, collector):
        query_params = {"ptr": "/{}".format(metric), "reducer": "avg",
                        "group": 5000}
        if snapshot.name != "all_data":
            ts_from = timegm(snapshot.ts_from.timetuple()) * 1000
            ts_to = timegm(snapshot.ts_to.timetuple()) * 1000
            group = max((ts_from - ts_to) / 500, 5000)  # min 5s; max 500 points
            query_params.update({"group": group, "from": ts_from, "to": ts_to})
        db_name = self.build_dbname(cluster, server, bucket, collector)
        if db_name in self.all_dbs:
            try:
                return self.db[db_name].query(query_params)
            except seriesly.exceptions.ConnectionError:
                return
        else:
            return

    def generate_png_meta(self, snapshot, cluster, server, bucket, metric):
        metric = metric.replace("/", "_")
        title = "{}] {}".format(bucket, metric)  # [server bucket] metric
        if server:
            title = "[{} {}".format(server, title)
        else:
            title = "[" + title

        filename = "".join((snapshot.name, cluster, title))
        filename = re.sub(r"[\[\]/\\:\*\?\"<>\|& ]", "", filename)
        filename += "{suffix}.png"

        media_url = settings.MEDIA_URL + filename
        media_path = os.path.join(settings.MEDIA_ROOT, filename)
        return title, media_url, media_path

    def get_series(self, metric, data):
        data = {k: v[0] for k, v in data.iteritems()}
        series = pd.Series(data)
        series.dropna()  # otherwise it may break kde
        if metric in constants.NON_ZERO_VALUES and (series == 0).all():
            return None
        series.index = series.index.astype("uint64")
        series.rename(lambda x: x - series.index.values.min(), inplace=True)
        series.rename(lambda x: x / 1000, inplace=True)  # ms -> s
        return series

    def extract_meta(self, metric):
        if metric.bucket_id:
            bucket = str(models.Bucket.objects.get(id=metric.bucket_id))
        else:
            bucket = ""
        if metric.server_id:
            server = str(models.Server.objects.get(id=metric.server_id))
        else:
            server = ""
        cluster = metric.cluster_id
        name = metric.name
        collector = metric.collector

        return cluster, server, bucket, name, collector

    def extract(self, meta):
        merge = defaultdict(list)
        merge_cluster = server = bucket = name = snapshot = ""
        for sub_metric, snapshot in meta:
            cluster, server, bucket, name, collector = self.extract_meta(
                sub_metric)
            data = self.query_data(snapshot, cluster, server, bucket, name,
                                   collector)
            if data:
                series = self.get_series(metric=name, data=data)
                if series is not None:
                    merge["series"].append(series)
                    if snapshot.name == "all_data":
                        merge["labels"].append(cluster)
                    else:
                        merge["labels"].append(snapshot.name)
            merge_cluster += cluster
        title, url, filename = self.generate_png_meta(snapshot, merge_cluster,
                                                      server, bucket, name)

        return merge["series"], merge["labels"], title, filename, url

    def detect_rebalance(self, observables):
        rebalances = []
        if observables[0][0].name == "rebalance_progress":
            series, _, _, _, _ = self.extract(observables)
            for s in series:
                s = s.dropna()
                rebalance = s[s > 0]
                rebalances.append((rebalance.index[0], rebalance.index[-1]))
        return rebalances

    def plot(self, metrics):
        apply_results = list()
        metrics = tuple(metrics)
        rebalances = self.detect_rebalance(metrics[0])

        for data in self.eventlet_pool.imap(self.extract, metrics):
            series, labels, title, filename, url = data
            metric = title.split()[-1]
            if series:
                ylabel = constants.LABELS.get(metric, metric)

                chart_ids = [""]
                if metric in constants.HISTOGRAMS:
                    chart_ids += ["_histo"]
                if metric in constants.ZOOM_HISTOGRAMS:
                    chart_ids += ["_lt90", "_gt90"]
                if metric in constants.KDE:
                    chart_ids += ["_kde"]
                if metric in constants.SMOOTH_SUBPLOTS:
                    chart_ids[0] = "_subplot"

                if not os.path.exists(filename):
                    for chart_id in chart_ids:
                        apply_results.append(self.mp_pool.apply_async(
                            save_png,
                            args=(filename.format(suffix=chart_id),
                                  series, ylabel, labels, chart_id, rebalances)
                        ))
                for chart_id in chart_ids:
                    self.urls.append([title, url.format(suffix=chart_id)])
                    self.images.append(filename.format(suffix=chart_id))
        for result in apply_results:
            result.get()
<commit_msg>add charts with rolling percentile of score<commit_after>import os
import re
from calendar import timegm
from collections import defaultdict
from itertools import cycle
from multiprocessing import Pool, cpu_count

import matplotlib
matplotlib.use("Agg")
matplotlib.rcParams.update({"font.size": 5})
matplotlib.rcParams.update({"lines.linewidth": 0.5})
matplotlib.rcParams.update({"lines.marker": "."})
matplotlib.rcParams.update({"lines.markersize": 3})
matplotlib.rcParams.update({"lines.linestyle": 'None'})
matplotlib.rcParams.update({"axes.linewidth": 0.5})
matplotlib.rcParams.update({"axes.grid": True})
matplotlib.rcParams.update({"axes.formatter.limits": (-6, 6)})
matplotlib.rcParams.update({"legend.numpoints": 1})
matplotlib.rcParams.update({"legend.fancybox": True})
matplotlib.rcParams.update({"legend.markerscale": 1.5})
matplotlib.rcParams.update({"legend.loc": 0})
matplotlib.rcParams.update({"legend.frameon": True})
import matplotlib.pyplot as plt

import numpy as np
import pandas as pd
import seriesly
from django.conf import settings
from eventlet import GreenPool
from scipy import stats

from cbmonitor import constants, models

class Colors(object):

    def __init__(self):
        self.cycle = cycle(constants.PALETTE)

    def next(self):
        return self.cycle.next()

# Defined externally in order to be pickled
def save_png(filename, series, ylabel, labels, chart_id, rebalances):
    fig = plt.figure(figsize=(4.66, 2.625))

    colors = Colors()

    ax = fig.add_subplot(1, 1, 1)
    ax.ticklabel_format(useOffset=False)

    if chart_id in ("_lt90", "_gt90", "_histo"):
        ax.set_ylabel(ylabel)
        ax.set_xlabel("Percentile")
        width = cycle((0.6, 0.4))
        align = cycle(("edge", "center"))
        if chart_id == "_lt90":
            percentiles = range(1, 90)
            x = percentiles
            plt.xlim(0, 90)
        elif chart_id == "_gt90":
            percentiles = (90, 95, 97.5, 99, 99.9, 99.99, 99.999)
            x = range(len(percentiles))
            plt.xticks(x, percentiles)
        else:
            percentiles = range(1, 100)
            x = percentiles
            plt.xlim(0, 100)
        for i, s in enumerate(series):
            y = np.percentile(s, percentiles)
            ax.bar(x, y, linewidth=0.0, label=labels[i],
                   width=width.next(), align=align.next(), color=colors.next())
    elif chart_id == "_subplot":
        ax.set_ylabel(ylabel)
        map(lambda s: s.set_color('none'), ax.spines.values())
        ax.tick_params(top='off', bottom='off', left='off', right='off',
                       labelcolor='w')
        ax.grid(None)

        for i, s in enumerate(series):
            color = colors.next()

            ax = fig.add_subplot(2, 1, 1)
            ax.plot(s.index, s, label=labels[i], color=color)
            plt.setp(ax.get_xticklabels(), visible=False)
            ymin, ymax = ax.get_ylim()
            plt.ylim(ymin=0, ymax=max(1, ymax * 1.05))

            ax = fig.add_subplot(2, 1, 2)
            rolling_median = pd.rolling_median(s, window=5)
            ax.plot(s.index, rolling_median, label=labels[i], color=color)
            ax.set_xlabel("Time elapsed, sec")
            ymin, ymax = ax.get_ylim()
            plt.ylim(ymin=0, ymax=max(1, ymax * 1.05))
    elif chart_id == "_kde":
        ax.set_ylabel("Kernel density estimation")
        ax.set_xlabel(ylabel)
        for i, s in enumerate(series):
            x = np.linspace(0, int(s.quantile(0.99)), 200)
            kde = stats.kde.gaussian_kde(s)
            ax.plot(x, kde(x), label=labels[i], color=colors.next())
    elif chart_id == "_score":
        ax.set_ylabel("Percentile of score ({})".format(ylabel))
        ax.set_xlabel("Time elapsed, sec")
        for i, s in enumerate(series):
            scoref = lambda x: stats.percentileofscore(x, s.quantile(0.9))
            rolling_score = pd.rolling_apply(s, min(len(s) / 15, 40), scoref)
            ax.plot(s.index, rolling_score, label=labels[i],
                    color=colors.next())
            plt.ylim(ymin=0, ymax=105)
    else:
        ax.set_ylabel(ylabel)
        ax.set_xlabel("Time elapsed, sec")
        for i, s in enumerate(series):
            ax.plot(s.index, s, label=labels[i], color=colors.next())
        ymin, ymax = ax.get_ylim()
        plt.ylim(ymin=0, ymax=max(1, ymax * 1.05))

        colors = Colors()
        for rebalance_start, rebalance_end in rebalances:
            plt.axvspan(rebalance_start, rebalance_end,
                        facecolor=colors.next(), alpha=0.1, linewidth=0.5)

    legend = ax.legend()
    legend.get_frame().set_linewidth(0.5)

    fig.tight_layout()
    fig.savefig(filename, dpi=200)
    plt.close()

class Plotter(object):

    def __init__(self):
        self.db = seriesly.Seriesly()
        self.all_dbs = self.db.list_dbs()

        self.urls = list()
        self.images = list()

        self.eventlet_pool = GreenPool()
        self.mp_pool = Pool(cpu_count())

    def __del__(self):
        self.mp_pool.close()

    @staticmethod
    def build_dbname(cluster, server, bucket, collector):
        db_name = (collector or "") + cluster + (bucket or "") + (server or "")
        for char in "[]/\;.,><&*:%=+@!#^()|?^'\"":
            db_name = db_name.replace(char, "")
        return db_name

    def query_data(self, snapshot, cluster, server, bucket, metric, collector):
        query_params = {"ptr": "/{}".format(metric), "reducer": "avg",
                        "group": 5000}
        if snapshot.name != "all_data":
            ts_from = timegm(snapshot.ts_from.timetuple()) * 1000
            ts_to = timegm(snapshot.ts_to.timetuple()) * 1000
            group = max((ts_from - ts_to) / 500, 5000)  # min 5s; max 500 points
            query_params.update({"group": group, "from": ts_from, "to": ts_to})
        db_name = self.build_dbname(cluster, server, bucket, collector)
        if db_name in self.all_dbs:
            try:
                return self.db[db_name].query(query_params)
            except seriesly.exceptions.ConnectionError:
                return
        else:
            return

    def generate_png_meta(self, snapshot, cluster, server, bucket, metric):
        metric = metric.replace("/", "_")
        title = "{}] {}".format(bucket, metric)  # [server bucket] metric
        if server:
            title = "[{} {}".format(server, title)
        else:
            title = "[" + title

        filename = "".join((snapshot.name, cluster, title))
        filename = re.sub(r"[\[\]/\\:\*\?\"<>\|& ]", "", filename)
        filename += "{suffix}.png"

        media_url = settings.MEDIA_URL + filename
        media_path = os.path.join(settings.MEDIA_ROOT, filename)
        return title, media_url, media_path

    def get_series(self, metric, data):
        data = {k: v[0] for k, v in data.iteritems()}
        series = pd.Series(data)
        series.dropna()  # otherwise it may break kde
        if metric in constants.NON_ZERO_VALUES and (series == 0).all():
            return None
        series.index = series.index.astype("uint64")
        series.rename(lambda x: x - series.index.values.min(), inplace=True)
        series.rename(lambda x: x / 1000, inplace=True)  # ms -> s
        return series

    def extract_meta(self, metric):
        if metric.bucket_id:
            bucket = str(models.Bucket.objects.get(id=metric.bucket_id))
        else:
            bucket = ""
        if metric.server_id:
            server = str(models.Server.objects.get(id=metric.server_id))
        else:
            server = ""
        cluster = metric.cluster_id
        name = metric.name
        collector = metric.collector

        return cluster, server, bucket, name, collector

    def extract(self, meta):
        merge = defaultdict(list)
        merge_cluster = server = bucket = name = snapshot = ""
        for sub_metric, snapshot in meta:
            cluster, server, bucket, name, collector = self.extract_meta(
                sub_metric)
            data = self.query_data(snapshot, cluster, server, bucket, name,
                                   collector)
            if data:
                series = self.get_series(metric=name, data=data)
                if series is not None:
                    merge["series"].append(series)
                    if snapshot.name == "all_data":
                        merge["labels"].append(cluster)
                    else:
                        merge["labels"].append(snapshot.name)
            merge_cluster += cluster
        title, url, filename = self.generate_png_meta(snapshot, merge_cluster,
                                                      server, bucket, name)

        return merge["series"], merge["labels"], title, filename, url

    def detect_rebalance(self, observables):
        rebalances = []
        if observables[0][0].name == "rebalance_progress":
            series, _, _, _, _ = self.extract(observables)
            for s in series:
                s = s.dropna()
                rebalance = s[s > 0]
                rebalances.append((rebalance.index[0], rebalance.index[-1]))
        return rebalances

    def plot(self, metrics):
        apply_results = list()
        metrics = tuple(metrics)
        rebalances = self.detect_rebalance(metrics[0])

        for data in self.eventlet_pool.imap(self.extract, metrics):
            series, labels, title, filename, url = data
            metric = title.split()[-1]
            if series:
                ylabel = constants.LABELS.get(metric, metric)

                chart_ids = [""]
                if metric in constants.HISTOGRAMS:
                    chart_ids += ["_histo"]
                if metric in constants.ZOOM_HISTOGRAMS:
                    chart_ids += ["_lt90", "_gt90"]
                if metric in constants.KDE:
                    chart_ids += ["_kde"]
                if metric in constants.SMOOTH_SUBPLOTS:
                    chart_ids[0] = "_subplot"
                    chart_ids += ["_score"]

                if not os.path.exists(filename):
                    for chart_id in chart_ids:
                        apply_results.append(self.mp_pool.apply_async(
                            save_png,
                            args=(filename.format(suffix=chart_id),
                                  series, ylabel, labels, chart_id, rebalances)
                        ))
                for chart_id in chart_ids:
                    self.urls.append([title, url.format(suffix=chart_id)])
                    self.images.append(filename.format(suffix=chart_id))
        for result in apply_results:
            result.get()
<|endoftext|>


<commit_before>#!/usr/bin/env python
import rospy
from std_msgs.msg import Int32
from gazebo_msgs.srv import ApplyBodyWrench
from geometry_msgs.msg import Wrench
THRUSTER_FORCE = 9.8
MOTION_LIBRARY_FREQUENCY = 0.1
ERR_FACTOR = 1000
FULL_FORCE = THRUSTER_FORCE * 2 * ERR_FACTOR

def PWMCbForward(data):
    "Apply effort on varun on recieving forward pwm"
    apply_body_wrench = rospy.ServiceProxy('/gazebo/apply_body_wrench', ApplyBodyWrench)
    body_name = "varun::body"
    wrench = Wrench()
    pwm = data.data
    if abs(pwm) > 255:
        print "pwm can't have value more than 255"
    wrench.force.x = FULL_FORCE * pwm / 255  # each thruster exerts about 1 kgf
    duration = rospy.Duration()  # this is inverse of motion library update rate
    duration.secs = MOTION_LIBRARY_FREQUENCY
    duration.nsecs = 0
    apply_body_wrench(body_name, None, None, wrench, None, duration)

def PWMCbSideward(data):
    "Apply effort on varun on recieving sideward pwm"
    apply_body_wrench = rospy.ServiceProxy('/gazebo/apply_body_wrench', ApplyBodyWrench)
    body_name = "varun::body"
    wrench = Wrench()
    pwm = data.data
    # we are multiplying pwm with -1 to change its sign
    # Since we have defined positive pwm to move the bot in right but in the simulation y axis is towards the left of the bot
    pwm = pwm * -1
    if abs(pwm) > 255:
        print "pwm can't have value more than 255"
    wrench.force.y = FULL_FORCE * pwm / 255  # each thruster exerts about 1 kgf
    duration = rospy.Duration()  # this is inverse of motion library update rate
    duration.secs = MOTION_LIBRARY_FREQUENCY
    duration.nsecs = 0
    apply_body_wrench(body_name, None, None, wrench, None, duration)

def PWMCbUpward(data):
    "Apply effort on varun on recieving upward pwm"
    apply_body_wrench = rospy.ServiceProxy('/gazebo/apply_body_wrench', ApplyBodyWrench)
    body_name = "varun::body"
    wrench = Wrench()
    pwm = data.data
    if abs(pwm) > 255:
        print "pwm can't have value more than 255"
    wrench.force.z = FULL_FORCE * pwm / 255  # each thruster exerts about 1 kgf
    duration = rospy.Duration()  # this is inverse of motion library update rate
    duration.secs = MOTION_LIBRARY_FREQUENCY
    duration.nsecs = 0
    apply_body_wrench(body_name, None, None, wrench, None, duration)

def PWMCbTurn(data):
    "Apply wrench on varun on recieving turn pwm"
    apply_body_wrench = rospy.ServiceProxy('/gazebo/apply_body_wrench', ApplyBodyWrench)
    body_name = "varun::body"
    wrench = Wrench()
    pwm = data.data
    if abs(pwm) > 255:
        print "pwm can't have value more than 255"
    wrench.torque.z = FULL_FORCE * pwm / 255  # each thruster exerts about 1 kgf
    duration = rospy.Duration()  # this is inverse of motion library update rate
    duration.secs = MOTION_LIBRARY_FREQUENCY
    duration.nsecs = 0
    apply_body_wrench(body_name, None, None, wrench, None, duration)

def send_motion():
    "sends instructions to gazebo to do the appropriate motion"
    rospy.init_node('gazebo_controller')
    rospy.wait_for_service('/gazebo/apply_body_wrench')
    rospy.Subscriber("/pwm/forward", Int32, PWMCbForward)
    rospy.Subscriber("/pwm/sideward", Int32, PWMCbSideward)
    rospy.Subscriber("/pwm/upward", Int32, PWMCbUpward)
    rospy.Subscriber("/pwm/turn", Int32, PWMCbTurn)
    rospy.spin()

if __name__ == '__main__':
    try:
        send_motion()
    except rospy.ROSInterruptException:
        pass
<commit_msg>removed python node due to slow speed<commit_after><|endoftext|>


<commit_before>import datetime
import json
import discord
from discord.ext import commands
from utils import aiohttp_wrap as aw

class Crypto(commands.Cog):
    """ Allows users to track bitcoin and other currencies (eventually) """

    URL_BTC = "https://pro-api.coinmarketcap.com/v1/cryptocurrency/quotes/latest"
    BTC_LOGO_URL = "https://en.bitcoin.it/w/images/en/2/29/BC_Logo_.png"
    BTC_ID = 1
    CACHE_TTL = 10 * 60

    def __init__(self, bot):
        self.bot = bot
        self.session = bot.aio_session
        self.redis = bot.redis_client
        self.HEADERS = {"X-CMC_PRO_API_KEY": bot.api_keys["cmc"]}

    @commands.command(aliases=["btc", "buttcoin"])
    async def bitcoin(self, ctx):
        """ Get current information regarding the value of bitcoin """

        # Check the cache for this information
        if await self.redis.exists("btc"):
            resp = json.loads(await self.redis.get("btc"))

        # If not found, cache for 5 minutes
        else:
            resp = await aw.aio_get_json(
                self.session,
                self.URL_BTC,
                headers=self.HEADERS,
                params={"id": self.BTC_ID},
            )["data"]["quote"]["USD"]
            await self.redis.set("btc", json.dumps(resp), ex=self.CACHE_TTL)

        # Create a neat embed with the information
        em = discord.Embed(color=0xF7931A)
        em.set_author(name="Bitcoin", icon_url=self.BTC_LOGO_URL)
        em.add_field(
            name="Price USD", value=f"${float(resp['price']):,.2f}", inline=False
        )
        em.set_footer(text="Last updated")
        em.timestamp = datetime.datetime.fromtimestamp(int(resp["last_updated"]))

        # Hourly trend
        change_1h = resp["percent_change_1h"]
        change_1h_str = (
            f":arrow_up: {float(change_1h):.2f}%"
            if "-" not in change_1h
            else f":arrow_down: {float(change_1h):.2f}%"
        )
        em.add_field(name="Hourly trend", value=change_1h_str)

        # Daily trend
        change_24h = resp["percent_change_24h"]
        change_24h_str = (
            f":arrow_up: {float(change_24h):.2f}%"
            if "-" not in change_24h
            else f":arrow_down: {float(change_24h):.2f}%"
        )
        em.add_field(name="Daily trend", value=change_24h_str)

        # Weekly trend
        change_7d = resp["percent_change_7d"]
        change_7d_str = (
            f":arrow_up: {float(change_7d):.2f}%"
            if "-" not in change_7d
            else f":arrow_down: {float(change_7d):.2f}%"
        )
        em.add_field(name="Weekly trend", value=change_7d_str)

        # Ticker graph
        em.set_image(
            url="https://s2.coinmarketcap.com/generated/sparklines/web/7d/usd/1.png"
        )

        await ctx.send(embed=em)

def setup(bot):
    bot.add_cog(Crypto(bot))
<commit_msg>missed some parens<commit_after>import datetime
import json
import discord
from discord.ext import commands
from utils import aiohttp_wrap as aw

class Crypto(commands.Cog):
    """ Allows users to track bitcoin and other currencies (eventually) """

    URL_BTC = "https://pro-api.coinmarketcap.com/v1/cryptocurrency/quotes/latest"
    BTC_LOGO_URL = "https://en.bitcoin.it/w/images/en/2/29/BC_Logo_.png"
    BTC_ID = 1
    CACHE_TTL = 10 * 60

    def __init__(self, bot):
        self.bot = bot
        self.session = bot.aio_session
        self.redis = bot.redis_client
        self.HEADERS = {"X-CMC_PRO_API_KEY": bot.api_keys["cmc"]}

    @commands.command(aliases=["btc", "buttcoin"])
    async def bitcoin(self, ctx):
        """ Get current information regarding the value of bitcoin """

        # Check the cache for this information
        if await self.redis.exists("btc"):
            resp = json.loads(await self.redis.get("btc"))

        # If not found, cache for 5 minutes
        else:
            resp = (
                await aw.aio_get_json(
                    self.session,
                    self.URL_BTC,
                    headers=self.HEADERS,
                    params={"id": self.BTC_ID},
                )
            )["data"]["quote"]["USD"]
            await self.redis.set("btc", json.dumps(resp), ex=self.CACHE_TTL)

        # Create a neat embed with the information
        em = discord.Embed(color=0xF7931A)
        em.set_author(name="Bitcoin", icon_url=self.BTC_LOGO_URL)
        em.add_field(
            name="Price USD", value=f"${float(resp['price']):,.2f}", inline=False
        )
        em.set_footer(text="Last updated")
        em.timestamp = datetime.datetime.fromtimestamp(int(resp["last_updated"]))

        # Hourly trend
        change_1h = resp["percent_change_1h"]
        change_1h_str = (
            f":arrow_up: {float(change_1h):.2f}%"
            if "-" not in change_1h
            else f":arrow_down: {float(change_1h):.2f}%"
        )
        em.add_field(name="Hourly trend", value=change_1h_str)

        # Daily trend
        change_24h = resp["percent_change_24h"]
        change_24h_str = (
            f":arrow_up: {float(change_24h):.2f}%"
            if "-" not in change_24h
            else f":arrow_down: {float(change_24h):.2f}%"
        )
        em.add_field(name="Daily trend", value=change_24h_str)

        # Weekly trend
        change_7d = resp["percent_change_7d"]
        change_7d_str = (
            f":arrow_up: {float(change_7d):.2f}%"
            if "-" not in change_7d
            else f":arrow_down: {float(change_7d):.2f}%"
        )
        em.add_field(name="Weekly trend", value=change_7d_str)

        # Ticker graph
        em.set_image(
            url="https://s2.coinmarketcap.com/generated/sparklines/web/7d/usd/1.png"
        )

        await ctx.send(embed=em)

def setup(bot):
    bot.add_cog(Crypto(bot))
<|endoftext|>


<commit_before>#
#  Copyright 2019 The FATE Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#

import logging
from logging.handlers import TimedRotatingFileHandler
import os
import inspect
from threading import RLock
from arch.api.utils import file_utils

class LoggerFactory(object):
    TYPE = "FILE"
    LEVEL = logging.DEBUG
    logger_dict = {}
    global_handler_dict = {}

    LOG_DIR = None
    PARENT_LOG_DIR = None

    append_to_parent_log = None

    lock = RLock()
    # CRITICAL = 50
    # FATAL = CRITICAL
    # ERROR = 40
    # WARNING = 30
    # WARN = WARNING
    # INFO = 20
    # DEBUG = 10
    # NOTSET = 0
    levels = (10, 20, 30, 40)

    @staticmethod
    def set_directory(directory=None, parent_log_dir=None, append_to_parent_log=None, force=False):
        if parent_log_dir:
           LoggerFactory.PARENT_LOG_DIR = parent_log_dir
        if append_to_parent_log:
            LoggerFactory.append_to_parent_log = append_to_parent_log
        with LoggerFactory.lock:
            if not directory:
                directory = os.path.join(file_utils.get_project_base_directory(), 'logs')
            if not LoggerFactory.LOG_DIR or force:
                LoggerFactory.LOG_DIR = directory
            os.makedirs(LoggerFactory.LOG_DIR, exist_ok=True)
            for loggerName, ghandler in LoggerFactory.global_handler_dict.items():
                for className, (logger, handler) in LoggerFactory.logger_dict.items():
                    logger.removeHandler(ghandler)
                ghandler.close()
            LoggerFactory.global_handler_dict={}
            for className, (logger, handler) in LoggerFactory.logger_dict.items():
                logger.removeHandler(handler)
                _hanlder=None
                if handler:
                    handler.close()
                if className != "default":
                    _hanlder = LoggerFactory.get_handler(className)
                    logger.addHandler(_hanlder)
                LoggerFactory.assemble_global_handler(logger)
                LoggerFactory.logger_dict[className] = logger, _hanlder

    @staticmethod
    def get_logger(class_name=None):
        with LoggerFactory.lock:
            if class_name in LoggerFactory.logger_dict.keys():
                logger, hanlder = LoggerFactory.logger_dict[class_name]
                if not logger:
                    logger, handler = LoggerFactory.init_logger(class_name)
            else:
                logger, handler = LoggerFactory.init_logger(class_name)
            return logger

    @staticmethod
    def get_global_hanlder(logger_name, level=None, log_dir=None):
        if not LoggerFactory.LOG_DIR:
            return logging.StreamHandler()
        if log_dir:
            logger_name_key = logger_name + "_" + log_dir
        else:
            logger_name_key = logger_name + "_" + LoggerFactory.LOG_DIR
        # if loggerName not in LoggerFactory.globalHandlerDict:
        if logger_name_key not in LoggerFactory.global_handler_dict:
            with LoggerFactory.lock:
                if logger_name_key not in LoggerFactory.global_handler_dict:
                    handler = LoggerFactory.get_handler(logger_name, level, log_dir)
                    LoggerFactory.global_handler_dict[logger_name_key]=handler
        return LoggerFactory.global_handler_dict[logger_name_key]

    @staticmethod
    def get_handler(class_name, level=None, log_dir=None):
        if not LoggerFactory.LOG_DIR or not class_name:
            return logging.StreamHandler()
        formatter = logging.Formatter('"%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s"')
        if not log_dir:
            log_file = os.path.join(LoggerFactory.LOG_DIR, "{}.log".format(class_name))
        else:
            log_file = os.path.join(log_dir, "{}.log".format(class_name))
        handler = TimedRotatingFileHandler(log_file,
                                           when='H',
                                           interval=4,
                                           backupCount=7,
                                           delay=True)

        if level:
            handler.level=level

        handler.setFormatter(formatter)
        return handler

    @staticmethod
    def init_logger(class_name):
        with LoggerFactory.lock:
            logger = logging.getLogger(class_name)
            logger.setLevel(LoggerFactory.LEVEL)
            handler = None
            if class_name:
                handler = LoggerFactory.get_handler(class_name)
                logger.addHandler(handler)
                LoggerFactory.logger_dict[class_name] = logger, handler

            else:
                LoggerFactory.logger_dict["default"] = logger, handler

            LoggerFactory.assemble_global_handler(logger)

            return logger, handler

    @staticmethod
    def assemble_global_handler(logger):
        if LoggerFactory.LOG_DIR:
            for level in LoggerFactory.levels:
                if level >= LoggerFactory.LEVEL:
                    level_logger_name = logging._levelToName[level]
                    logger.addHandler(LoggerFactory.get_global_hanlder(level_logger_name, level))
        if LoggerFactory.append_to_parent_log and LoggerFactory.PARENT_LOG_DIR:
            for level in LoggerFactory.levels:
                if level >= LoggerFactory.LEVEL:
                    level_logger_name = logging._levelToName[level]
                    logger.addHandler(LoggerFactory.get_global_hanlder(level_logger_name, level, LoggerFactory.PARENT_LOG_DIR))

def setDirectory(directory=None):
    LoggerFactory.set_directory(directory)

def setLevel(level):
    LoggerFactory.LEVEL = level

def getLogger(className=None, useLevelFile=False):
    if className is None:
        frame = inspect.stack()[1]
        module = inspect.getmodule(frame[0])
        className = 'stat'
    return LoggerFactory.get_logger(className)
<commit_msg>Modify python service log cutting time to day, keep 14 days<commit_after>#
#  Copyright 2019 The FATE Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#

import logging
from logging.handlers import TimedRotatingFileHandler
import os
import inspect
from threading import RLock
from arch.api.utils import file_utils

class LoggerFactory(object):
    TYPE = "FILE"
    LEVEL = logging.DEBUG
    logger_dict = {}
    global_handler_dict = {}

    LOG_DIR = None
    PARENT_LOG_DIR = None

    append_to_parent_log = None

    lock = RLock()
    # CRITICAL = 50
    # FATAL = CRITICAL
    # ERROR = 40
    # WARNING = 30
    # WARN = WARNING
    # INFO = 20
    # DEBUG = 10
    # NOTSET = 0
    levels = (10, 20, 30, 40)

    @staticmethod
    def set_directory(directory=None, parent_log_dir=None, append_to_parent_log=None, force=False):
        if parent_log_dir:
           LoggerFactory.PARENT_LOG_DIR = parent_log_dir
        if append_to_parent_log:
            LoggerFactory.append_to_parent_log = append_to_parent_log
        with LoggerFactory.lock:
            if not directory:
                directory = os.path.join(file_utils.get_project_base_directory(), 'logs')
            if not LoggerFactory.LOG_DIR or force:
                LoggerFactory.LOG_DIR = directory
            os.makedirs(LoggerFactory.LOG_DIR, exist_ok=True)
            for loggerName, ghandler in LoggerFactory.global_handler_dict.items():
                for className, (logger, handler) in LoggerFactory.logger_dict.items():
                    logger.removeHandler(ghandler)
                ghandler.close()
            LoggerFactory.global_handler_dict={}
            for className, (logger, handler) in LoggerFactory.logger_dict.items():
                logger.removeHandler(handler)
                _hanlder=None
                if handler:
                    handler.close()
                if className != "default":
                    _hanlder = LoggerFactory.get_handler(className)
                    logger.addHandler(_hanlder)
                LoggerFactory.assemble_global_handler(logger)
                LoggerFactory.logger_dict[className] = logger, _hanlder

    @staticmethod
    def get_logger(class_name=None):
        with LoggerFactory.lock:
            if class_name in LoggerFactory.logger_dict.keys():
                logger, hanlder = LoggerFactory.logger_dict[class_name]
                if not logger:
                    logger, handler = LoggerFactory.init_logger(class_name)
            else:
                logger, handler = LoggerFactory.init_logger(class_name)
            return logger

    @staticmethod
    def get_global_hanlder(logger_name, level=None, log_dir=None):
        if not LoggerFactory.LOG_DIR:
            return logging.StreamHandler()
        if log_dir:
            logger_name_key = logger_name + "_" + log_dir
        else:
            logger_name_key = logger_name + "_" + LoggerFactory.LOG_DIR
        # if loggerName not in LoggerFactory.globalHandlerDict:
        if logger_name_key not in LoggerFactory.global_handler_dict:
            with LoggerFactory.lock:
                if logger_name_key not in LoggerFactory.global_handler_dict:
                    handler = LoggerFactory.get_handler(logger_name, level, log_dir)
                    LoggerFactory.global_handler_dict[logger_name_key]=handler
        return LoggerFactory.global_handler_dict[logger_name_key]

    @staticmethod
    def get_handler(class_name, level=None, log_dir=None):
        if not LoggerFactory.LOG_DIR or not class_name:
            return logging.StreamHandler()
        formatter = logging.Formatter('"%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s"')
        if not log_dir:
            log_file = os.path.join(LoggerFactory.LOG_DIR, "{}.log".format(class_name))
        else:
            log_file = os.path.join(log_dir, "{}.log".format(class_name))
        handler = TimedRotatingFileHandler(log_file,
                                           when='D',
                                           interval=1,
                                           backupCount=14,
                                           delay=True)

        if level:
            handler.level=level

        handler.setFormatter(formatter)
        return handler

    @staticmethod
    def init_logger(class_name):
        with LoggerFactory.lock:
            logger = logging.getLogger(class_name)
            logger.setLevel(LoggerFactory.LEVEL)
            handler = None
            if class_name:
                handler = LoggerFactory.get_handler(class_name)
                logger.addHandler(handler)
                LoggerFactory.logger_dict[class_name] = logger, handler

            else:
                LoggerFactory.logger_dict["default"] = logger, handler

            LoggerFactory.assemble_global_handler(logger)

            return logger, handler

    @staticmethod
    def assemble_global_handler(logger):
        if LoggerFactory.LOG_DIR:
            for level in LoggerFactory.levels:
                if level >= LoggerFactory.LEVEL:
                    level_logger_name = logging._levelToName[level]
                    logger.addHandler(LoggerFactory.get_global_hanlder(level_logger_name, level))
        if LoggerFactory.append_to_parent_log and LoggerFactory.PARENT_LOG_DIR:
            for level in LoggerFactory.levels:
                if level >= LoggerFactory.LEVEL:
                    level_logger_name = logging._levelToName[level]
                    logger.addHandler(LoggerFactory.get_global_hanlder(level_logger_name, level, LoggerFactory.PARENT_LOG_DIR))

def setDirectory(directory=None):
    LoggerFactory.set_directory(directory)

def setLevel(level):
    LoggerFactory.LEVEL = level

def getLogger(className=None, useLevelFile=False):
    if className is None:
        frame = inspect.stack()[1]
        module = inspect.getmodule(frame[0])
        className = 'stat'
    return LoggerFactory.get_logger(className)
<|endoftext|>