<commit_before>"""rbf - Radial basis functions for interpolation/smoothing scattered N-D data.

Written by John Travers <jtravs@gmail.com>, February 2007
Based closely on Matlab code by Alex Chirokov
Additional, large, improvements by Robert Hetland
Some additional alterations by Travis Oliphant
Interpolation with multi-dimensional target domain by Josua Sassen

Permission to use, modify, and distribute this software is given under the
terms of the SciPy (BSD style) license. See LICENSE.txt that came with
this distribution for specifics.

NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK.

Copyright (c) 2006-2007, Robert Hetland <hetland@tamu.edu>
Copyright (c) 2007, John Travers <jtravs@gmail.com>

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:

    * Redistributions of source code must retain the above copyright
       notice, this list of conditions and the following disclaimer.

    * Redistributions in binary form must reproduce the above
       copyright notice, this list of conditions and the following
       disclaimer in the documentation and/or other materials provided
       with the distribution.

    * Neither the name of Robert Hetland nor the names of any
       contributors may be used to endorse or promote products derived
       from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""
import numpy as np

from scipy import linalg
from scipy.special import xlogy
from scipy.spatial.distance import cdist, pdist, squareform

__all__ = ['Rbf']

class Rbf:
    """
    Rbf(*args)

    A class for radial basis function interpolation of functions from
    N-D scattered data to an M-D domain.

    .. note::
        `Rbf` is legacy code, for new usage please use `RBFInterpolator`
        instead.

    Parameters
    ----------
    *args : arrays
        x, y, z, ..., d, where x, y, z, ... are the coordinates of the nodes
        and d is the array of values at the nodes
    function : str or callable, optional
        The radial basis function, based on the radius, r, given by the norm
        (default is Euclidean distance); the default is 'multiquadric'::

            'multiquadric': sqrt((r/self.epsilon)**2 + 1)
            'inverse': 1.0/sqrt((r/self.epsilon)**2 + 1)
            'gaussian': exp(-(r/self.epsilon)**2)
            'linear': r
            'cubic': r**3
            'quintic': r**5
            'thin_plate': r**2 * log(r)

        If callable, then it must take 2 arguments (self, r). The epsilon
        parameter will be available as self.epsilon. Other keyword
        arguments passed in will be available as well.

    epsilon : float, optional
        Adjustable constant for gaussian or multiquadrics functions
        - defaults to approximate average distance between nodes (which is
        a good start).
    smooth : float, optional
        Values greater than zero increase the smoothness of the
        approximation. 0 is for interpolation (default), the function will
        always go through the nodal points in this case.
    norm : str, callable, optional
        A function that returns the 'distance' between two points, with
        inputs as arrays of positions (x, y, z, ...), and an output as an
        array of distance. E.g., the default: 'euclidean', such that the result
        is a matrix of the distances from each point in ``x1`` to each point in
        ``x2``. For more options, see documentation of
        `scipy.spatial.distances.cdist`.
    mode : str, optional
        Mode of the interpolation, can be '1-D' (default) or 'N-D'. When it is
        '1-D' the data `d` will be considered as 1-D and flattened
        internally. When it is 'N-D' the data `d` is assumed to be an array of
        shape (n_samples, m), where m is the dimension of the target domain.

    Attributes
    ----------
    N : int
        The number of data points (as determined by the input arrays).
    di : ndarray
        The 1-D array of data values at each of the data coordinates `xi`.
    xi : ndarray
        The 2-D array of data coordinates.
    function : str or callable
        The radial basis function. See description under Parameters.
    epsilon : float
        Parameter used by gaussian or multiquadrics functions. See Parameters.
    smooth : float
        Smoothing parameter. See description under Parameters.
    norm : str or callable
        The distance function. See description under Parameters.
    mode : str
        Mode of the interpolation. See description under Parameters.
    nodes : ndarray
        A 1-D array of node values for the interpolation.
    A : internal property, do not use

    See Also
    --------
    RBFInterpolator

    Examples
    --------
    >>> from scipy.interpolate import Rbf
    >>> rng = np.random.default_rng()
    >>> x, y, z, d = rng.random((4, 50))
    >>> rbfi = Rbf(x, y, z, d)  # radial basis function interpolator instance
    >>> xi = yi = zi = np.linspace(0, 1, 20)
    >>> di = rbfi(xi, yi, zi)   # interpolated values
    >>> di.shape
    (20,)

    """
    # Available radial basis functions that can be selected as strings;
    # they all start with _h_ (self._init_function relies on that)
    def _h_multiquadric(self, r):
        return np.sqrt((1.0/self.epsilon*r)**2 + 1)

    def _h_inverse_multiquadric(self, r):
        return 1.0/np.sqrt((1.0/self.epsilon*r)**2 + 1)

    def _h_gaussian(self, r):
        return np.exp(-(1.0/self.epsilon*r)**2)

    def _h_linear(self, r):
        return r

    def _h_cubic(self, r):
        return r**3

    def _h_quintic(self, r):
        return r**5

    def _h_thin_plate(self, r):
        return xlogy(r**2, r)

    # Setup self._function and do smoke test on initial r
    def _init_function(self, r):
        if isinstance(self.function, str):
            self.function = self.function.lower()
            _mapped = {'inverse': 'inverse_multiquadric',
                       'inverse multiquadric': 'inverse_multiquadric',
                       'thin-plate': 'thin_plate'}
            if self.function in _mapped:
                self.function = _mapped[self.function]

            func_name = "_h_" + self.function
            if hasattr(self, func_name):
                self._function = getattr(self, func_name)
            else:
                functionlist = [x[3:] for x in dir(self)
                                if x.startswith('_h_')]
                raise ValueError("function must be a callable or one of " +
                                 ", ".join(functionlist))
            self._function = getattr(self, "_h_"+self.function)
        elif callable(self.function):
            allow_one = False
            if hasattr(self.function, 'func_code') or \
               hasattr(self.function, '__code__'):
                val = self.function
                allow_one = True
            elif hasattr(self.function, "__call__"):
                val = self.function.__call__.__func__
            else:
                raise ValueError("Cannot determine number of arguments to "
                                 "function")

            argcount = val.__code__.co_argcount
            if allow_one and argcount == 1:
                self._function = self.function
            elif argcount == 2:
                self._function = self.function.__get__(self, Rbf)
            else:
                raise ValueError("Function argument must take 1 or 2 "
                                 "arguments.")

        a0 = self._function(r)
        if a0.shape != r.shape:
            raise ValueError("Callable must take array and return array of "
                             "the same shape")
        return a0

    def __init__(self, *args, **kwargs):
        # `args` can be a variable number of arrays; we flatten them and store
        # them as a single 2-D array `xi` of shape (n_args-1, array_size),
        # plus a 1-D array `di` for the values.
        # All arrays must have the same number of elements
        self.xi = np.asarray([np.asarray(a, dtype=np.float_).flatten()
                              for a in args[:-1]])
        self.N = self.xi.shape[-1]

        self.mode = kwargs.pop('mode', '1-D')

        if self.mode == '1-D':
            self.di = np.asarray(args[-1]).flatten()
            self._target_dim = 1
        elif self.mode == 'N-D':
            self.di = np.asarray(args[-1])
            self._target_dim = self.di.shape[-1]
        else:
            raise ValueError("Mode has to be 1-D or N-D.")

        if not all([x.size == self.di.shape[0] for x in self.xi]):
            raise ValueError("All arrays must be equal length.")

        self.norm = kwargs.pop('norm', 'euclidean')
        self.epsilon = kwargs.pop('epsilon', None)
        if self.epsilon is None:
            # default epsilon is the "the average distance between nodes" based
            # on a bounding hypercube
            ximax = np.amax(self.xi, axis=1)
            ximin = np.amin(self.xi, axis=1)
            edges = ximax - ximin
            edges = edges[np.nonzero(edges)]
            self.epsilon = np.power(np.prod(edges)/self.N, 1.0/edges.size)

        self.smooth = kwargs.pop('smooth', 0.0)
        self.function = kwargs.pop('function', 'multiquadric')

        # attach anything left in kwargs to self for use by any user-callable
        # function or to save on the object returned.
        for item, value in kwargs.items():
            setattr(self, item, value)

        # Compute weights
        if self._target_dim > 1:  # If we have more than one target dimension,
            # we first factorize the matrix
            self.nodes = np.zeros((self.N, self._target_dim), dtype=self.di.dtype)
            lu, piv = linalg.lu_factor(self.A)
            for i in range(self._target_dim):
                self.nodes[:, i] = linalg.lu_solve((lu, piv), self.di[:, i])
        else:
            self.nodes = linalg.solve(self.A, self.di)

    @property
    def A(self):
        # this only exists for backwards compatibility: self.A was available
        # and, at least technically, public.
        r = squareform(pdist(self.xi.T, self.norm))  # Pairwise norm
        return self._init_function(r) - np.eye(self.N)*self.smooth

    def _call_norm(self, x1, x2):
        return cdist(x1.T, x2.T, self.norm)

    def __call__(self, *args):
        args = [np.asarray(x) for x in args]
        if not all([x.shape == y.shape for x in args for y in args]):
            raise ValueError("Array lengths must be equal")
        if self._target_dim > 1:
            shp = args[0].shape + (self._target_dim,)
        else:
            shp = args[0].shape
        xa = np.asarray([a.flatten() for a in args], dtype=np.float_)
        r = self._call_norm(xa, self.xi)
        return np.dot(self._function(r), self.nodes).reshape(shp)
<commit_msg>DOC: interpolate: rbf has kwargs too.<commit_after>"""rbf - Radial basis functions for interpolation/smoothing scattered N-D data.

Written by John Travers <jtravs@gmail.com>, February 2007
Based closely on Matlab code by Alex Chirokov
Additional, large, improvements by Robert Hetland
Some additional alterations by Travis Oliphant
Interpolation with multi-dimensional target domain by Josua Sassen

Permission to use, modify, and distribute this software is given under the
terms of the SciPy (BSD style) license. See LICENSE.txt that came with
this distribution for specifics.

NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK.

Copyright (c) 2006-2007, Robert Hetland <hetland@tamu.edu>
Copyright (c) 2007, John Travers <jtravs@gmail.com>

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:

    * Redistributions of source code must retain the above copyright
       notice, this list of conditions and the following disclaimer.

    * Redistributions in binary form must reproduce the above
       copyright notice, this list of conditions and the following
       disclaimer in the documentation and/or other materials provided
       with the distribution.

    * Neither the name of Robert Hetland nor the names of any
       contributors may be used to endorse or promote products derived
       from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""
import numpy as np

from scipy import linalg
from scipy.special import xlogy
from scipy.spatial.distance import cdist, pdist, squareform

__all__ = ['Rbf']

class Rbf:
    """
    Rbf(*args, **kwargs)

    A class for radial basis function interpolation of functions from
    N-D scattered data to an M-D domain.

    .. note::
        `Rbf` is legacy code, for new usage please use `RBFInterpolator`
        instead.

    Parameters
    ----------
    *args : arrays
        x, y, z, ..., d, where x, y, z, ... are the coordinates of the nodes
        and d is the array of values at the nodes
    function : str or callable, optional
        The radial basis function, based on the radius, r, given by the norm
        (default is Euclidean distance); the default is 'multiquadric'::

            'multiquadric': sqrt((r/self.epsilon)**2 + 1)
            'inverse': 1.0/sqrt((r/self.epsilon)**2 + 1)
            'gaussian': exp(-(r/self.epsilon)**2)
            'linear': r
            'cubic': r**3
            'quintic': r**5
            'thin_plate': r**2 * log(r)

        If callable, then it must take 2 arguments (self, r). The epsilon
        parameter will be available as self.epsilon. Other keyword
        arguments passed in will be available as well.

    epsilon : float, optional
        Adjustable constant for gaussian or multiquadrics functions
        - defaults to approximate average distance between nodes (which is
        a good start).
    smooth : float, optional
        Values greater than zero increase the smoothness of the
        approximation. 0 is for interpolation (default), the function will
        always go through the nodal points in this case.
    norm : str, callable, optional
        A function that returns the 'distance' between two points, with
        inputs as arrays of positions (x, y, z, ...), and an output as an
        array of distance. E.g., the default: 'euclidean', such that the result
        is a matrix of the distances from each point in ``x1`` to each point in
        ``x2``. For more options, see documentation of
        `scipy.spatial.distances.cdist`.
    mode : str, optional
        Mode of the interpolation, can be '1-D' (default) or 'N-D'. When it is
        '1-D' the data `d` will be considered as 1-D and flattened
        internally. When it is 'N-D' the data `d` is assumed to be an array of
        shape (n_samples, m), where m is the dimension of the target domain.

    Attributes
    ----------
    N : int
        The number of data points (as determined by the input arrays).
    di : ndarray
        The 1-D array of data values at each of the data coordinates `xi`.
    xi : ndarray
        The 2-D array of data coordinates.
    function : str or callable
        The radial basis function. See description under Parameters.
    epsilon : float
        Parameter used by gaussian or multiquadrics functions. See Parameters.
    smooth : float
        Smoothing parameter. See description under Parameters.
    norm : str or callable
        The distance function. See description under Parameters.
    mode : str
        Mode of the interpolation. See description under Parameters.
    nodes : ndarray
        A 1-D array of node values for the interpolation.
    A : internal property, do not use

    See Also
    --------
    RBFInterpolator

    Examples
    --------
    >>> from scipy.interpolate import Rbf
    >>> rng = np.random.default_rng()
    >>> x, y, z, d = rng.random((4, 50))
    >>> rbfi = Rbf(x, y, z, d)  # radial basis function interpolator instance
    >>> xi = yi = zi = np.linspace(0, 1, 20)
    >>> di = rbfi(xi, yi, zi)   # interpolated values
    >>> di.shape
    (20,)

    """
    # Available radial basis functions that can be selected as strings;
    # they all start with _h_ (self._init_function relies on that)
    def _h_multiquadric(self, r):
        return np.sqrt((1.0/self.epsilon*r)**2 + 1)

    def _h_inverse_multiquadric(self, r):
        return 1.0/np.sqrt((1.0/self.epsilon*r)**2 + 1)

    def _h_gaussian(self, r):
        return np.exp(-(1.0/self.epsilon*r)**2)

    def _h_linear(self, r):
        return r

    def _h_cubic(self, r):
        return r**3

    def _h_quintic(self, r):
        return r**5

    def _h_thin_plate(self, r):
        return xlogy(r**2, r)

    # Setup self._function and do smoke test on initial r
    def _init_function(self, r):
        if isinstance(self.function, str):
            self.function = self.function.lower()
            _mapped = {'inverse': 'inverse_multiquadric',
                       'inverse multiquadric': 'inverse_multiquadric',
                       'thin-plate': 'thin_plate'}
            if self.function in _mapped:
                self.function = _mapped[self.function]

            func_name = "_h_" + self.function
            if hasattr(self, func_name):
                self._function = getattr(self, func_name)
            else:
                functionlist = [x[3:] for x in dir(self)
                                if x.startswith('_h_')]
                raise ValueError("function must be a callable or one of " +
                                 ", ".join(functionlist))
            self._function = getattr(self, "_h_"+self.function)
        elif callable(self.function):
            allow_one = False
            if hasattr(self.function, 'func_code') or \
               hasattr(self.function, '__code__'):
                val = self.function
                allow_one = True
            elif hasattr(self.function, "__call__"):
                val = self.function.__call__.__func__
            else:
                raise ValueError("Cannot determine number of arguments to "
                                 "function")

            argcount = val.__code__.co_argcount
            if allow_one and argcount == 1:
                self._function = self.function
            elif argcount == 2:
                self._function = self.function.__get__(self, Rbf)
            else:
                raise ValueError("Function argument must take 1 or 2 "
                                 "arguments.")

        a0 = self._function(r)
        if a0.shape != r.shape:
            raise ValueError("Callable must take array and return array of "
                             "the same shape")
        return a0

    def __init__(self, *args, **kwargs):
        # `args` can be a variable number of arrays; we flatten them and store
        # them as a single 2-D array `xi` of shape (n_args-1, array_size),
        # plus a 1-D array `di` for the values.
        # All arrays must have the same number of elements
        self.xi = np.asarray([np.asarray(a, dtype=np.float_).flatten()
                              for a in args[:-1]])
        self.N = self.xi.shape[-1]

        self.mode = kwargs.pop('mode', '1-D')

        if self.mode == '1-D':
            self.di = np.asarray(args[-1]).flatten()
            self._target_dim = 1
        elif self.mode == 'N-D':
            self.di = np.asarray(args[-1])
            self._target_dim = self.di.shape[-1]
        else:
            raise ValueError("Mode has to be 1-D or N-D.")

        if not all([x.size == self.di.shape[0] for x in self.xi]):
            raise ValueError("All arrays must be equal length.")

        self.norm = kwargs.pop('norm', 'euclidean')
        self.epsilon = kwargs.pop('epsilon', None)
        if self.epsilon is None:
            # default epsilon is the "the average distance between nodes" based
            # on a bounding hypercube
            ximax = np.amax(self.xi, axis=1)
            ximin = np.amin(self.xi, axis=1)
            edges = ximax - ximin
            edges = edges[np.nonzero(edges)]
            self.epsilon = np.power(np.prod(edges)/self.N, 1.0/edges.size)

        self.smooth = kwargs.pop('smooth', 0.0)
        self.function = kwargs.pop('function', 'multiquadric')

        # attach anything left in kwargs to self for use by any user-callable
        # function or to save on the object returned.
        for item, value in kwargs.items():
            setattr(self, item, value)

        # Compute weights
        if self._target_dim > 1:  # If we have more than one target dimension,
            # we first factorize the matrix
            self.nodes = np.zeros((self.N, self._target_dim), dtype=self.di.dtype)
            lu, piv = linalg.lu_factor(self.A)
            for i in range(self._target_dim):
                self.nodes[:, i] = linalg.lu_solve((lu, piv), self.di[:, i])
        else:
            self.nodes = linalg.solve(self.A, self.di)

    @property
    def A(self):
        # this only exists for backwards compatibility: self.A was available
        # and, at least technically, public.
        r = squareform(pdist(self.xi.T, self.norm))  # Pairwise norm
        return self._init_function(r) - np.eye(self.N)*self.smooth

    def _call_norm(self, x1, x2):
        return cdist(x1.T, x2.T, self.norm)

    def __call__(self, *args):
        args = [np.asarray(x) for x in args]
        if not all([x.shape == y.shape for x in args for y in args]):
            raise ValueError("Array lengths must be equal")
        if self._target_dim > 1:
            shp = args[0].shape + (self._target_dim,)
        else:
            shp = args[0].shape
        xa = np.asarray([a.flatten() for a in args], dtype=np.float_)
        r = self._call_norm(xa, self.xi)
        return np.dot(self._function(r), self.nodes).reshape(shp)
<|endoftext|>


<commit_before>import logging
import traceback
import requests
import os
import sys
from xml.etree.ElementTree import Element, tostring, SubElement
from . import __app_name__, __version__, __app_url__

_API_URL = 'https://airbrake.io/notifier_api/v2/notices'
_DEFAULT_ENV_VARIABLES = ['PATH', 'USER', 'HOME']
_DEFAULT_META_VARIABLES = ['HTTP_USER_AGENT', 'HTTP_COOKIE', 'REMOTE_ADDR',
                           'SERVER_NAME', 'SERVER_SOFTWARE']

class AirbrakeErrorHandler(logging.Handler):
    def __init__(self, api_key, env_name, request, api_url=_API_URL,
                 timeout=30, env_variables=_DEFAULT_ENV_VARIABLES,
                 meta_variables=_DEFAULT_META_VARIABLES, session=None,
                 root_path=None):
        logging.Handler.__init__(self)
        self.api_key = api_key
        self.api_url = api_url
        self.env_name = env_name
        self.env_variables = env_variables
        self.meta_variables = meta_variables
        self.timeout = timeout
        self.session = session
        self.root_path = root_path

        if self.root_path is None:
            self.root_path = os.path.dirname(os.path.abspath(__file__))

        # handle Flask request object:
        self.request = {
            'url': request.url,
            'path': request.path,
            'method': request.method,
            'values': request.values,
            'body': request.data,
            'headers': request.headers,
            'remote_addr': request.remote_addr
        }

        try:
            self.requests['json'] = request.json
        except Exception, exc:
            pass # Werkzeug couldn't parse json, ignored

    def emit(self, exception, exc_info=None):
        headers = {"Content-Type": "text/xml"}
        data = self._generate_xml(exception=exception, exc_info=exc_info)

        status = requests.post(url=self.api_url, headers=headers, data=data)

        return (status.status_code, status.text)

    def _generate_xml(self, exception, exc_info=None):
        # pass in exc_info for traceback to work with gevent:
        _, _, trace = exc_info or sys.exc_info()

        xml = Element('notice', dict(version='2.0'))
        SubElement(xml, 'api-key').text = self.api_key

        notifier = SubElement(xml, 'notifier')
        SubElement(notifier, 'name').text = __app_name__
        SubElement(notifier, 'version').text = __version__
        SubElement(notifier, 'url').text = __app_url__

        server_env = SubElement(xml, 'server-environment')
        SubElement(server_env, 'environment-name').text = self.env_name
        SubElement(server_env, 'project-root').text = self.root_path

        request_xml = SubElement(xml, 'request')
        SubElement(request_xml, 'url').text = self.request['url']

        if self.request['path']:
            SubElement(request_xml, 'component').text = self.request['path']
            SubElement(request_xml, 'action').text = self.request['method']

        # check for session:
        if self.session:
            request_session = SubElement(request_xml, 'params')
            for key in self.session:
                SubElement(request_session, 'var', dict(key=key)).text = (str(
                    self.session[key]))

        # check for form, args, body or json data
        params = SubElement(request_xml, 'params')
        if self.request['values']:
            for key in self.request['values']:
                SubElement(params, 'var', dict(key=key)).text = (str(
                    self.request['values'].get(key)))

        if self.request['body']:
            SubElement(params, 'var', dict(key='body')).text = (str(
                self.request['body']))

        if self.request.get('json'):
            for key in self.request['json']:
                SubElement(params, 'var', dict(key=key)).text = (str(
                    self.request['json'].get(key)))

        # TODO: add self.meta_variables here, learn how to extract from Flask
        cgi_data = SubElement(request_xml, 'cgi-data')
        for key, value in os.environ.items():
            if key in self.env_variables:
                SubElement(cgi_data, 'var', dict(key=key)).text = str(value)
        if self.request['remote_addr']:
            SubElement(cgi_data, 'var', dict(key='remote_addr')).text = str(
                self.request['remote_addr'])

        if self.request['headers']:
            for key, value in self.request['headers']:
                SubElement(cgi_data, 'var', dict(key=key)).text = str(value)

        # setting class name
        error = SubElement(xml, 'error')
        SubElement(error, 'class').text = exception.__class__.__name__
        SubElement(error, 'message').text = str(exception)

        # setting backtrace line
        backtrace = SubElement(error, 'backtrace')
        if trace is not None:
            for (pathname, lineno, func_name,
                 text) in traceback.extract_tb(trace)[::-1]:
                SubElement(backtrace, 'line', dict(file=pathname,
                                                   number=str(lineno),
                                                   method='%s: %s' % (func_name,
                                                                      text)))

        return tostring(xml)
<commit_msg>make python 3.4 compatible as well.<commit_after>import logging
import traceback
import requests
import os
import sys
from xml.etree.ElementTree import Element, tostring, SubElement
from . import __app_name__, __version__, __app_url__

_API_URL = 'https://airbrake.io/notifier_api/v2/notices'
_DEFAULT_ENV_VARIABLES = ['PATH', 'USER', 'HOME']
_DEFAULT_META_VARIABLES = ['HTTP_USER_AGENT', 'HTTP_COOKIE', 'REMOTE_ADDR',
                           'SERVER_NAME', 'SERVER_SOFTWARE']

class AirbrakeErrorHandler(logging.Handler):
    def __init__(self, api_key, env_name, request, api_url=_API_URL,
                 timeout=30, env_variables=_DEFAULT_ENV_VARIABLES,
                 meta_variables=_DEFAULT_META_VARIABLES, session=None,
                 root_path=None):
        logging.Handler.__init__(self)
        self.api_key = api_key
        self.api_url = api_url
        self.env_name = env_name
        self.env_variables = env_variables
        self.meta_variables = meta_variables
        self.timeout = timeout
        self.session = session
        self.root_path = root_path

        if self.root_path is None:
            self.root_path = os.path.dirname(os.path.abspath(__file__))

        # handle Flask request object:
        self.request = {
            'url': request.url,
            'path': request.path,
            'method': request.method,
            'values': request.values,
            'body': request.data,
            'headers': request.headers,
            'remote_addr': request.remote_addr
        }

        try:
            self.requests['json'] = request.json
        except Exception as exc:
            pass # Werkzeug couldn't parse json, ignored

    def emit(self, exception, exc_info=None):
        headers = {"Content-Type": "text/xml"}
        data = self._generate_xml(exception=exception, exc_info=exc_info)

        status = requests.post(url=self.api_url, headers=headers, data=data)

        return (status.status_code, status.text)

    def _generate_xml(self, exception, exc_info=None):
        # pass in exc_info for traceback to work with gevent:
        _, _, trace = exc_info or sys.exc_info()

        xml = Element('notice', dict(version='2.0'))
        SubElement(xml, 'api-key').text = self.api_key

        notifier = SubElement(xml, 'notifier')
        SubElement(notifier, 'name').text = __app_name__
        SubElement(notifier, 'version').text = __version__
        SubElement(notifier, 'url').text = __app_url__

        server_env = SubElement(xml, 'server-environment')
        SubElement(server_env, 'environment-name').text = self.env_name
        SubElement(server_env, 'project-root').text = self.root_path

        request_xml = SubElement(xml, 'request')
        SubElement(request_xml, 'url').text = self.request['url']

        if self.request['path']:
            SubElement(request_xml, 'component').text = self.request['path']
            SubElement(request_xml, 'action').text = self.request['method']

        # check for session:
        if self.session:
            request_session = SubElement(request_xml, 'params')
            for key in self.session:
                SubElement(request_session, 'var', dict(key=key)).text = (str(
                    self.session[key]))

        # check for form, args, body or json data
        params = SubElement(request_xml, 'params')
        if self.request['values']:
            for key in self.request['values']:
                SubElement(params, 'var', dict(key=key)).text = (str(
                    self.request['values'].get(key)))

        if self.request['body']:
            SubElement(params, 'var', dict(key='body')).text = (str(
                self.request['body']))

        if self.request.get('json'):
            for key in self.request['json']:
                SubElement(params, 'var', dict(key=key)).text = (str(
                    self.request['json'].get(key)))

        # TODO: add self.meta_variables here, learn how to extract from Flask
        cgi_data = SubElement(request_xml, 'cgi-data')
        for key, value in os.environ.items():
            if key in self.env_variables:
                SubElement(cgi_data, 'var', dict(key=key)).text = str(value)
        if self.request['remote_addr']:
            SubElement(cgi_data, 'var', dict(key='remote_addr')).text = str(
                self.request['remote_addr'])

        if self.request['headers']:
            for key, value in self.request['headers']:
                SubElement(cgi_data, 'var', dict(key=key)).text = str(value)

        # setting class name
        error = SubElement(xml, 'error')
        SubElement(error, 'class').text = exception.__class__.__name__
        SubElement(error, 'message').text = str(exception)

        # setting backtrace line
        backtrace = SubElement(error, 'backtrace')
        if trace is not None:
            for (pathname, lineno, func_name,
                 text) in traceback.extract_tb(trace)[::-1]:
                SubElement(backtrace, 'line', dict(file=pathname,
                                                   number=str(lineno),
                                                   method='%s: %s' % (func_name,
                                                                      text)))

        return tostring(xml)
<|endoftext|>


<commit_before>from __future__ import print_function
import time
import sys
from collections import defaultdict
from unittest import TextTestRunner, TextTestResult as _TextTestResult

from scrapy.command import ScrapyCommand
from scrapy.contracts import ContractsManager
from scrapy.utils.misc import load_object
from scrapy.utils.conf import build_component_list

class TextTestResult(_TextTestResult):
    def printSummary(self, start, stop):
        write = self.stream.write
        writeln = self.stream.writeln

        run = self.testsRun
        plural = "s" if run != 1 else ""

        writeln(self.separator2)
        writeln("Ran %d contract%s in %.3fs" % (run, plural, stop - start))
        writeln()

        infos = []
        if not self.wasSuccessful():
            write("FAILED")
            failed, errored = map(len, (self.failures, self.errors))
            if failed:
                infos.append("failures=%d" % failed)
            if errored:
                infos.append("errors=%d" % errored)
        else:
            write("OK")

        if infos:
            writeln(" (%s)" % (", ".join(infos),))
        else:
            write("\n")

class Command(ScrapyCommand):
    requires_project = True
    default_settings = {'LOG_ENABLED': False}

    def syntax(self):
        return "[options] <spider>"

    def short_desc(self):
        return "Check spider contracts"

    def add_options(self, parser):
        ScrapyCommand.add_options(self, parser)
        parser.add_option("-l", "--list", dest="list", action="store_true",
                          help="only list contracts, without checking them")
        parser.add_option("-v", "--verbose", dest="verbose", default=1, action="count",
                          help="print all contract hooks")

    def run(self, args, opts):
        # load contracts
        contracts = build_component_list(
            self.settings['SPIDER_CONTRACTS_BASE'],
            self.settings['SPIDER_CONTRACTS'],
        )
        conman = ContractsManager([load_object(c) for c in contracts])
        runner = TextTestRunner(verbosity=opts.verbose)
        result = TextTestResult(runner.stream, runner.descriptions, runner.verbosity)

        # contract requests
        contract_reqs = defaultdict(list)

        spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS'])
        spiders = spman_cls.from_settings(self.settings)

        for spider in args or spiders.list():
            spider = spiders.create(spider)
            requests = self.get_requests(spider, conman, result)

            if opts.list:
                for req in requests:
                    contract_reqs[spider.name].append(req.callback.__name__)
            elif requests:
                crawler = self.crawler_process.create_crawler(spider.name)
                crawler.crawl(spider, requests)

        # start checks
        if opts.list:
            for spider, methods in sorted(contract_reqs.iteritems()):
                print(spider)
                for method in sorted(methods):
                    print('  * %s' % method)
        else:
            start = time.time()
            self.crawler_process.start()
            stop = time.time()

            result.printErrors()
            result.printSummary(start, stop)
            self.exitcode = int(not result.wasSuccessful())

    def get_requests(self, spider, conman, result):
        requests = []

        for key, value in vars(type(spider)).items():
            if callable(value) and value.__doc__:
                bound_method = value.__get__(spider, type(spider))
                request = conman.from_method(bound_method, result)

                if request:
                    requests.append(request)

        return requests
<commit_msg>print spider name even it has no contract tests when -v is specified<commit_after>from __future__ import print_function
import time
import sys
from collections import defaultdict
from unittest import TextTestRunner, TextTestResult as _TextTestResult

from scrapy.command import ScrapyCommand
from scrapy.contracts import ContractsManager
from scrapy.utils.misc import load_object
from scrapy.utils.conf import build_component_list

class TextTestResult(_TextTestResult):
    def printSummary(self, start, stop):
        write = self.stream.write
        writeln = self.stream.writeln

        run = self.testsRun
        plural = "s" if run != 1 else ""

        writeln(self.separator2)
        writeln("Ran %d contract%s in %.3fs" % (run, plural, stop - start))
        writeln()

        infos = []
        if not self.wasSuccessful():
            write("FAILED")
            failed, errored = map(len, (self.failures, self.errors))
            if failed:
                infos.append("failures=%d" % failed)
            if errored:
                infos.append("errors=%d" % errored)
        else:
            write("OK")

        if infos:
            writeln(" (%s)" % (", ".join(infos),))
        else:
            write("\n")

class Command(ScrapyCommand):
    requires_project = True
    default_settings = {'LOG_ENABLED': False}

    def syntax(self):
        return "[options] <spider>"

    def short_desc(self):
        return "Check spider contracts"

    def add_options(self, parser):
        ScrapyCommand.add_options(self, parser)
        parser.add_option("-l", "--list", dest="list", action="store_true",
                          help="only list contracts, without checking them")
        parser.add_option("-v", "--verbose", dest="verbose", default=False, action='store_true',
                          help="print contract tests for all spiders")

    def run(self, args, opts):
        # load contracts
        contracts = build_component_list(
            self.settings['SPIDER_CONTRACTS_BASE'],
            self.settings['SPIDER_CONTRACTS'],
        )
        conman = ContractsManager([load_object(c) for c in contracts])
        runner = TextTestRunner(verbosity=2 if opts.verbose else 1)
        result = TextTestResult(runner.stream, runner.descriptions, runner.verbosity)

        # contract requests
        contract_reqs = defaultdict(list)

        spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS'])
        spiders = spman_cls.from_settings(self.settings)

        for spider in args or spiders.list():
            spider = spiders.create(spider)
            requests = self.get_requests(spider, conman, result)
            contract_reqs[spider.name] = []

            if opts.list:
                for req in requests:
                    contract_reqs[spider.name].append(req.callback.__name__)
            elif requests:
                crawler = self.crawler_process.create_crawler(spider.name)
                crawler.crawl(spider, requests)

        # start checks
        if opts.list:
            for spider, methods in sorted(contract_reqs.iteritems()):
                if not methods and not opts.verbose:
                    continue
                print(spider)
                for method in sorted(methods):
                    print('  * %s' % method)
        else:
            start = time.time()
            self.crawler_process.start()
            stop = time.time()

            result.printErrors()
            result.printSummary(start, stop)
            self.exitcode = int(not result.wasSuccessful())

    def get_requests(self, spider, conman, result):
        requests = []

        for key, value in vars(type(spider)).items():
            if callable(value) and value.__doc__:
                bound_method = value.__get__(spider, type(spider))
                request = conman.from_method(bound_method, result)

                if request:
                    requests.append(request)

        return requests
<|endoftext|>


<commit_before><commit_msg>更新回傳資料格式<commit_after><|endoftext|>


<commit_before><commit_msg>Trial and error girder 301 location header<commit_after><|endoftext|>


<commit_before>"""Worker process parts.
"""

import collections
import logging
import os
import Queue
import threading

from oslo.config import cfg

from akanda.rug import commands
from akanda.rug import event
from akanda.rug import tenant
from akanda.rug.api import nova
from akanda.rug.api import quantum

LOG = logging.getLogger(__name__)

class WorkerContext(object):
    """Holds resources owned by the worker and used by the Automaton.
    """

    def __init__(self):
        self.neutron = quantum.Quantum(cfg.CONF)
        self.nova_client = nova.Nova(cfg.CONF)

class Worker(object):
    """Manages state for the worker process.

    The Scheduler gets a callable as an argument, but we need to keep
    track of a bunch of the state machines, so the callable is a
    method of an instance of this class instead of a simple function.
    """

    def __init__(self, num_threads, notifier, ignore_directory=None):
        self._ignore_directory = ignore_directory
        self.work_queue = Queue.Queue()
        self.lock = threading.Lock()
        self._keep_going = True
        self.tenant_managers = {}
        # This process-global context should not be used in the
        # threads, since the clients are not thread-safe.
        self._context = WorkerContext()
        self.threads = [
            threading.Thread(
                name='worker-thread-%02d' % i,
                target=self._thread_target,
            )
            for i in xrange(num_threads)
        ]
        for t in self.threads:
            t.setDaemon(True)
            t.start()
        self.notifier = notifier
        # The notifier needs to be started here to ensure that it
        # happens inside the worker process and not the parent.
        self.notifier.start()
        # Track the routers and tenants we are told to ignore
        self._debug_routers = set()
        self._debug_tenants = set()
        # Thread locks for the routers so we only put one copy in the
        # work queue at a time
        self._router_locks = collections.defaultdict(threading.Lock)

    def _thread_target(self):
        """This method runs in each worker thread.
        """
        LOG.debug('starting thread')
        # Use a separate context from the one we use when receiving
        # messages and talking to the tenant router manager because we
        # are in a different thread and the clients are not
        # thread-safe.
        context = WorkerContext()
        while self._keep_going:
            try:
                # Try to get a state machine from the work queue. If
                # there's nothing to do, we will block for a while.
                sm = self.work_queue.get(timeout=10)
            except Queue.Empty:
                continue
            if not sm:
                break
            # Make sure we didn't already have some updates under way
            # for a router we've been told to ignore for debug mode.
            if sm.router_id in self._debug_routers:
                LOG.debug('skipping update of router %s (in debug mode)',
                          sm.router_id)
                continue
            # FIXME(dhellmann): Need to look at the router to see if
            # it belongs to a tenant which is in debug mode, but we
            # don't have that data in the sm, yet.
            LOG.debug('updating router %s', sm.router_id)
            try:
                sm.update(context)
            except:
                LOG.exception('could not complete update for %s',
                              sm.router_id)
            finally:
                self.work_queue.task_done()
                with self.lock:
                    # Release the lock that prevents us from adding
                    # the state machine back into the queue. If we
                    # find more work, we will re-acquire it. If we do
                    # not find more work, we hold the primary work
                    # queue lock so the main thread cannot put the
                    # state machine back into the queue until we
                    # release that lock.
                    self._release_router_lock(sm)
                    # The state machine has indicated that it is done
                    # by returning. If there is more work for it to
                    # do, reschedule it by placing it at the end of
                    # the queue.
                    if sm.has_more_work():
                        LOG.debug('%s has more work, returning to work queue',
                                  sm.router_id)
                        self._add_router_to_work_queue(sm)
                    else:
                        LOG.debug('%s has no more work', sm.router_id)
        # Return the context object so tests can look at it
        return context

    def _shutdown(self):
        """Stop the worker.
        """
        # Tell the notifier to stop
        if self.notifier:
            self.notifier.stop()
        # Stop the worker threads
        self._keep_going = False
        # Drain the task queue by discarding it
        # FIXME(dhellmann): This could prevent us from deleting
        # routers that need to be deleted.
        self.work_queue = Queue.Queue()
        for t in self.threads:
            LOG.debug('sending stop message to %s', t.getName())
            self.work_queue.put((None, None))
        # Wait for our threads to finish
        for t in self.threads:
            LOG.debug('waiting for %s to finish', t.getName())
            t.join(timeout=1)
        # Shutdown all of the tenant router managers. The lock is
        # probably not necessary, since this should be running in the
        # same thread where new messages are being received (and
        # therefore those messages aren't being processed).
        with self.lock:
            for trm in self.tenant_managers.values():
                LOG.debug('stopping tenant manager for %s', trm.tenant_id)
                trm.shutdown()

    def _get_trms(self, target):
        if target == '*':
            return list(self.tenant_managers.values())
        if target not in self.tenant_managers:
            LOG.debug('creating tenant manager for %s', target)
            self.tenant_managers[target] = tenant.TenantRouterManager(
                tenant_id=target,
                notify_callback=self.notifier.publish,
            )
        return [self.tenant_managers[target]]

    def handle_message(self, target, message):
        """Callback to be used in main
        """
        LOG.debug('got: %s %r', target, message)
        if target is None:
            # We got the shutdown instruction from our parent process.
            self._shutdown()
            return
        if message.crud == event.COMMAND:
            self._dispatch_command(target, message)
        else:
            # This is an update command for the router, so deliver it
            # to the state machine.
            with self.lock:
                self._deliver_message(target, message)

    def _dispatch_command(self, target, message):
        instructions = message.body['payload']

        if instructions['command'] == commands.WORKERS_DEBUG:
            self.report_status()

        elif instructions['command'] == commands.ROUTER_DEBUG:
            router_id = instructions['router_id']
            LOG.info('Placing router %s in debug mode', router_id)
            self._debug_routers.add(router_id)

        elif instructions['command'] == commands.ROUTER_MANAGE:
            router_id = instructions['router_id']
            try:
                self._debug_routers.remove(router_id)
                LOG.info('Resuming management of router %s', router_id)
            except KeyError:
                pass

        elif instructions['command'] == commands.TENANT_DEBUG:
            tenant_id = instructions['tenant_id']
            LOG.info('Placing tenant %s in debug mode', tenant_id)
            self._debug_tenants.add(tenant_id)

        elif instructions['command'] == commands.TENANT_MANAGE:
            tenant_id = instructions['tenant_id']
            try:
                self._debug_tenants.remove(tenant_id)
                LOG.info('Resuming management of tenant %s', tenant_id)
            except KeyError:
                pass

        else:
            LOG.warn('unrecognized command: %s', instructions)

    def _get_routers_to_ignore(self):
        ignores = set()
        try:
            if self._ignore_directory:
                ignores = set(os.listdir(self._ignore_directory))
        except OSError:
            pass
        return ignores

    def _deliver_message(self, target, message):
        if target in self._debug_tenants:
            LOG.info(
                'Ignoring message intended for tenant %s: %s',
                target, message,
            )
            return
        LOG.debug('preparing to deliver %r to %r', message, tenant)
        routers_to_ignore = self._debug_routers.union(
            self._get_routers_to_ignore()
        )
        trms = self._get_trms(target)
        for trm in trms:
            sms = trm.get_state_machines(message, self._context)
            for sm in sms:
                if sm.router_id in routers_to_ignore:
                    LOG.info(
                        'Ignoring message intended for %s: %s',
                        sm.router_id, message,
                    )
                    continue
                # Add the message to the state machine's inbox. If
                # there is already a thread working on the router,
                # that thread will pick up the new work when it is
                # done with the current job. The work queue lock is
                # acquired before asking the state machine if it has
                # more work, so this block of code won't be executed
                # at the same time as the thread trying to decide if
                # the router is done.
                sm.send_message(message)
                self._add_router_to_work_queue(sm)

    def _add_router_to_work_queue(self, sm):
        """Queue up the state machine by router id.

        The work queue lock should be held before calling this method.
        """
        l = self._router_locks[sm.router_id]
        locked = l.acquire(False)
        if locked:
            self.work_queue.put(sm)
        else:
            LOG.debug('%s is already in the work queue', sm.router_id)

    def _release_router_lock(self, sm):
        self._router_locks[sm.router_id].release()

    def report_status(self):
        LOG.debug(
            'Number of state machines in work queue: %d',
            self.work_queue.qsize()
        )
        LOG.debug(
            'Number of tenant router managers managed: %d',
            len(self.tenant_managers)
        )
        for thread in self.threads:
            LOG.debug(
                'Thread %s is %s',
                thread.name, 'alive' if thread.isAlive() else 'DEAD')
        for tid in sorted(self._debug_tenants):
            LOG.debug('Debugging tenant: %s', tid)
        if not self._debug_tenants:
            LOG.debug('No tenants in debug mode')
        for rid in sorted(self._debug_routers):
            LOG.debug('Debugging router: %s', rid)
        if not self._debug_routers:
            LOG.debug('No routers in debug mode')
        ignored_routers = sorted(self._get_routers_to_ignore())
        for rid in ignored_routers:
            LOG.debug('Ignoring router: %s', rid)
        if not ignored_routers:
            LOG.debug('No routers being ignored')
<commit_msg>Fix log message argument<commit_after>"""Worker process parts.
"""

import collections
import logging
import os
import Queue
import threading

from oslo.config import cfg

from akanda.rug import commands
from akanda.rug import event
from akanda.rug import tenant
from akanda.rug.api import nova
from akanda.rug.api import quantum

LOG = logging.getLogger(__name__)

class WorkerContext(object):
    """Holds resources owned by the worker and used by the Automaton.
    """

    def __init__(self):
        self.neutron = quantum.Quantum(cfg.CONF)
        self.nova_client = nova.Nova(cfg.CONF)

class Worker(object):
    """Manages state for the worker process.

    The Scheduler gets a callable as an argument, but we need to keep
    track of a bunch of the state machines, so the callable is a
    method of an instance of this class instead of a simple function.
    """

    def __init__(self, num_threads, notifier, ignore_directory=None):
        self._ignore_directory = ignore_directory
        self.work_queue = Queue.Queue()
        self.lock = threading.Lock()
        self._keep_going = True
        self.tenant_managers = {}
        # This process-global context should not be used in the
        # threads, since the clients are not thread-safe.
        self._context = WorkerContext()
        self.threads = [
            threading.Thread(
                name='worker-thread-%02d' % i,
                target=self._thread_target,
            )
            for i in xrange(num_threads)
        ]
        for t in self.threads:
            t.setDaemon(True)
            t.start()
        self.notifier = notifier
        # The notifier needs to be started here to ensure that it
        # happens inside the worker process and not the parent.
        self.notifier.start()
        # Track the routers and tenants we are told to ignore
        self._debug_routers = set()
        self._debug_tenants = set()
        # Thread locks for the routers so we only put one copy in the
        # work queue at a time
        self._router_locks = collections.defaultdict(threading.Lock)

    def _thread_target(self):
        """This method runs in each worker thread.
        """
        LOG.debug('starting thread')
        # Use a separate context from the one we use when receiving
        # messages and talking to the tenant router manager because we
        # are in a different thread and the clients are not
        # thread-safe.
        context = WorkerContext()
        while self._keep_going:
            try:
                # Try to get a state machine from the work queue. If
                # there's nothing to do, we will block for a while.
                sm = self.work_queue.get(timeout=10)
            except Queue.Empty:
                continue
            if not sm:
                break
            # Make sure we didn't already have some updates under way
            # for a router we've been told to ignore for debug mode.
            if sm.router_id in self._debug_routers:
                LOG.debug('skipping update of router %s (in debug mode)',
                          sm.router_id)
                continue
            # FIXME(dhellmann): Need to look at the router to see if
            # it belongs to a tenant which is in debug mode, but we
            # don't have that data in the sm, yet.
            LOG.debug('updating router %s', sm.router_id)
            try:
                sm.update(context)
            except:
                LOG.exception('could not complete update for %s',
                              sm.router_id)
            finally:
                self.work_queue.task_done()
                with self.lock:
                    # Release the lock that prevents us from adding
                    # the state machine back into the queue. If we
                    # find more work, we will re-acquire it. If we do
                    # not find more work, we hold the primary work
                    # queue lock so the main thread cannot put the
                    # state machine back into the queue until we
                    # release that lock.
                    self._release_router_lock(sm)
                    # The state machine has indicated that it is done
                    # by returning. If there is more work for it to
                    # do, reschedule it by placing it at the end of
                    # the queue.
                    if sm.has_more_work():
                        LOG.debug('%s has more work, returning to work queue',
                                  sm.router_id)
                        self._add_router_to_work_queue(sm)
                    else:
                        LOG.debug('%s has no more work', sm.router_id)
        # Return the context object so tests can look at it
        return context

    def _shutdown(self):
        """Stop the worker.
        """
        # Tell the notifier to stop
        if self.notifier:
            self.notifier.stop()
        # Stop the worker threads
        self._keep_going = False
        # Drain the task queue by discarding it
        # FIXME(dhellmann): This could prevent us from deleting
        # routers that need to be deleted.
        self.work_queue = Queue.Queue()
        for t in self.threads:
            LOG.debug('sending stop message to %s', t.getName())
            self.work_queue.put((None, None))
        # Wait for our threads to finish
        for t in self.threads:
            LOG.debug('waiting for %s to finish', t.getName())
            t.join(timeout=1)
        # Shutdown all of the tenant router managers. The lock is
        # probably not necessary, since this should be running in the
        # same thread where new messages are being received (and
        # therefore those messages aren't being processed).
        with self.lock:
            for trm in self.tenant_managers.values():
                LOG.debug('stopping tenant manager for %s', trm.tenant_id)
                trm.shutdown()

    def _get_trms(self, target):
        if target == '*':
            return list(self.tenant_managers.values())
        if target not in self.tenant_managers:
            LOG.debug('creating tenant manager for %s', target)
            self.tenant_managers[target] = tenant.TenantRouterManager(
                tenant_id=target,
                notify_callback=self.notifier.publish,
            )
        return [self.tenant_managers[target]]

    def handle_message(self, target, message):
        """Callback to be used in main
        """
        LOG.debug('got: %s %r', target, message)
        if target is None:
            # We got the shutdown instruction from our parent process.
            self._shutdown()
            return
        if message.crud == event.COMMAND:
            self._dispatch_command(target, message)
        else:
            # This is an update command for the router, so deliver it
            # to the state machine.
            with self.lock:
                self._deliver_message(target, message)

    def _dispatch_command(self, target, message):
        instructions = message.body['payload']

        if instructions['command'] == commands.WORKERS_DEBUG:
            self.report_status()

        elif instructions['command'] == commands.ROUTER_DEBUG:
            router_id = instructions['router_id']
            LOG.info('Placing router %s in debug mode', router_id)
            self._debug_routers.add(router_id)

        elif instructions['command'] == commands.ROUTER_MANAGE:
            router_id = instructions['router_id']
            try:
                self._debug_routers.remove(router_id)
                LOG.info('Resuming management of router %s', router_id)
            except KeyError:
                pass

        elif instructions['command'] == commands.TENANT_DEBUG:
            tenant_id = instructions['tenant_id']
            LOG.info('Placing tenant %s in debug mode', tenant_id)
            self._debug_tenants.add(tenant_id)

        elif instructions['command'] == commands.TENANT_MANAGE:
            tenant_id = instructions['tenant_id']
            try:
                self._debug_tenants.remove(tenant_id)
                LOG.info('Resuming management of tenant %s', tenant_id)
            except KeyError:
                pass

        else:
            LOG.warn('unrecognized command: %s', instructions)

    def _get_routers_to_ignore(self):
        ignores = set()
        try:
            if self._ignore_directory:
                ignores = set(os.listdir(self._ignore_directory))
        except OSError:
            pass
        return ignores

    def _deliver_message(self, target, message):
        if target in self._debug_tenants:
            LOG.info(
                'Ignoring message intended for tenant %s: %s',
                target, message,
            )
            return
        LOG.debug('preparing to deliver %r to %r', message, target)
        routers_to_ignore = self._debug_routers.union(
            self._get_routers_to_ignore()
        )
        trms = self._get_trms(target)
        for trm in trms:
            sms = trm.get_state_machines(message, self._context)
            for sm in sms:
                if sm.router_id in routers_to_ignore:
                    LOG.info(
                        'Ignoring message intended for %s: %s',
                        sm.router_id, message,
                    )
                    continue
                # Add the message to the state machine's inbox. If
                # there is already a thread working on the router,
                # that thread will pick up the new work when it is
                # done with the current job. The work queue lock is
                # acquired before asking the state machine if it has
                # more work, so this block of code won't be executed
                # at the same time as the thread trying to decide if
                # the router is done.
                sm.send_message(message)
                self._add_router_to_work_queue(sm)

    def _add_router_to_work_queue(self, sm):
        """Queue up the state machine by router id.

        The work queue lock should be held before calling this method.
        """
        l = self._router_locks[sm.router_id]
        locked = l.acquire(False)
        if locked:
            self.work_queue.put(sm)
        else:
            LOG.debug('%s is already in the work queue', sm.router_id)

    def _release_router_lock(self, sm):
        self._router_locks[sm.router_id].release()

    def report_status(self):
        LOG.debug(
            'Number of state machines in work queue: %d',
            self.work_queue.qsize()
        )
        LOG.debug(
            'Number of tenant router managers managed: %d',
            len(self.tenant_managers)
        )
        for thread in self.threads:
            LOG.debug(
                'Thread %s is %s',
                thread.name, 'alive' if thread.isAlive() else 'DEAD')
        for tid in sorted(self._debug_tenants):
            LOG.debug('Debugging tenant: %s', tid)
        if not self._debug_tenants:
            LOG.debug('No tenants in debug mode')
        for rid in sorted(self._debug_routers):
            LOG.debug('Debugging router: %s', rid)
        if not self._debug_routers:
            LOG.debug('No routers in debug mode')
        ignored_routers = sorted(self._get_routers_to_ignore())
        for rid in ignored_routers:
            LOG.debug('Ignoring router: %s', rid)
        if not ignored_routers:
            LOG.debug('No routers being ignored')
<|endoftext|>


<commit_before>import logging
import sys
import csv
import re
import ssl
import json
import time

if sys.version_info >= (3, 0):
    from urllib.parse import urlencode, urlunsplit
else:
	from urlparse import urlunsplit
	from urllib import urlencode

import requests
import websocket
from bs4 import BeautifulSoup

logger = logging.getLogger(__name__)
logger.setLevel(logging.WARN)

class AlarmDealerClient(object):

    DOMAIN = 'alarmdealer.com'
    WEBSOCKET_PORT = 8800

    EVENT_LOG_HEADERS = [
        'Received', 'System', 'Signal', 'SIA Code', 'Partition', 'Extra',
        'Zone/User', 'Relay Status'
    ]

    def __init__(self):
        self.session = requests.Session()
        self.ws = None

    @classmethod
    def get_url(cls, mod, action):
        """Generate a URL given the query params

        ex:
        >>> get_url('auth', 'login')
        'https://alarmdealer.com/index.php?action=login&mod=auth'

        """
        query = urlencode({'mod': mod, 'action': action})
        parts = ('https', cls.DOMAIN, 'index.php', query, '')
        url = urlunsplit(parts)
        return url

    def login(self, username, password):
        url = self.get_url('auth', 'login')
        r = self.session.get(url)
        assert "Customer Login" in r.text
        url = self.get_url('auth', 'authenticate')
        data = {
            'user_name': username,
            'user_pass': password,
        }
        r = self.session.post(url, data)
        assert "Event Log" in r.text
        self.ws = None  # now we need to reestablish the websocket

    def get_event_log(self):
        url = self.get_url('eventlog', 'index')
        r = self.session.get(url)
        soup = BeautifulSoup(r.text, "html.parser")
        table = soup.find('table', attrs={'class': 'listView'})
        headers = [th.text for th in table.find_all('th')]
        assert headers == self.EVENT_LOG_HEADERS, \
            "headers(%s) have changed" % headers
        trs = table.find_all('tr')
        events = []
        for tr in trs[1:]:
            tds = tr.find_all('td')
            values = [td.text.strip() for td in tds]
            assert len(values) == len(headers)
            events.append(values)
        return events

    def _get_websocket_login_info(self):
        """The login info for the websocket is embedded in a script tag
        on the page.  Find that stuff here.

        """
        url = self.get_url('devices', 'keypad')
        r = self.session.get(url)
        soup = BeautifulSoup(r.text, "html.parser")

        var_text = soup.find(text=re.compile("window.username"))
        var_lines = [l.strip() for l in var_text.strip().splitlines()]
        vars = [re.match('^window\.([^ ]+) = "([^"]*)";$', l).groups()
                for l in var_lines]
        info = dict(vars)
        return info['username'], info['epass'], info['user_type']

    def _get_websocket(self):
        """Return the existing AlarmDealerWebSocket or create one and return
        it.

        """
        if self.ws is None:
            logger.debug("ws is None so setting up")
            username, epass, user_type = self._get_websocket_login_info()
            url = "wss://%s:%d/ws" % (self.DOMAIN, self.WEBSOCKET_PORT)
            self.ws = AlarmDealerWebSocket(url)
            self.ws.connect()
            self.ws.login(username, epass, user_type)
        else:
            logger.debug("returning existing ws")
        return self.ws

    def get_status(self):
        ws = self._get_websocket()
        result = ws.send("send_cmd", cmd="status")
        try:
            data = json.loads(result['data'])
        except ValueError:
            status = None
        else:
            status = ' '.join([data['LCD_L1'], data['LCD_L2']])
        return status

    def wait_for_status(self, expected_status):
        while True:
            status = self.get_status()
            if status and re.search(expected_status, status, re.IGNORECASE):
                return status
            time.sleep(1)

    def arm_stay(self):
        ws = self._get_websocket()
        ws.send("send_cmd", cmd="s")

    def arm_away(self):
        ws = self._get_websocket()
        ws.send("send_cmd", cmd="a")

    def disarm(self, code):
        ws = self._get_websocket()
        for digit in code:
            logger.debug("disarm sending digit: %r", digit)
            result = ws.send("send_cmd", cmd=digit)
            logger.debug("disarm send digit result: %r", result)
            time.sleep(1)

class AlarmDealerWebSocket(object):
    """Part of http://alarmdealer.com does stuff via websocket

    This is meant to be used as a part of something like AlarmDealerClient.

    """

    def __init__(self, url):
        self.url = url
        logger.debug("url: %s", self.url)

    def connect(self):
        ssl_opt = {"cert_reqs": ssl.CERT_NONE}
        self.ws = websocket.create_connection(self.url, sslopt=ssl_opt)
        logger.debug("ws: %s", self.ws)

    def close(self):
        self.ws.close()

    def login(self, username, epass, user_type):
        result = self.send("login", username=username, epass=epass,
                           pass_hashed="true", user_type=user_type)
        assert result['msg'] == 'Logged in successfully'

    def send(self, action, **input):
        text = json.dumps({"action": action, "input": input})
        logger.debug("send: %s", text)
        self.ws.send(text)
        result = self.ws.recv()
        logger.debug("recv: %s", result)
        data = json.loads(result)
        assert data['status'] == "OK"
        return data
<commit_msg>Better way to handle python 2/3 differences<commit_after>import logging
import sys
import csv
import re
import ssl
import json
import time

try:
    from urllib.parse import urlencode, urlunsplit
except ImportError:
    from urlparse import urlunsplit
    from urllib import urlencode

import requests
import websocket
from bs4 import BeautifulSoup

logger = logging.getLogger(__name__)
logger.setLevel(logging.WARN)

class AlarmDealerClient(object):

    DOMAIN = 'alarmdealer.com'
    WEBSOCKET_PORT = 8800

    EVENT_LOG_HEADERS = [
        'Received', 'System', 'Signal', 'SIA Code', 'Partition', 'Extra',
        'Zone/User', 'Relay Status'
    ]

    def __init__(self):
        self.session = requests.Session()
        self.ws = None

    @classmethod
    def get_url(cls, mod, action):
        """Generate a URL given the query params

        ex:
        >>> get_url('auth', 'login')
        'https://alarmdealer.com/index.php?action=login&mod=auth'

        """
        query = urlencode({'mod': mod, 'action': action})
        parts = ('https', cls.DOMAIN, 'index.php', query, '')
        url = urlunsplit(parts)
        return url

    def login(self, username, password):
        url = self.get_url('auth', 'login')
        r = self.session.get(url)
        assert "Customer Login" in r.text
        url = self.get_url('auth', 'authenticate')
        data = {
            'user_name': username,
            'user_pass': password,
        }
        r = self.session.post(url, data)
        assert "Event Log" in r.text
        self.ws = None  # now we need to reestablish the websocket

    def get_event_log(self):
        url = self.get_url('eventlog', 'index')
        r = self.session.get(url)
        soup = BeautifulSoup(r.text, "html.parser")
        table = soup.find('table', attrs={'class': 'listView'})
        headers = [th.text for th in table.find_all('th')]
        assert headers == self.EVENT_LOG_HEADERS, \
            "headers(%s) have changed" % headers
        trs = table.find_all('tr')
        events = []
        for tr in trs[1:]:
            tds = tr.find_all('td')
            values = [td.text.strip() for td in tds]
            assert len(values) == len(headers)
            events.append(values)
        return events

    def _get_websocket_login_info(self):
        """The login info for the websocket is embedded in a script tag
        on the page.  Find that stuff here.

        """
        url = self.get_url('devices', 'keypad')
        r = self.session.get(url)
        soup = BeautifulSoup(r.text, "html.parser")

        var_text = soup.find(text=re.compile("window.username"))
        var_lines = [l.strip() for l in var_text.strip().splitlines()]
        vars = [re.match('^window\.([^ ]+) = "([^"]*)";$', l).groups()
                for l in var_lines]
        info = dict(vars)
        return info['username'], info['epass'], info['user_type']

    def _get_websocket(self):
        """Return the existing AlarmDealerWebSocket or create one and return
        it.

        """
        if self.ws is None:
            logger.debug("ws is None so setting up")
            username, epass, user_type = self._get_websocket_login_info()
            url = "wss://%s:%d/ws" % (self.DOMAIN, self.WEBSOCKET_PORT)
            self.ws = AlarmDealerWebSocket(url)
            self.ws.connect()
            self.ws.login(username, epass, user_type)
        else:
            logger.debug("returning existing ws")
        return self.ws

    def get_status(self):
        ws = self._get_websocket()
        result = ws.send("send_cmd", cmd="status")
        try:
            data = json.loads(result['data'])
        except ValueError:
            status = None
        else:
            status = ' '.join([data['LCD_L1'], data['LCD_L2']])
        return status

    def wait_for_status(self, expected_status):
        while True:
            status = self.get_status()
            if status and re.search(expected_status, status, re.IGNORECASE):
                return status
            time.sleep(1)

    def arm_stay(self):
        ws = self._get_websocket()
        ws.send("send_cmd", cmd="s")

    def arm_away(self):
        ws = self._get_websocket()
        ws.send("send_cmd", cmd="a")

    def disarm(self, code):
        ws = self._get_websocket()
        for digit in code:
            logger.debug("disarm sending digit: %r", digit)
            result = ws.send("send_cmd", cmd=digit)
            logger.debug("disarm send digit result: %r", result)
            time.sleep(1)

class AlarmDealerWebSocket(object):
    """Part of http://alarmdealer.com does stuff via websocket

    This is meant to be used as a part of something like AlarmDealerClient.

    """

    def __init__(self, url):
        self.url = url
        logger.debug("url: %s", self.url)

    def connect(self):
        ssl_opt = {"cert_reqs": ssl.CERT_NONE}
        self.ws = websocket.create_connection(self.url, sslopt=ssl_opt)
        logger.debug("ws: %s", self.ws)

    def close(self):
        self.ws.close()

    def login(self, username, epass, user_type):
        result = self.send("login", username=username, epass=epass,
                           pass_hashed="true", user_type=user_type)
        assert result['msg'] == 'Logged in successfully'

    def send(self, action, **input):
        text = json.dumps({"action": action, "input": input})
        logger.debug("send: %s", text)
        self.ws.send(text)
        result = self.ws.recv()
        logger.debug("recv: %s", result)
        data = json.loads(result)
        assert data['status'] == "OK"
        return data
<|endoftext|>


<commit_before>from __future__ import unicode_literals
import json
import jinja2

HTML_TEMPLATE = jinja2.Template("""
{%- if fullhtml -%}
<!DOCTYPE html>

{%- endif %}

{%- if not requirejs %}

  {%- if mode == 'vega-lite' %}

  {%- endif %}

{%- endif %}
{%- if fullhtml %}
{%- if requirejs %}

{%- endif %}

{%- endif %}

{%- if fullhtml %}

{%- endif %}
"""
)

HTML_TEMPLATE_UNIVERSAL = jinja2.Template("""

""")

TEMPLATES = {
    'standard': HTML_TEMPLATE,
    'universal': HTML_TEMPLATE_UNIVERSAL,
}

def spec_to_html(spec, mode,
                 vega_version, vegaembed_version, vegalite_version=None,
                 base_url="https://cdn.jsdelivr.net/npm/",
                 output_div='vis', embed_options=None, json_kwds=None,
                 fullhtml=True, requirejs=False, template='standard'):
    """Embed a Vega/Vega-Lite spec into an HTML page

    Parameters
    ----------
    spec : dict
        a dictionary representing a vega-lite plot spec.
    mode : string {'vega' | 'vega-lite'}
        The rendering mode. This value is overridden by embed_options['mode'],
        if it is present.
    vega_version : string
        For html output, the version of vega.js to use.
    vegalite_version : string
        For html output, the version of vegalite.js to use.
    vegaembed_version : string
        For html output, the version of vegaembed.js to use.
    base_url : string (optional)
        The base url from which to load the javascript libraries.
    output_div : string (optional)
        The id of the div element where the plot will be shown.
    embed_options : dict (optional)
        Dictionary of options to pass to the vega-embed script. Default
        entry is {'mode': mode}.
    json_kwds : dict (optional)
        Dictionary of keywords to pass to json.dumps().
    fullhtml : boolean (optional)
        If True (default) then return a full html page. If False, then return
        an HTML snippet that can be embedded into an HTML page.
    requirejs : boolean (optional)
        If False (default) then load libraries from base_url using
  {%- if mode == 'vega-lite' %}

  {%- endif %}

{%- endif %}
{%- if fullhtml %}
{%- if requirejs %}

{%- endif %}

{%- endif %}

{%- if fullhtml %}

{%- endif %}
"""
)

HTML_TEMPLATE_UNIVERSAL = jinja2.Template("""

""")

TEMPLATES = {
    'standard': HTML_TEMPLATE,
    'universal': HTML_TEMPLATE_UNIVERSAL,
}

def spec_to_html(spec, mode,
                 vega_version, vegaembed_version, vegalite_version=None,
                 base_url="https://cdn.jsdelivr.net/npm/",
                 output_div='vis', embed_options=None, json_kwds=None,
                 fullhtml=True, requirejs=False, template='standard'):
    """Embed a Vega/Vega-Lite spec into an HTML page

    Parameters
    ----------
    spec : dict
        a dictionary representing a vega-lite plot spec.
    mode : string {'vega' | 'vega-lite'}
        The rendering mode. This value is overridden by embed_options['mode'],
        if it is present.
    vega_version : string
        For html output, the version of vega.js to use.
    vegalite_version : string
        For html output, the version of vegalite.js to use.
    vegaembed_version : string
        For html output, the version of vegaembed.js to use.
    base_url : string (optional)
        The base url from which to load the javascript libraries.
    output_div : string (optional)
        The id of the div element where the plot will be shown.
    embed_options : dict (optional)
        Dictionary of options to pass to the vega-embed script. Default
        entry is {'mode': mode}.
    json_kwds : dict (optional)
        Dictionary of keywords to pass to json.dumps().
    fullhtml : boolean (optional)
        If True (default) then return a full html page. If False, then return
        an HTML snippet that can be embedded into an HTML page.
    requirejs : boolean (optional)
        If False (default) then load libraries from base_url using


<commit_before>from . import commands, log, player, run, playerinfo_handler, game
from .players_database import database_handler
<commit_msg>Fixed import error for start<commit_after>from . import commands, log, player, run, playerinfo_handler, game, start
from .players_database import database_handler
<|endoftext|>


<commit_before>import logging, datetime
from amcat.models import Project

usage_log = logging.getLogger("amcat.usage")

def log_usage(username, type, action, project=None, **extra):
    if isinstance(project, Project):
        project = "{}: {}".format(project.id, project.name)

    extra.update({
        "username": username,
        "type": type,
        "action": action,
        "project": project,
    })

    message = "{username}: {type} {action}".format(**locals())
    usage_log.info(message, extra=extra)

def log_request_usage(request, type, action, project=None, **extra):
    extra.update({
        "path": request.path,
        "meta": repr(request.META),
    })
    log_usage(request.user.username, type, action, project, **extra)

<commit_msg>remote addr and server name<commit_after>import logging, datetime
from amcat.models import Project

usage_log = logging.getLogger("amcat.usage")

def log_usage(username, type, action, project=None, **extra):
    if isinstance(project, Project):
        project = "{}: {}".format(project.id, project.name)

    extra.update({
        "username": username,
        "type": type,
        "action": action,
        "project": project,
    })

    message = "{username}: {type} {action}".format(**locals())
    usage_log.info(message, extra=extra)

def log_request_usage(request, type, action, project=None, **extra):
    extra.update({
        "path": request.path,
        "remote_addr": request.META.get('REMOTE_ADDR'),
        "server_name": request.META.get('SERVER_NAME'),
    })
    log_usage(request.user.username, type, action, project, **extra)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

#
#
#  Copyright 2013 Netflix, Inc.
#
#     Licensed under the Apache License, Version 2.0 (the "License");
#     you may not use this file except in compliance with the License.
#     You may obtain a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#     Unless required by applicable law or agreed to in writing, software
#     distributed under the License is distributed on an "AS IS" BASIS,
#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#     See the License for the specific language governing permissions and
#     limitations under the License.
#
#

"""
aminator
========
Create images from packages for deployment in various cloud formations
"""
import logging
try:
    from logging import NullHandler
except ImportError:
    # py26
    try:
        from logutils import NullHandler
    except ImportError:
        class NullHandler(logging.Handler):
            def emit(self, record):
                pass

__version__ = '1.2.167-dev'
__versioninfo__ = __version__.split('.')
__all__ = ()

logging.getLogger(__name__).addHandler(NullHandler())
<commit_msg>prepare for release 1.2.168-dev<commit_after># -*- coding: utf-8 -*-

#
#
#  Copyright 2013 Netflix, Inc.
#
#     Licensed under the Apache License, Version 2.0 (the "License");
#     you may not use this file except in compliance with the License.
#     You may obtain a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#     Unless required by applicable law or agreed to in writing, software
#     distributed under the License is distributed on an "AS IS" BASIS,
#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#     See the License for the specific language governing permissions and
#     limitations under the License.
#
#

"""
aminator
========
Create images from packages for deployment in various cloud formations
"""
import logging
try:
    from logging import NullHandler
except ImportError:
    # py26
    try:
        from logutils import NullHandler
    except ImportError:
        class NullHandler(logging.Handler):
            def emit(self, record):
                pass

__version__ = '1.2.168-dev'
__versioninfo__ = __version__.split('.')
__all__ = ()

logging.getLogger(__name__).addHandler(NullHandler())
<|endoftext|>