<commit_before># -*- coding: utf-8 -*-
"""Parses JSON output from T1 and returns a Python object"""

from __future__ import absolute_import
import json

from .errors import (T1Error, ValidationError, ParserException, STATUS_CODES)
from terminalone.vendor import six

class FindKey:
    def __init__(self, data, key):
        self.i = 0
        self.json = data
        self.key = key

    def __iter__(self):
        return self

    def __next__(self):
        try:
            self.json = self.json[self.key]
            return self.json
        except KeyError:
            raise StopIteration()

    def next(self):
        return self.__next__()

class JSONParser(object):
    """Parses JSON response"""

    def __init__(self, body):
        self.status_code = False
        try:
            parsed_data = json.loads(body)
        except ValueError as e:
            raise ParserException(e)

        self.get_status(parsed_data, body)

        try:
            self.entity_count = int(parsed_data['meta']['total_count'])
        except KeyError:
            self.entity_count = 1

        data = parsed_data.get('data')

        if type(data) == list:
            self.entities = map(self.process_entity, data)

        elif data.get('include') is not None or \
                data.get('exclude') is not None or \
                data.get('enabled') is not None:
            self._parse_target_dimensions(data)

        elif data.get('permissions') is not None:
            self.entities = self._parse_permissions(data['permissions'])

        else:
            self.entities = map(self.process_entity, FindKey(parsed_data, 'data'))

    def get_status(self, data, body):
        """Gets the status code of T1 JSON.

        If code is valid, returns None; otherwise raises the appropriate Error.
        """
        try:
            status_code = data['meta']['status']
        except KeyError:
            raise T1Error(None, body)

        try:
            exc = STATUS_CODES[status_code]
        except KeyError:
            self.status_code = False
            raise T1Error(status_code, status_code)

        if exc is None:
            self.status_code = True
            return

        if exc is True:
            message = self._parse_field_error(data)
            exc = ValidationError
            raise exc(code=status_code, content=message)

    def _parse_permissions(self, permissions):
        """Iterate over permissions and parse into dicts"""
        entity_root = permissions.get('entities')
        organization, agency, advertiser = None, None, None
        if entity_root:
            advertiser = self.process_permission(entity_root.get('advertiser'), 'advertiser')
            agency = self.process_permission(entity_root.get('agency'), 'agency')
            organization = self.process_permission(entity_root.get('organization'), 'organization')

        flags = self.process_permission(permissions.get('flags'), 'flags')
        flags.update({
            '_type': 'permission',
            'advertiser': advertiser,
            'agency': agency,
            'organization': organization,
        })

        # There will only be one instance here.
        # But the caller expects an iterator, so make a list of it
        return [flags, ]

    def _parse_target_dimensions(self, data):
        """Iterate over target dimensions and parse into dicts"""
        exclude_entities = data.get('exclude')
        include_entities = data.get('include')
        if include_entities is not None:
            include_list = map(self.process_entity, include_entities)
        else:
            include_list = []
        if exclude_entities is not None:
            exclude_list = map(self.process_entity, exclude_entities)
        else:
            exclude_list = []
        self.entity_count = 1
        self.entities = [{
            '_type': 'target_dimension',
            'exclude': exclude_list,
            'include': include_list,
        }]

    @staticmethod
    def _parse_field_error(data):
        """Iterate over field errors and parse into dicts"""
        errors = {}
        for error in data['errors']:
            errors[error['field']] = {'code': error['field-error'],
                                      'error': error['message']}
        return errors

    @classmethod
    def process_entity(cls, entity):
        """Turn json entity into a dictionary"""
        output = entity.copy()
        # Hold relation objects in specific dict. T1Service instantiates the
        # correct classes.
        relations = {}
        # for legacy/compatibility reasons with existing Entity code.
        if 'entity_type' in output:
            output['_type'] = output['entity_type']

        for key, val in six.iteritems(output):
            if type(val) is dict and 'entity_type' in val:  # Get parent entities recursively
                cls.process_related_entity(relations, val)
            elif type(val) is list and 'entity_type' in cls.get_first(val, {}):
                for child in val:
                    cls.process_related_entity(relations, child)
            # this is new as we are potentially returning multiple
            # currency types, but for now let's grab the first value
            elif type(val) is list and 'currency_code' in cls.get_first(val, {}):
                output[key] = val[0]['value']

        if relations:
            output['relations'] = relations
        return output

    @classmethod
    def process_related_entity(cls, relations, val):
        ent = cls.process_entity(val)
        if val['rel'] == ent['_type']:
            relations[val['rel']] = ent
        else:
            relations.setdefault(val['rel'], []).append(ent)

    @staticmethod
    def process_permission(permission, type):
        if not permission:
            return
        output = {}
        permission = permission[0]
        for access in permission['access']:
            if type == 'flags':
                output[access['type']] = access['value']
            else:
                output[int(access['id'])] = JSONParser.dictify_access_flag(access)
        return output

    @staticmethod
    def dictify_access_flag(flag):
        output = flag
        for key in output.keys():
            if 'id' == key or key.endswith('_id'):
                output[key] = int(output[key])
        return output

    @classmethod
    def get_first(cls, iterable, default=None):
        if iterable:
            for item in iterable:
                return item
        return default
<commit_msg>remove the get_first method<commit_after># -*- coding: utf-8 -*-
"""Parses JSON output from T1 and returns a Python object"""

from __future__ import absolute_import
import json

from .errors import (T1Error, ValidationError, ParserException, STATUS_CODES)
from terminalone.vendor import six

class FindKey:
    def __init__(self, data, key):
        self.i = 0
        self.json = data
        self.key = key

    def __iter__(self):
        return self

    def __next__(self):
        try:
            self.json = self.json[self.key]
            return self.json
        except KeyError:
            raise StopIteration()

    def next(self):
        return self.__next__()

class JSONParser(object):
    """Parses JSON response"""

    def __init__(self, body):
        self.status_code = False
        try:
            parsed_data = json.loads(body)
        except ValueError as e:
            raise ParserException(e)

        self.get_status(parsed_data, body)

        try:
            self.entity_count = int(parsed_data['meta']['total_count'])
        except KeyError:
            self.entity_count = 1

        data = parsed_data.get('data')

        if type(data) == list:
            self.entities = map(self.process_entity, data)

        elif data.get('include') is not None or \
                data.get('exclude') is not None or \
                data.get('enabled') is not None:
            self._parse_target_dimensions(data)

        elif data.get('permissions') is not None:
            self.entities = self._parse_permissions(data['permissions'])

        else:
            self.entities = map(self.process_entity, FindKey(parsed_data, 'data'))

    def get_status(self, data, body):
        """Gets the status code of T1 JSON.

        If code is valid, returns None; otherwise raises the appropriate Error.
        """
        try:
            status_code = data['meta']['status']
        except KeyError:
            raise T1Error(None, body)

        try:
            exc = STATUS_CODES[status_code]
        except KeyError:
            self.status_code = False
            raise T1Error(status_code, status_code)

        if exc is None:
            self.status_code = True
            return

        if exc is True:
            message = self._parse_field_error(data)
            exc = ValidationError
            raise exc(code=status_code, content=message)

    def _parse_permissions(self, permissions):
        """Iterate over permissions and parse into dicts"""
        entity_root = permissions.get('entities')
        organization, agency, advertiser = None, None, None
        if entity_root:
            advertiser = self.process_permission(entity_root.get('advertiser'), 'advertiser')
            agency = self.process_permission(entity_root.get('agency'), 'agency')
            organization = self.process_permission(entity_root.get('organization'), 'organization')

        flags = self.process_permission(permissions.get('flags'), 'flags')
        flags.update({
            '_type': 'permission',
            'advertiser': advertiser,
            'agency': agency,
            'organization': organization,
        })

        # There will only be one instance here.
        # But the caller expects an iterator, so make a list of it
        return [flags, ]

    def _parse_target_dimensions(self, data):
        """Iterate over target dimensions and parse into dicts"""
        exclude_entities = data.get('exclude')
        include_entities = data.get('include')
        if include_entities is not None:
            include_list = map(self.process_entity, include_entities)
        else:
            include_list = []
        if exclude_entities is not None:
            exclude_list = map(self.process_entity, exclude_entities)
        else:
            exclude_list = []
        self.entity_count = 1
        self.entities = [{
            '_type': 'target_dimension',
            'exclude': exclude_list,
            'include': include_list,
        }]

    @staticmethod
    def _parse_field_error(data):
        """Iterate over field errors and parse into dicts"""
        errors = {}
        for error in data['errors']:
            errors[error['field']] = {'code': error['field-error'],
                                      'error': error['message']}
        return errors

    @classmethod
    def process_entity(cls, entity):
        """Turn json entity into a dictionary"""
        output = entity.copy()
        # Hold relation objects in specific dict. T1Service instantiates the
        # correct classes.
        relations = {}
        # for legacy/compatibility reasons with existing Entity code.
        if 'entity_type' in output:
            output['_type'] = output['entity_type']

        for key, val in six.iteritems(output):
            if type(val) is dict and 'entity_type' in val:  # Get parent entities recursively
                cls.process_related_entity(relations, val)
            elif type(val) is list and 'entity_type' in next(iter(val), {}):
                for child in val:
                    cls.process_related_entity(relations, child)
            # this is new as we are potentially returning multiple
            # currency types, but for now let's grab the first value
            elif type(val) is list and 'currency_code' in next(iter(val), {}):
                output[key] = val[0]['value']

        if relations:
            output['relations'] = relations
        return output

    @classmethod
    def process_related_entity(cls, relations, val):
        ent = cls.process_entity(val)
        if val['rel'] == ent['_type']:
            relations[val['rel']] = ent
        else:
            relations.setdefault(val['rel'], []).append(ent)

    @staticmethod
    def process_permission(permission, type):
        if not permission:
            return
        output = {}
        permission = permission[0]
        for access in permission['access']:
            if type == 'flags':
                output[access['type']] = access['value']
            else:
                output[int(access['id'])] = JSONParser.dictify_access_flag(access)
        return output

    @staticmethod
    def dictify_access_flag(flag):
        output = flag
        for key in output.keys():
            if 'id' == key or key.endswith('_id'):
                output[key] = int(output[key])
        return output

<|endoftext|>


<commit_before>"""Kernel Principal Components Analysis"""

# Author: Gael Varoquaux <gael.varoquaux@normalesup.org>
#         Wei LI <kuantkid@gmail.com>
# License: BSD Style.

import warnings
import numpy as np

from ..base import BaseEstimator, TransformerMixin
from ..utils import check_random_state
from ..utils.graph import graph_laplacian
from ..metrics.pairwise import rbf_kernel
from ..neighbors import kneighbors_graph
from ..metrics.pairwise import pairwise_kernels

class SpectralEmbedding(BaseEstimator, TransformerMixin):
    """Spectral Embedding

    Non-linear dimensionality reduction through Spectral Embedding

    Parameters
    ----------
    n_components: int or None
        Number of components. If None, all non-zero components are kept.

    mode: {None, 'arpack' or 'amg'}
        The eigenvalue decomposition strategy to use. AMG requires pyamg
        to be installed. It can be faster on very large, sparse problems,
        but may also lead to instabilities

    random_state: int seed, RandomState instance, or None (default)
        A pseudo random number generator used for the initialization of the
        lobpcg eigen vectors decomposition when mode == 'amg'. By default
        arpack is used.

    Attributes
    ----------

    `lambdas_`, `alphas_`:
        Eigenvalues and eigenvectors of the centered kernel matrix

    `dual_coef_`:
        Inverse transform matrix

    `X_transformed_fit_`:
        Projection of the fitted data on the kernel principal components

    References
    ----------
    Kernel PCA was intoduced in:
        Bernhard Schoelkopf, Alexander J. Smola,
        and Klaus-Robert Mueller. 1999. Kernel principal
        component analysis. In Advances in kernel methods,
        MIT Press, Cambridge, MA, USA 327-352.
    """

    def __init__(self, n_components=None, kernel="linear", gamma=0, degree=3,
                 coef0=1, alpha=1.0, fit_inverse_transform=False,
                 eigen_solver='auto', tol=0, random_state=None):
        if fit_inverse_transform and kernel == 'precomputed':
            raise ValueError(
                "Cannot fit_inverse_transform with a precomputed kernel.")
        self.n_components = n_components
        self.kernel = kernel.lower()
        self.gamma = gamma
        self.degree = degree
        self.coef0 = coef0
        self.alpha = alpha
        self.fit_inverse_transform = fit_inverse_transform
        self.eigen_solver = eigen_solver
        self.tol = tol
        self.random_state = check_random_state(random_state)

    @property
    def _pairwise(self):
        return self.kernel == "precomputed"

    def _get_kernel(self, X, Y=None):
        params = {"gamma": self.gamma,
                  "degree": self.degree,
                  "coef0": self.coef0}
        try:
            return pairwise_kernels(X, Y, metric=self.kernel,
                                    filter_params=True, **params)
        except AttributeError:
            raise ValueError("%s is not a valid kernel. Valid kernels are: "
                             "rbf, poly, sigmoid, linear and precomputed."
                             % self.kernel)

    def _fit_transform(self, affinity):
        """ Fit's using kernel K"""

    def fit(self, X, y=None):
        """Fit the model from data in X.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            Training vector, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        self : object
            Returns the instance itself.
        """

        # get the adjacency matrix
        adjacency = self._get_kernel(X)
        # get the embedding
        self.embedding_ = self._spectra_embedding(adjacency)

    def fit_transform(self, X, y=None, **params):
        """Fit the model from data in X and transform X.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            Training vector, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        X_new: array-like, shape (n_samples, n_components)
        """

        self.fit(X, **params)
        return self.embedding_

    def transform(self, X):
        """Transform X.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)

        Returns
        -------
        X_new: array-like, shape (n_samples, n_components)
        """

        # tranform using
        raise NotImplementedError(
            "out of sample extension is currently unavailable")

    def _spectra_embedding(self, adjacency):
        """Project the sample on the first eigen vectors of the graph Laplacian

        The adjacency matrix is used to compute a normalized graph Laplacian
        whose spectrum (especially the eigen vectors associated to the
        smallest eigen values) has an interpretation in terms of minimal
        number of cuts necessary to split the graph into comparably sized
        components.

        This embedding can also 'work' even if the ``adjacency`` variable is
        not strictly the adjacency matrix of a graph but more generally
        an affinity or similarity matrix between samples (for instance the
        heat kernel of a euclidean distance matrix or a k-NN matrix).

        However care must taken to always make the affinity matrix symmetric
        so that the eigen vector decomposition works as expected.

        Parameters
        -----------
        adjacency: array-like or sparse matrix, shape: (n_samples, n_samples)
            The adjacency matrix of the graph to embed.

        n_components: integer, optional
            The dimension of the projection subspace.

        mode: {None, 'arpack' or 'amg'}
            The eigenvalue decomposition strategy to use. AMG requires pyamg
            to be installed. It can be faster on very large, sparse problems,
            but may also lead to instabilities

        random_state: int seed, RandomState instance, or None (default)
            A pseudo random number generator used for the initialization of the
            lobpcg eigen vectors decomposition when mode == 'amg'. By default
            arpack is used.

        Returns
        --------
        embedding: array, shape: (n_samples, n_components)
            The reduced samples

        Notes
        ------
        The graph should contain only one connected component, elsewhere the
        results make little sense.
        """

        from scipy import sparse
        from ..utils.arpack import eigsh
        from scipy.sparse.linalg import lobpcg
        try:
            from pyamg import smoothed_aggregation_solver
        except ImportError:
            if self.mode == "amg":
                raise ValueError("The mode was set to 'amg', but pyamg is "
                                 "not available.")
        n_nodes = adjacency.shape[0]
        # XXX: Should we check that the matrices given is symmetric
        if self.mode is None:
            self.mode = 'arpack'
        laplacian, dd = graph_laplacian(adjacency,
                                        normed=True, return_diag=True)
        if (self.mode == 'arpack'
            or not sparse.isspmatrix(laplacian)
            or n_nodes < 5 * self.n_components):
            # lobpcg used with mode='amg' has bugs for low number of nodes

            # We need to put the diagonal at zero
            if not sparse.isspmatrix(laplacian):
                laplacian.flat[::n_nodes + 1] = 0
            else:
                laplacian = laplacian.tocoo()
                diag_idx = (laplacian.row == laplacian.col)
                laplacian.data[diag_idx] = 0
                # If the matrix has a small number of diagonals (as in the
                # case of structured matrices comming from images), the
                # dia format might be best suited for matvec products:
                n_diags = np.unique(laplacian.row - laplacian.col).size
                if n_diags <= 7:
                    # 3 or less outer diagonals on each side
                    laplacian = laplacian.todia()
                else:
                    # csr has the fastest matvec and is thus best suited to
                    # arpack
                    laplacian = laplacian.tocsr()

            # Here we'll use shift-invert mode for fast eigenvalues (see
            # http://docs.scipy.org/doc/scipy/reference/tutorial/arpack.html
            # for a short explanation of what this means) Because the
            # normalized Laplacian has eigenvalues between 0 and 2, I - L has
            # eigenvalues between -1 and 1.  ARPACK is most efficient when
            # finding eigenvalues of largest magnitude (keyword which='LM') and
            # when these eigenvalues are very large compared to the rest. For
            # very large, very sparse graphs, I - L can have many, many
            # eigenvalues very near 1.0.  This leads to slow convergence.  So
            # instead, we'll use ARPACK's shift-invert mode, asking for the
            # eigenvalues near 1.0.  This effectively spreads-out the spectrum
            # near 1.0 and leads to much faster convergence: potentially an
            # orders-of-magnitude speedup over simply using keyword which='LA'
            # in standard mode.
            lambdas, diffusion_map = eigsh(-laplacian, k=self.n_components,
                                           sigma=1.0, which='LM')
            embedding = diffusion_map.T[::-1] * dd
        elif self.mode == 'amg':
            # Use AMG to get a preconditioner and speed up the eigenvalue
            # problem.
            laplacian = laplacian.astype(np.float)  # lobpcg needs native float
            ml = smoothed_aggregation_solver(laplacian.tocsr())
            X = self.random_state.rand(laplacian.shape[0],
                                       self.n_components)
            X[:, 0] = 1. / dd.ravel()
            M = ml.aspreconditioner()
            lambdas, diffusion_map = lobpcg(laplacian, X, M=M, tol=1.e-12,
                                            largest=False)
            embedding = diffusion_map.T * dd
            if embedding.shape[0] == 1:
                raise ValueError
        else:
            raise ValueError("Unknown value for mode: '%s'."
                             "Should be 'amg' or 'arpack'" % self.mode)
        return embedding
<commit_msg>ENH add warning for inverse transform<commit_after>"""Kernel Principal Components Analysis"""

# Author: Gael Varoquaux <gael.varoquaux@normalesup.org>
#         Wei LI <kuantkid@gmail.com>
# License: BSD Style.

import warnings
import numpy as np

from ..base import BaseEstimator, TransformerMixin
from ..utils import check_random_state
from ..utils.graph import graph_laplacian
from ..metrics.pairwise import rbf_kernel
from ..neighbors import kneighbors_graph
from ..metrics.pairwise import pairwise_kernels

from IPython.core.debugger import Tracer; debug_here = Tracer()

class SpectralEmbedding(BaseEstimator, TransformerMixin):
    """Spectral Embedding

    Non-linear dimensionality reduction through Spectral Embedding

    Project the sample on the first eigen vectors of the graph Laplacian

    The adjacency matrix is used to compute a normalized graph Laplacian
    whose spectrum (especially the eigen vectors associated to the
    smallest eigen values) has an interpretation in terms of minimal
    number of cuts necessary to split the graph into comparably sized
    components.

    This embedding can also 'work' even if the ``adjacency`` variable is
    not strictly the adjacency matrix of a graph but more generally
    an affinity or similarity matrix between samples (for instance the
    heat kernel of a euclidean distance matrix or a k-NN matrix).

    However care must taken to always make the affinity matrix symmetric
    so that the eigen vector decomposition works as expected.

    Parameters
    -----------
    adjacency: array-like or sparse matrix, shape: (n_samples, n_samples)
        The adjacency matrix of the graph to embed.

    n_components: integer, optional
        The dimension of the projection subspace.

    mode: {None, 'arpack' or 'amg'}
        The eigenvalue decomposition strategy to use. AMG requires pyamg
        to be installed. It can be faster on very large, sparse problems,
        but may also lead to instabilities

    random_state: int seed, RandomState instance, or None (default)
        A pseudo random number generator used for the initialization of the
        lobpcg eigen vectors decomposition when mode == 'amg'. By default
        arpack is used.

    kernel: "rbf" | "sigmoid" | "precomputed"
        Kernel.
        Default: "rbf"

    degree : int, optional
        Degree for poly, rbf and sigmoid kernels.
        Default: 3.

    gamma : float, optional
        Kernel coefficient for rbf and poly kernels.
        Default: 1/n_features.

    coef0 : float, optional
        Independent term in poly and sigmoid kernels.

    Attributes
    ----------

    `embedding_`
        Spectral embedding of the training matrix

    References
    ----------
    Kernel PCA was intoduced in:
        Bernhard Schoelkopf, Alexander J. Smola,
        and Klaus-Robert Mueller. 1999. Kernel principal
        component analysis. In Advances in kernel methods,
        MIT Press, Cambridge, MA, USA 327-352.
    """

    def __init__(self, n_components=None, kernel="rbf", gamma=0, degree=3,
                 coef0=1, fit_inverse_transform=False,
                 random_state=None, mode = None):
        if kernel not in {'precomputed', 'rbf', 'knn'}:
            raise ValueError(
                "Only precomputed, rbf, knn kernel supported.")
        elif fit_inverse_transform and kernel == 'precomputed':
            raise ValueError(
                "Cannot fit_inverse_transform with a precomputed kernel.")
        self.n_components = n_components
        self.kernel = kernel.lower()
        self.gamma = gamma
        self.degree = degree
        self.coef0 = coef0
        self.fit_inverse_transform = fit_inverse_transform
        self.random_state = check_random_state(random_state)
        self.mode = None;

    @property
    def _pairwise(self):
        return self.kernel == "precomputed"

    def _get_kernel(self, X, Y=None):
        params = {"gamma": self.gamma,
                  "degree": self.degree,
                  "coef0": self.coef0}
        if kernel == 'knn':
            return self.nn_fit.kneighbors_graph(self.nn_fit._fit_X,
            self.n_neighbors, mode='connectivity')

        try:
            return pairwise_kernels(X, Y, metric=self.kernel,
                                    filter_params=True, **params)
        except AttributeError:
            raise ValueError("%s is not a valid kernel. Valid kernels are: "
                             "rbf, poly, sigmoid, linear and precomputed."
                             % self.kernel)

    def _fit_transform(self, affinity):
        """ Fit's using kernel K"""
        # inverse tranform not supported
        raise NotImplementedError(
            "inverse transformation is not supported")

    def fit(self, X, y=None):
        """Fit the model from data in X.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            Training vector, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        self : object
            Returns the instance itself.
        """

        # get the adjacency matrix
        adjacency = self._get_kernel(X)
        # get the embedding
        self.embedding_ = self._spectra_embedding(adjacency)
        return self

    def fit_transform(self, X, y=None, **params):
        """Fit the model from data in X and transform X.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            Training vector, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        X_new: array-like, shape (n_samples, n_components)
        """

        self.fit(X, **params)
        return self.embedding_

    def transform(self, X):
        """Transform X.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)

        Returns
        -------
        X_new: array-like, shape (n_samples, n_components)
        """

        # out of sample not implemented
        raise NotImplementedError(
            "out of sample extension is currently unavailable")

    def _spectra_embedding(self, adjacency):
        """Project the sample on the first eigen vectors of the graph Laplacian

        The adjacency matrix is used to compute a normalized graph Laplacian
        whose spectrum (especially the eigen vectors associated to the
        smallest eigen values) has an interpretation in terms of minimal
        number of cuts necessary to split the graph into comparably sized
        components.

        This embedding can also 'work' even if the ``adjacency`` variable is
        not strictly the adjacency matrix of a graph but more generally
        an affinity or similarity matrix between samples (for instance the
        heat kernel of a euclidean distance matrix or a k-NN matrix).

        However care must taken to always make the affinity matrix symmetric
        so that the eigen vector decomposition works as expected.

        Parameters
        -----------
        adjacency: array-like or sparse matrix, shape: (n_samples, n_samples)
            The adjacency matrix of the graph to embed.

        n_components: integer, optional
            The dimension of the projection subspace.

        mode: {None, 'arpack' or 'amg'}
            The eigenvalue decomposition strategy to use. AMG requires pyamg
            to be installed. It can be faster on very large, sparse problems,
            but may also lead to instabilities

        random_state: int seed, RandomState instance, or None (default)
            A pseudo random number generator used for the initialization of the
            lobpcg eigen vectors decomposition when mode == 'amg'. By default
            arpack is used.

        Returns
        --------
        embedding: array, shape: (n_samples, n_components)
            The reduced samples

        Notes
        ------
        The graph should contain only one connected component, elsewhere the
        results make little sense.
        """

        from scipy import sparse
        from ..utils.arpack import eigsh
        from scipy.sparse.linalg import lobpcg
        try:
            from pyamg import smoothed_aggregation_solver
        except ImportError:
            if self.mode == "amg":
                raise ValueError("The mode was set to 'amg', but pyamg is "
                                 "not available.")
        n_nodes = adjacency.shape[0]
        # XXX: Should we check that the matrices given is symmetric
        if self.mode is None:
            self.mode = 'arpack'
        laplacian, dd = graph_laplacian(adjacency,
                                        normed=True, return_diag=True)
        debug_here()
        if (self.mode == 'arpack'
            or not sparse.isspmatrix(laplacian)
            or n_nodes < 5 * self.n_components):
            # lobpcg used with mode='amg' has bugs for low number of nodes

            # We need to put the diagonal at zero
            if not sparse.isspmatrix(laplacian):
                laplacian.flat[::n_nodes + 1] = 0
            else:
                laplacian = laplacian.tocoo()
                diag_idx = (laplacian.row == laplacian.col)
                laplacian.data[diag_idx] = 0
                # If the matrix has a small number of diagonals (as in the
                # case of structured matrices comming from images), the
                # dia format might be best suited for matvec products:
                n_diags = np.unique(laplacian.row - laplacian.col).size
                if n_diags <= 7:
                    # 3 or less outer diagonals on each side
                    laplacian = laplacian.todia()
                else:
                    # csr has the fastest matvec and is thus best suited to
                    # arpack
                    laplacian = laplacian.tocsr()

            # Here we'll use shift-invert mode for fast eigenvalues (see
            # http://docs.scipy.org/doc/scipy/reference/tutorial/arpack.html
            # for a short explanation of what this means) Because the
            # normalized Laplacian has eigenvalues between 0 and 2, I - L has
            # eigenvalues between -1 and 1.  ARPACK is most efficient when
            # finding eigenvalues of largest magnitude (keyword which='LM') and
            # when these eigenvalues are very large compared to the rest. For
            # very large, very sparse graphs, I - L can have many, many
            # eigenvalues very near 1.0.  This leads to slow convergence.  So
            # instead, we'll use ARPACK's shift-invert mode, asking for the
            # eigenvalues near 1.0.  This effectively spreads-out the spectrum
            # near 1.0 and leads to much faster convergence: potentially an
            # orders-of-magnitude speedup over simply using keyword which='LA'
            # in standard mode.
            lambdas, diffusion_map = eigsh(-laplacian, k=self.n_components,
                                           sigma=1.0, which='LM')
            embedding = diffusion_map.T[::-1] * dd
        elif self.mode == 'amg':
            # Use AMG to get a preconditioner and speed up the eigenvalue
            # problem.
            laplacian = laplacian.astype(np.float)  # lobpcg needs native float
            ml = smoothed_aggregation_solver(laplacian.tocsr())
            X = self.random_state.rand(laplacian.shape[0],
                                       self.n_components)
            X[:, 0] = 1. / dd.ravel()
            M = ml.aspreconditioner()
            lambdas, diffusion_map = lobpcg(laplacian, X, M=M, tol=1.e-12,
                                            largest=False)
            embedding = diffusion_map.T * dd
            if embedding.shape[0] == 1:
                raise ValueError
        else:
            raise ValueError("Unknown value for mode: '%s'."
                             "Should be 'amg' or 'arpack'" % self.mode)
        return embedding
<|endoftext|>


<commit_before>import argparse
import signal
from naoqi import ALProxy, ALBroker, ALModule
from google_client import *
from event_abstract import *
from os.path import expanduser

class SpeechRecognition(EventAbstractClass):
    WR_EVENT = "WordRecognized"
    TD_EVENT = "ALTextToSpeech/TextDone"
    ASR_ENABLE = "ASR_enable"
    FLAC_COMM = 'flac -f '
    CHANNELS = [0, 0, 1, 0]
    timeout = 0

    def __init__(self, ip, port, language, word_spotting, audio, visual, vocabulary_file, google_keys, asr_logging):
        super(self.__class__, self).__init__(self, ip, port)

        self.__shutdown_requested = False
        signal.signal(signal.SIGINT, self.signal_handler)

        vocabulary = slu_utils.lines_to_list(vocabulary_file)
        if (language == 'en') or (language == 'eng') or (language == 'english') or (language == 'English'):
            nuance_language = 'English'
            google_language = "en-US"

        self.logging = asr_logging

        self.nuance_asr = ALProxy("ALSpeechRecognition")

        self.audio_recorder = ALProxy("ALAudioRecorder")

        self.google_asr = GoogleClient(google_language, google_keys)

        self.memory_proxy = ALProxy("ALMemory")

        self.configure(
            vocabulary=vocabulary,
            nuance_language=nuance_language,
            word_spotting=word_spotting,
            audio=audio,
            visual=visual
        )

    def start(self, *args, **kwargs):
        self.subscribe(
            event=SpeechRecognition.TD_EVENT,
            callback=self.text_done_callback
        )

        self.subscribe(
            event=SpeechRecognition.ASR_ENABLE,
            callback=self.enable_callback
        )

        print "[" + self.inst.__class__.__name__ + "] Subscribers:", self.memory.getSubscribers(
            SpeechRecognition.WR_EVENT)
        print "[" + self.inst.__class__.__name__ + "] Subscribers:", self.memory.getSubscribers(
            SpeechRecognition.TD_EVENT)
        print "[" + self.inst.__class__.__name__ + "] Subscribers:", self.memory.getSubscribers(
            SpeechRecognition.ASR_ENABLE)

        self.is_enabled = False

        print "[" + self.inst.__class__.__name__ + "] ASR disabled"

        if self.logging:
            self.AUDIO_FILE_DIR = expanduser('~') + '/bags/asr_logs/'
        else:
            self.AUDIO_FILE_DIR = '/tmp/asr_logs/'
        if not os.path.exists(self.AUDIO_FILE_DIR):
            os.makedirs(self.AUDIO_FILE_DIR)
        self.AUDIO_FILE_PATH = self.AUDIO_FILE_DIR + 'SPQReL_mic_'

        self._spin()

        if self.is_enabled == True:
            self.unsubscribe(SpeechRecognition.WR_EVENT)
        self.unsubscribe(SpeechRecognition.TD_EVENT)
        self.unsubscribe(SpeechRecognition.ASR_ENABLE)
        self.broker.shutdown()

    def stop(self):
        self.audio_recorder.stopMicrophonesRecording()
        self.is_enabled = False
        self.__shutdown_requested = True
        print '[' + self.inst.__class__.__name__ + '] Good-bye'

    def configure(self, word_spotting, nuance_language, audio, visual, vocabulary):
        self.nuance_asr.pause(True)
        self.nuance_asr.setVocabulary(vocabulary, word_spotting)
        self.nuance_asr.setLanguage(nuance_language)
        self.nuance_asr.setAudioExpression(audio)
        self.nuance_asr.setVisualExpression(visual)
        self.nuance_asr.pause(False)

    def word_recognized_callback(self, *args, **kwargs):
        self.audio_recorder.stopMicrophonesRecording()
        self.nuance_asr.pause(True)
        """
        Convert Wave file into Flac file
        """
        if os.path.exists(self.FLAC_COMM + self.AUDIO_FILE + '.wav'):
            print "exist"
            if os.path.getsize(self.FLAC_COMM + self.AUDIO_FILE + '.wav') > 0:
                print "more than 0"
                os.system(self.FLAC_COMM + self.AUDIO_FILE + '.wav')
                f = open(self.AUDIO_FILE + '.flac', 'rb')
                flac_cont = f.read()
                f.close()

                results = {}
                results['GoogleASR'] = [r.encode('ascii', 'ignore').lower() for r in self.google_asr.recognize_data(flac_cont)]
                results['NuanceASR'] = [args[1][0].lower()]
                print "[" + self.inst.__class__.__name__ + "] " + str(results)
                self.memory.raiseEvent("VordRecognized", results)
            else:
                print "equals to 0"
        else:
            print "do not exist"
        self.timeout = 0
        self.nuance_asr.pause(False)
        self.audio_recorder.stopMicrophonesRecording()
        self.AUDIO_FILE = self.AUDIO_FILE_PATH + str(time.time())
        self.audio_recorder.startMicrophonesRecording(self.AUDIO_FILE + ".wav", "wav", 44100, self.CHANNELS)

    def text_done_callback(self, *args, **kwargs):
        try:
            if self.is_enabled:
                if args[1] == 0:
                    self.audio_recorder.stopMicrophonesRecording()
                    self.nuance_asr.pause(True)
                else:
                    self.audio_recorder.stopMicrophonesRecording()
                    self.AUDIO_FILE = self.AUDIO_FILE_PATH + str(time.time())
                    self.audio_recorder.startMicrophonesRecording(self.AUDIO_FILE + ".wav", "wav", 44100, self.CHANNELS)
                    self.nuance_asr.pause(False)
        except Exception as e:
            print e.message

    def enable_callback(self, *args, **kwargs):
        if args[1] == "0":
            if self.is_enabled:
                self.is_enabled = False
                self.audio_recorder.stopMicrophonesRecording()
                self.unsubscribe(SpeechRecognition.WR_EVENT)
                print "[" + self.inst.__class__.__name__ + "] ASR disabled"
            else:
                print "[" + self.inst.__class__.__name__ + "] ASR already disabled"
        else:
            if not self.is_enabled:
                #try:
                #    self.AUDIO_FILE_DIR = self.memory_proxy.getData("NAOqibag/CurrentLogFolder") + "/asr_logs/"
                #except:
                self.AUDIO_FILE_DIR = expanduser('~') + '/bags/no_data/asr_logs/'
                if not os.path.exists(self.AUDIO_FILE_DIR):
                    os.makedirs(self.AUDIO_FILE_DIR)
                self.AUDIO_FILE_PATH = self.AUDIO_FILE_DIR + 'SPQReL_mic_'
                self.is_enabled = True
                self.audio_recorder.stopMicrophonesRecording()
                self.AUDIO_FILE = self.AUDIO_FILE_PATH + str(time.time())
                self.audio_recorder.startMicrophonesRecording(self.AUDIO_FILE + ".wav", "wav", 44100, self.CHANNELS)
                self.subscribe(
                    event=SpeechRecognition.WR_EVENT,
                    callback=self.word_recognized_callback
                )
                print "[" + self.inst.__class__.__name__ + "] ASR enabled"
            else:
                print "[" + self.inst.__class__.__name__ + "] ASR already enabled"

    def reset(self):
        if self.is_enabled:
            print "[" + self.inst.__class__.__name__ + "] Reset recording.."
            self.audio_recorder.stopMicrophonesRecording()
            try:
                os.remove(self.AUDIO_FILE + ".wav")
            except:
                print "No such file: " + self.AUDIO_FILE + ".wav"
            self.AUDIO_FILE = self.AUDIO_FILE_PATH + str(time.time())
            self.audio_recorder.startMicrophonesRecording(self.AUDIO_FILE + ".wav", "wav", 44100, self.CHANNELS)

    def _spin(self, *args):
        while not self.__shutdown_requested:
            for f in args:
                f()
            time.sleep(.1)
            self.timeout = self.timeout + 1
            if self.timeout > 300:
                self.timeout = 0
                self.reset()

    def signal_handler(self, signal, frame):
        print "[" + self.inst.__class__.__name__ + "] Caught Ctrl+C, stopping."
        self.audio_recorder.stopMicrophonesRecording()
        self.__shutdown_requested = True
        print "[" + self.inst.__class__.__name__ + "] Good-bye"

def main():
    parser = argparse.ArgumentParser()

    parser.add_argument("-i", "--pip", type=str, default="127.0.0.1",
                        help="Robot ip address")
    parser.add_argument("-p", "--pport", type=int, default=9559,
                        help="Robot port number")
    parser.add_argument("-l", "--lang", type=str, default="en",
                        help="Use one of the supported languages (only English at the moment)")
    parser.add_argument("--word-spotting", action="store_true",
                        help="Run in word spotting mode")
    parser.add_argument("--no-audio", action="store_true",
                        help="Turn off bip sound when recognition starts")
    parser.add_argument("--no-visual", action="store_true",
                        help="Turn off blinking eyes when recognition starts")
    parser.add_argument("-v", "--vocabulary", type=str, default="resources/nuance_grammar.txt",
                        help="A txt file containing the list of sentences composing the vocabulary")
    parser.add_argument("-k", "--keys", type=str, default="resources/google_keys.txt",
                        help="A txt file containing the list of the keys for the Google ASR")
    parser.add_argument("-o", "--asr-logging", type=bool, default=False,
                        help="Logs the audio files")
    args = parser.parse_args()

    sr = SpeechRecognition(
        ip=args.pip,
        port=args.pport,
        language=args.lang,
        word_spotting=args.word_spotting,
        audio=not args.no_audio,
        visual=not args.no_visual,
        vocabulary_file=args.vocabulary,
        google_keys=args.keys,
        asr_logging=args.asr_logging
    )
    sr.update_globals(globals())
    sr.start()

if __name__ == "__main__":
    main()
<commit_msg>fixes on speech recognition<commit_after>import argparse
import signal
from naoqi import ALProxy, ALBroker, ALModule
from google_client import *
from event_abstract import *
from os.path import expanduser

class SpeechRecognition(EventAbstractClass):
    WR_EVENT = "WordRecognized"
    TD_EVENT = "ALTextToSpeech/TextDone"
    ASR_ENABLE = "ASR_enable"
    FLAC_COMM = 'flac -f '
    CHANNELS = [0, 0, 1, 0]
    timeout = 0

    def __init__(self, ip, port, language, word_spotting, audio, visual, vocabulary_file, google_keys, asr_logging):
        super(self.__class__, self).__init__(self, ip, port)

        self.__shutdown_requested = False
        signal.signal(signal.SIGINT, self.signal_handler)

        vocabulary = slu_utils.lines_to_list(vocabulary_file)
        if (language == 'en') or (language == 'eng') or (language == 'english') or (language == 'English'):
            nuance_language = 'English'
            google_language = "en-US"

        self.logging = asr_logging

        self.nuance_asr = ALProxy("ALSpeechRecognition")

        self.audio_recorder = ALProxy("ALAudioRecorder")

        self.google_asr = GoogleClient(google_language, google_keys)

        self.memory_proxy = ALProxy("ALMemory")

        self.configure(
            vocabulary=vocabulary,
            nuance_language=nuance_language,
            word_spotting=word_spotting,
            audio=audio,
            visual=visual
        )

    def start(self, *args, **kwargs):
        self.subscribe(
            event=SpeechRecognition.TD_EVENT,
            callback=self.text_done_callback
        )

        self.subscribe(
            event=SpeechRecognition.ASR_ENABLE,
            callback=self.enable_callback
        )

        print "[" + self.inst.__class__.__name__ + "] Subscribers:", self.memory.getSubscribers(
            SpeechRecognition.WR_EVENT)
        print "[" + self.inst.__class__.__name__ + "] Subscribers:", self.memory.getSubscribers(
            SpeechRecognition.TD_EVENT)
        print "[" + self.inst.__class__.__name__ + "] Subscribers:", self.memory.getSubscribers(
            SpeechRecognition.ASR_ENABLE)

        self.is_enabled = False

        print "[" + self.inst.__class__.__name__ + "] ASR disabled"

        if self.logging:
            self.AUDIO_FILE_DIR = expanduser('~') + '/bags/asr_logs/'
        else:
            self.AUDIO_FILE_DIR = '/tmp/asr_logs/'
        if not os.path.exists(self.AUDIO_FILE_DIR):
            os.makedirs(self.AUDIO_FILE_DIR)
        self.AUDIO_FILE_PATH = self.AUDIO_FILE_DIR + 'SPQReL_mic_'

        self._spin()

        if self.is_enabled == True:
            self.unsubscribe(SpeechRecognition.WR_EVENT)
        self.unsubscribe(SpeechRecognition.TD_EVENT)
        self.unsubscribe(SpeechRecognition.ASR_ENABLE)
        self.broker.shutdown()

    def stop(self):
        self.audio_recorder.stopMicrophonesRecording()
        self.is_enabled = False
        self.__shutdown_requested = True
        print '[' + self.inst.__class__.__name__ + '] Good-bye'

    def configure(self, word_spotting, nuance_language, audio, visual, vocabulary):
        self.nuance_asr.pause(True)
        self.nuance_asr.setVocabulary(vocabulary, word_spotting)
        self.nuance_asr.setLanguage(nuance_language)
        self.nuance_asr.setAudioExpression(audio)
        self.nuance_asr.setVisualExpression(visual)
        self.nuance_asr.pause(False)

    def word_recognized_callback(self, *args, **kwargs):
        self.audio_recorder.stopMicrophonesRecording()
        self.nuance_asr.pause(True)
        """
        Convert Wave file into Flac file
        """
        if os.path.exists(self.AUDIO_FILE + '.wav'):
            print "exist"
            if os.path.getsize(self.AUDIO_FILE + '.wav') > 0:
                print "more than 0"
                os.system(self.FLAC_COMM + self.AUDIO_FILE + '.wav')
                f = open(self.AUDIO_FILE + '.flac', 'rb')
                flac_cont = f.read()
                f.close()

                results = {}
                results['GoogleASR'] = [r.encode('ascii', 'ignore').lower() for r in self.google_asr.recognize_data(flac_cont)]
                results['NuanceASR'] = [args[1][0].lower()]
                print "[" + self.inst.__class__.__name__ + "] " + str(results)
                self.memory.raiseEvent("VordRecognized", results)
            else:
                print "equals to 0"
        else:
            print "do not exist"
        self.timeout = 0
        self.nuance_asr.pause(False)
        self.audio_recorder.stopMicrophonesRecording()
        self.AUDIO_FILE = self.AUDIO_FILE_PATH + str(time.time())
        self.audio_recorder.startMicrophonesRecording(self.AUDIO_FILE + ".wav", "wav", 44100, self.CHANNELS)

    def text_done_callback(self, *args, **kwargs):
        try:
            if self.is_enabled:
                if args[1] == 0:
                    self.audio_recorder.stopMicrophonesRecording()
                    self.nuance_asr.pause(True)
                else:
                    self.audio_recorder.stopMicrophonesRecording()
                    self.AUDIO_FILE = self.AUDIO_FILE_PATH + str(time.time())
                    self.audio_recorder.startMicrophonesRecording(self.AUDIO_FILE + ".wav", "wav", 44100, self.CHANNELS)
                    self.nuance_asr.pause(False)
        except Exception as e:
            print e.message

    def enable_callback(self, *args, **kwargs):
        if args[1] == "0":
            if self.is_enabled:
                self.is_enabled = False
                self.audio_recorder.stopMicrophonesRecording()
                self.unsubscribe(SpeechRecognition.WR_EVENT)
                print "[" + self.inst.__class__.__name__ + "] ASR disabled"
            else:
                print "[" + self.inst.__class__.__name__ + "] ASR already disabled"
        else:
            if not self.is_enabled:
                #try:
                #    self.AUDIO_FILE_DIR = self.memory_proxy.getData("NAOqibag/CurrentLogFolder") + "/asr_logs/"
                #except:
                self.AUDIO_FILE_DIR = expanduser('~') + '/bags/no_data/asr_logs/'
                if not os.path.exists(self.AUDIO_FILE_DIR):
                    os.makedirs(self.AUDIO_FILE_DIR)
                self.AUDIO_FILE_PATH = self.AUDIO_FILE_DIR + 'SPQReL_mic_'
                self.is_enabled = True
                self.audio_recorder.stopMicrophonesRecording()
                self.AUDIO_FILE = self.AUDIO_FILE_PATH + str(time.time())
                self.audio_recorder.startMicrophonesRecording(self.AUDIO_FILE + ".wav", "wav", 44100, self.CHANNELS)
                self.subscribe(
                    event=SpeechRecognition.WR_EVENT,
                    callback=self.word_recognized_callback
                )
                print "[" + self.inst.__class__.__name__ + "] ASR enabled"
            else:
                print "[" + self.inst.__class__.__name__ + "] ASR already enabled"

    def reset(self):
        if self.is_enabled:
            print "[" + self.inst.__class__.__name__ + "] Reset recording.."
            self.audio_recorder.stopMicrophonesRecording()
            try:
                os.remove(self.AUDIO_FILE + ".wav")
            except:
                print "No such file: " + self.AUDIO_FILE + ".wav"
            self.AUDIO_FILE = self.AUDIO_FILE_PATH + str(time.time())
            self.audio_recorder.startMicrophonesRecording(self.AUDIO_FILE + ".wav", "wav", 44100, self.CHANNELS)

    def _spin(self, *args):
        while not self.__shutdown_requested:
            for f in args:
                f()
            time.sleep(.1)
            self.timeout = self.timeout + 1
            if self.timeout > 300:
                self.timeout = 0
                self.reset()

    def signal_handler(self, signal, frame):
        print "[" + self.inst.__class__.__name__ + "] Caught Ctrl+C, stopping."
        self.audio_recorder.stopMicrophonesRecording()
        self.__shutdown_requested = True
        print "[" + self.inst.__class__.__name__ + "] Good-bye"

def main():
    parser = argparse.ArgumentParser()

    parser.add_argument("-i", "--pip", type=str, default="127.0.0.1",
                        help="Robot ip address")
    parser.add_argument("-p", "--pport", type=int, default=9559,
                        help="Robot port number")
    parser.add_argument("-l", "--lang", type=str, default="en",
                        help="Use one of the supported languages (only English at the moment)")
    parser.add_argument("--word-spotting", action="store_true",
                        help="Run in word spotting mode")
    parser.add_argument("--no-audio", action="store_true",
                        help="Turn off bip sound when recognition starts")
    parser.add_argument("--no-visual", action="store_true",
                        help="Turn off blinking eyes when recognition starts")
    parser.add_argument("-v", "--vocabulary", type=str, default="resources/nuance_grammar.txt",
                        help="A txt file containing the list of sentences composing the vocabulary")
    parser.add_argument("-k", "--keys", type=str, default="resources/google_keys.txt",
                        help="A txt file containing the list of the keys for the Google ASR")
    parser.add_argument("-o", "--asr-logging", type=bool, default=False,
                        help="Logs the audio files")
    args = parser.parse_args()

    sr = SpeechRecognition(
        ip=args.pip,
        port=args.pport,
        language=args.lang,
        word_spotting=args.word_spotting,
        audio=not args.no_audio,
        visual=not args.no_visual,
        vocabulary_file=args.vocabulary,
        google_keys=args.keys,
        asr_logging=args.asr_logging
    )
    sr.update_globals(globals())
    sr.start()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># #######
# Copyright (c) 2019 Cloudify Platform Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
from random import random

from integration_tests.tests.test_cases import PluginsTest

PLUGIN_NAME = 'cloudify-azure-plugin'
DEVELOPMENT_ROOT = os.path.abspath(
    os.path.join(os.path.dirname(
        os.path.realpath(__file__)), '../..'))

class AzurePluginTestCase(PluginsTest):

    base_path = os.path.dirname(os.path.realpath(__file__))

    @property
    def plugin_root_directory(self):
        return os.path.abspath(os.path.join(self.base_path, '..'))

    @property
    def inputs(self):
        return {
            'location': os.getenv('azure_location'),
            'resource_prefix': os.getenv('CIRCLE_JOB', 'cfy'),
            'resource_suffix': os.getenv(
                'CIRCLE_BUILD_NUM', str(random())[-4:-1])
        }

    def create_secrets(self):
        secrets = {
            'azure_subscription_id': os.getenv('azure_subscription_id'),
            'azure_tenant_id': os.getenv('azure_tenant_id'),
            'azure_client_id': os.getenv('azure_client_id'),
            'azure_client_secret': os.getenv('azure_client_secret'),
            'azure_location': os.getenv('azure_location'),
            'agent_key_private': os.getenv('agent_key_private'),
            'agent_key_public': os.getenv('agent_key_public'),
        }
        self._create_secrets(secrets)

    @property
    def wagon_build_time_limit(self):
        return 1800

    def upload_plugins(self):
        self.upload_mock_plugin(
            PLUGIN_NAME, self.plugin_root_directory)
        self.upload_mock_plugin(
            'cloudify-utilities-plugin',
            os.path.join(DEVELOPMENT_ROOT, 'cloudify-utilities-plugin'))
        self.upload_mock_plugin(
            'cloudify-ansible-plugin',
            os.path.join(DEVELOPMENT_ROOT, 'cloudify-ansible-plugin'))

    def test_blueprints(self):
        self.upload_plugins()
        self.create_secrets()
        self.check_hello_world_blueprint('azure', self.inputs, 800)
        self.check_db_lb_app_blueprint('azure', 800)
<commit_msg>update test<commit_after># #######
# Copyright (c) 2019 Cloudify Platform Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
from random import random

from integration_tests.tests.test_cases import PluginsTest

PLUGIN_NAME = 'cloudify-azure-plugin'
DEVELOPMENT_ROOT = os.path.abspath(
    os.path.join(os.path.dirname(
        os.path.realpath(__file__)), '../..'))
TEST_KEY_PATH = '/tmp/foo.rsa'
TEST_PUB_PATH = '/tmp/foo.rsa.pub'

class AzurePluginTestCase(PluginsTest):

    base_path = os.path.dirname(os.path.realpath(__file__))

    @property
    def plugin_root_directory(self):
        return os.path.abspath(os.path.join(self.base_path, '..'))

    @property
    def inputs(self):
        return {
            'location': os.getenv('azure_location'),
            'resource_prefix': os.getenv('CIRCLE_JOB', 'cfy'),
            'resource_suffix': os.getenv(
                'CIRCLE_BUILD_NUM', str(random())[-4:-1])
        }

    def create_secrets(self):
        secrets = {
            'agent_key_private': os.getenv('agent_key_private',
                                           open(TEST_KEY_PATH).read()),
            'agent_key_public': os.getenv('agent_key_public',
                                          open(TEST_PUB_PATH).read()),
            'azure_subscription_id': os.getenv('azure_subscription_id'),
            'azure_tenant_id': os.getenv('azure_tenant_id'),
            'azure_client_id': os.getenv('azure_client_id'),
            'azure_client_secret': os.getenv('azure_client_secret'),
            'azure_location': os.getenv('azure_location', 'westeurope'),
        }
        self._create_secrets(secrets)

    @property
    def wagon_build_time_limit(self):
        return 1800

    def upload_plugins(self):
        self.upload_mock_plugin(
            PLUGIN_NAME, self.plugin_root_directory)
        self.upload_mock_plugin(
            'cloudify-utilities-plugin',
            os.path.join(DEVELOPMENT_ROOT, 'cloudify-utilities-plugin'))
        self.upload_mock_plugin(
            'cloudify-ansible-plugin',
            os.path.join(DEVELOPMENT_ROOT, 'cloudify-ansible-plugin'))

    def test_blueprints(self):
        self.upload_plugins()
        self.create_secrets()
        self.check_hello_world_blueprint('azure', self.inputs, 800)
        self.check_db_lb_app_blueprint('azure', 800)
<|endoftext|>


<commit_before>from __future__ import unicode_literals
from ...en import English
import pytest

@pytest.fixture
def en_tokenizer():
    return English.Defaults.create_tokenizer()

@pytest.mark.parametrize('inputs', [("Robin's", "Robin"), ("Alexis's", "Alexis")])
def test_tokenizer_handles_poss_contraction(en_tokenizer, inputs):
    text_poss, text = inputs
    tokens = en_tokenizer(text_poss)
    assert len(tokens) == 2
    assert tokens[0].text == text
    assert tokens[1].text == "'s"

@pytest.mark.parametrize('text', ["schools'", "Alexis'"])
def test_tokenizer_splits_trailing_apos(en_tokenizer, text):
    tokens = en_tokenizer(text)
    assert len(tokens) == 2
    assert tokens[0].text == text.split("'")[0]
    assert tokens[1].text == "'"

@pytest.mark.parametrize('text', ["'em", "nothin'", "ol'"])
def text_tokenizer_doesnt_split_apos_exc(en_tokenizer, text):
    tokens = en_tokenizer(text)
    assert len(tokens) == 1
    assert tokens[0].text == text

@pytest.mark.parametrize('text', ["we'll", "You'll", "there'll"])
def test_tokenizer_handles_ll_contraction(en_tokenizer, text):
    tokens = en_tokenizer(text)
    assert len(tokens) == 2
    assert tokens[0].text == text.split("'")[0]
    assert tokens[1].text == "'ll"
    assert tokens[1].lemma_ == "will"

@pytest.mark.parametrize('inputs', [("can't", "Can't"), ("ain't", "Ain't")])
def test_tokenizer_handles_capitalization(en_tokenizer, inputs):
    text_lower, text_title = inputs
    tokens_lower = en_tokenizer(text_lower)
    tokens_title = en_tokenizer(text_title)
    assert tokens_title[0].text == tokens_lower[0].text.title()
    assert tokens_lower[0].text == tokens_title[0].text.lower()
    assert tokens_lower[1].text == tokens_title[1].text

@pytest.mark.parametrize('pron', ["I", "You", "He", "She", "It", "We", "They"])
def test_tokenizer_keeps_title_case(en_tokenizer, pron):
    for contraction in ["'ll", "'d"]:
        tokens = en_tokenizer(pron + contraction)
        assert tokens[0].text == pron
        assert tokens[1].text == contraction

@pytest.mark.parametrize('exc', ["Ill", "ill", "Hell", "hell", "Well", "well"])
def test_tokenizer_excludes_ambiguous(en_tokenizer, exc):
    tokens = en_tokenizer(exc)
    assert len(tokens) == 1

@pytest.mark.parametrize('inputs', [("We've", "``We've"), ("couldn't", "couldn't)")])
def test_tokenizer_splits_defined_punct(en_tokenizer, inputs):
    wo_punct, w_punct = inputs
    tokens = en_tokenizer(wo_punct)
    assert len(tokens) == 2
    tokens = en_tokenizer(w_punct)
    assert len(tokens) == 3
<commit_msg>Update tokenizer tests for contractions<commit_after>from __future__ import unicode_literals

import pytest

@pytest.mark.parametrize('text_poss,text', [("Robin's", "Robin"), ("Alexis's", "Alexis")])
def test_tokenizer_handles_poss_contraction(en_tokenizer, text_poss, text):
    tokens = en_tokenizer(text_poss)
    assert len(tokens) == 2
    assert tokens[0].text == text
    assert tokens[1].text == "'s"

@pytest.mark.parametrize('text', ["schools'", "Alexis'"])
def test_tokenizer_splits_trailing_apos(en_tokenizer, text):
    tokens = en_tokenizer(text)
    assert len(tokens) == 2
    assert tokens[0].text == text.split("'")[0]
    assert tokens[1].text == "'"

@pytest.mark.parametrize('text', ["'em", "nothin'", "ol'"])
def text_tokenizer_doesnt_split_apos_exc(en_tokenizer, text):
    tokens = en_tokenizer(text)
    assert len(tokens) == 1
    assert tokens[0].text == text

@pytest.mark.parametrize('text', ["we'll", "You'll", "there'll"])
def test_tokenizer_handles_ll_contraction(en_tokenizer, text):
    tokens = en_tokenizer(text)
    assert len(tokens) == 2
    assert tokens[0].text == text.split("'")[0]
    assert tokens[1].text == "'ll"
    assert tokens[1].lemma_ == "will"

@pytest.mark.parametrize('text_lower,text_title', [("can't", "Can't"), ("ain't", "Ain't")])
def test_tokenizer_handles_capitalization(en_tokenizer, text_lower, text_title):
    tokens_lower = en_tokenizer(text_lower)
    tokens_title = en_tokenizer(text_title)
    assert tokens_title[0].text == tokens_lower[0].text.title()
    assert tokens_lower[0].text == tokens_title[0].text.lower()
    assert tokens_lower[1].text == tokens_title[1].text

@pytest.mark.parametrize('pron', ["I", "You", "He", "She", "It", "We", "They"])
@pytest.mark.parametrize('contraction', ["'ll", "'d"])
def test_tokenizer_keeps_title_case(en_tokenizer, pron, contraction):
    tokens = en_tokenizer(pron + contraction)
    assert tokens[0].text == pron
    assert tokens[1].text == contraction

@pytest.mark.parametrize('exc', ["Ill", "ill", "Hell", "hell", "Well", "well"])
def test_tokenizer_excludes_ambiguous(en_tokenizer, exc):
    tokens = en_tokenizer(exc)
    assert len(tokens) == 1

@pytest.mark.parametrize('wo_punct,w_punct', [("We've", "``We've"), ("couldn't", "couldn't)")])
def test_tokenizer_splits_defined_punct(en_tokenizer, wo_punct, w_punct):
    tokens = en_tokenizer(wo_punct)
    assert len(tokens) == 2
    tokens = en_tokenizer(w_punct)
    assert len(tokens) == 3
<|endoftext|>


<commit_before># -*- mode: python; coding: utf-8 -*-
# Copyright 2014 Peter Williams <peter@newton.cx> and collaborators.
# Licensed under the MIT License.

"""pwkit.pdm - period-finding with phase dispersion minimization

As defined in Stellingwerf (1978ApJ...224..953S). See the update in
Schwarzenberg-Czerny (1997ApJ...489..941S), however, which corrects the
significance test formally; Linnell Nemec & Nemec (1985AJ.....90.2317L)
provide a Monte Carlo approach. Also, Stellingwerf has developed "PDM2" which
attempts to improve a few aspects; see

  http://www.stellingwerf.com/rfs-bin/index.cgi?action=PageView&id=29
  http://en.wikipedia.org/wiki/Phase_dispersion_minimization

"""
from __future__ import absolute_import, division, print_function, unicode_literals

# TODO: automatic rule for nbin?
# TODO: ditto for nr periods to try?
# TODO: confidence in peak value or something

__all__ = (b'PDMResult pdm').split ()

import numpy as np
from collections import namedtuple

from .numutil import weighted_variance

PDMResult = namedtuple ('PDMResult', 'thetas imin pmin mc_tmins '
                        'mc_pvalue mc_pmins mc_puncert'.split ())

def one_theta (t, x, wt, period, nbin, nshift, v_all):
    phase0 = t / period
    numer = denom = 0.

    for i in xrange (nshift):
        phase = (phase0 + float (i) / (nshift * nbin)) % 1.
        binloc = np.floor (phase * nbin).astype (np.int)

        for j in xrange (nbin):
            wh = np.where (binloc == j)[0]
            if wh.size < 3:
                continue

            numer += weighted_variance (x[wh], wt[wh]) * (wh.size - 1)
            denom += wh.size - 1

    return numer / (denom * v_all)

def pdm (t, x, u, periods, nbin, nshift=8, nsmc=256, numc=256, weights=False):
    """Perform phase dispersion minimization.

    `t` - 1D array - time coordinate
    `x` - 1D array, same size as `t` - observed value
    `u` - 1D array, same size as `t` - uncertainty on observed value; same units as `x`
    `periods` - 1D array - set of candidate periods to sample; same units as `t`
    `nbin` - int - number of phase bins to construct
    `nshift` - int=8 - number of shifted binnings to sample to combact statistical flukes
    `nsmc` - int=256 - number of Monte Carlo shufflings to compute, to evaluate the
       significance of the minimal theta value.
    `numc` - int=256 - number of Monte Carlo added-noise datasets to compute, to evaluate
       the uncertainty in the location of the minimal theta value.
    `weights` - bool=False - if True, 'u' is actually weights, not uncertainties.
       Usually weights = u**-2.

    Returns named tuple of:

    `thetas` - 1D array of values of theta statistic, same size as `periods`
    `imin` - index of smallest (best) value in `thetas`
    `pmin` - the `period` value with the smallest (best) `theta`
    `mc_tmins` - 1D array of size `nsmc` with Monte Carlo samplings of minimal
       theta values for shufflings of the data; assesses significance of the peak
    `mc_pvalue` - probability (between 0 and 1) of obtaining the best theta value
       in a randomly-shuffled dataset
    `mc_pmins` - 1D array of size `numc` with Monte Carlo samplings of best
       period values for noise-added data; assesses uncertainty of `pmin`
    `mc_puncert` - standard deviation of `mc_pmins`; approximate uncertainty
       on `pmin`.

    We don't do anything clever, so runtime scales at least as
      t.size * periods.size * nbin * nshift * (nsmc + numc + 1)."""

    t = np.asfarray (t)
    x = np.asfarray (x)
    u = np.asfarray (u)
    periods = np.asfarray (periods)
    t, x, u, periods = np.atleast_1d (t, x, u, periods)
    nbin = int (nbin)
    nshift = int (nshift)
    nsmc = int (nsmc)
    numc = int (numc)

    if t.ndim != 1:
        raise ValueError ('`t` must be <= 1D')

    if x.shape != t.shape:
        raise ValueError ('`t` and `x` arguments must be the same size')

    if u.shape != t.shape:
        raise ValueError ('`t` and `u` arguments must be the same size')

    if periods.ndim != 1:
        raise ValueError ('`periods` must be <= 1D')

    if nbin < 2:
        raise ValueError ('`nbin` must be at least 2')

    if nshift < 1:
        raise ValueError ('`nshift` must be at least 1')

    if nsmc < 0:
        raise ValueError ('`nsmc` must be nonnegative')

    if numc < 0:
        raise ValueError ('`numc` must be nonnegative')

    # We can finally get started!

    if weights:
        wt = u
        u = wt ** -0.5
    else:
        wt = u ** -2

    v_all = weighted_variance (x, wt)

    thetas = np.empty (periods.shape)

    for i in xrange (periods.size):
        thetas[i] = one_theta (t, x, wt, periods[i], nbin, nshift, v_all)

    imin = thetas.argmin ()
    pmin = periods[imin]

    # Now do the Monte Carlo jacknifing so that the caller can have some idea
    # as to the significance of the minimal value of `thetas`. XXX: ripe for
    # parallelization.

    mc_thetas = np.empty (periods.shape)
    mc_tmins = np.empty (nsmc)

    for i in xrange (nsmc):
        shuf = np.random.permutation (x.size)

        for j in xrange (periods.size):
            mc_thetas[j] = one_theta (t, x[shuf], wt[shuf], periods[j],
                                      nbin, nshift, v_all)

        mc_tmins[i] = mc_thetas.min ()

    mc_tmins.sort ()
    mc_pvalue = mc_tmins.searchsorted (thetas[imin]) / nsmc

    # Now add noise to assess the uncertainty of the period.

    mc_pmins = np.empty (numc)

    for i in xrange (numc):
        noised = np.random.normal (x, u)

        for j in xrange (periods.size):
            mc_thetas[j] = one_theta (t, noised, wt, periods[j],
                                      nbin, nshift, v_all)

        mc_pmins[i] = periods[mc_thetas.argmin ()]

    mc_pmins.sort ()
    mc_puncert = mc_pmins.std ()

    # All done.

    return PDMResult (thetas=thetas, imin=imin, pmin=pmin, mc_tmins=mc_tmins,
                      mc_pvalue=mc_pvalue, mc_pmins=mc_pmins,
                      mc_puncert=mc_puncert)
<commit_msg>pwkit/pdm.py: parallelize the PDM computation<commit_after># -*- mode: python; coding: utf-8 -*-
# Copyright 2014 Peter Williams <peter@newton.cx> and collaborators.
# Licensed under the MIT License.

"""pwkit.pdm - period-finding with phase dispersion minimization

As defined in Stellingwerf (1978ApJ...224..953S). See the update in
Schwarzenberg-Czerny (1997ApJ...489..941S), however, which corrects the
significance test formally; Linnell Nemec & Nemec (1985AJ.....90.2317L)
provide a Monte Carlo approach. Also, Stellingwerf has developed "PDM2" which
attempts to improve a few aspects; see

  http://www.stellingwerf.com/rfs-bin/index.cgi?action=PageView&id=29
  http://en.wikipedia.org/wiki/Phase_dispersion_minimization

"""
from __future__ import absolute_import, division, print_function, unicode_literals

# TODO: automatic rule for nbin?
# TODO: ditto for nr periods to try?
# TODO: confidence in peak value or something

__all__ = (b'PDMResult pdm').split ()

import numpy as np
from collections import namedtuple

from .numutil import weighted_variance
from .parallel import make_parallel_helper

PDMResult = namedtuple ('PDMResult', 'thetas imin pmin mc_tmins '
                        'mc_pvalue mc_pmins mc_puncert'.split ())

def one_theta (t, x, wt, period, nbin, nshift, v_all):
    phase0 = t / period
    numer = denom = 0.

    for i in xrange (nshift):
        phase = (phase0 + float (i) / (nshift * nbin)) % 1.
        binloc = np.floor (phase * nbin).astype (np.int)

        for j in xrange (nbin):
            wh = np.where (binloc == j)[0]
            if wh.size < 3:
                continue

            numer += weighted_variance (x[wh], wt[wh]) * (wh.size - 1)
            denom += wh.size - 1

    return numer / (denom * v_all)

def _map_one_theta (args):
    """Needed for the parallel map() call in pdm() due to the gross way in which
    Python multiprocessing works.

    """
    return one_theta (*args)

def pdm (t, x, u, periods, nbin, nshift=8, nsmc=256, numc=256, weights=False, parallel=True):
    """Perform phase dispersion minimization.

    `t` - 1D array - time coordinate
    `x` - 1D array, same size as `t` - observed value
    `u` - 1D array, same size as `t` - uncertainty on observed value; same units as `x`
    `periods` - 1D array - set of candidate periods to sample; same units as `t`
    `nbin` - int - number of phase bins to construct
    `nshift` - int=8 - number of shifted binnings to sample to combact statistical flukes
    `nsmc` - int=256 - number of Monte Carlo shufflings to compute, to evaluate the
       significance of the minimal theta value.
    `numc` - int=256 - number of Monte Carlo added-noise datasets to compute, to evaluate
       the uncertainty in the location of the minimal theta value.
    `weights` - bool=False - if True, 'u' is actually weights, not uncertainties.
       Usually weights = u**-2.
    `parallel` - default True - Controls parallelization of the algorithm. Default
       uses all available cores. See `pwkit.parallel.make_parallel_helper`.

    Returns named tuple of:

    `thetas` - 1D array of values of theta statistic, same size as `periods`
    `imin` - index of smallest (best) value in `thetas`
    `pmin` - the `period` value with the smallest (best) `theta`
    `mc_tmins` - 1D array of size `nsmc` with Monte Carlo samplings of minimal
       theta values for shufflings of the data; assesses significance of the peak
    `mc_pvalue` - probability (between 0 and 1) of obtaining the best theta value
       in a randomly-shuffled dataset
    `mc_pmins` - 1D array of size `numc` with Monte Carlo samplings of best
       period values for noise-added data; assesses uncertainty of `pmin`
    `mc_puncert` - standard deviation of `mc_pmins`; approximate uncertainty
       on `pmin`.

    We don't do anything clever, so runtime scales at least as
      t.size * periods.size * nbin * nshift * (nsmc + numc + 1)."""

    t = np.asfarray (t)
    x = np.asfarray (x)
    u = np.asfarray (u)
    periods = np.asfarray (periods)
    t, x, u, periods = np.atleast_1d (t, x, u, periods)
    nbin = int (nbin)
    nshift = int (nshift)
    nsmc = int (nsmc)
    numc = int (numc)
    phelp = make_parallel_helper (parallel)

    if t.ndim != 1:
        raise ValueError ('`t` must be <= 1D')

    if x.shape != t.shape:
        raise ValueError ('`t` and `x` arguments must be the same size')

    if u.shape != t.shape:
        raise ValueError ('`t` and `u` arguments must be the same size')

    if periods.ndim != 1:
        raise ValueError ('`periods` must be <= 1D')

    if nbin < 2:
        raise ValueError ('`nbin` must be at least 2')

    if nshift < 1:
        raise ValueError ('`nshift` must be at least 1')

    if nsmc < 0:
        raise ValueError ('`nsmc` must be nonnegative')

    if numc < 0:
        raise ValueError ('`numc` must be nonnegative')

    # We can finally get started!

    if weights:
        wt = u
        u = wt ** -0.5
    else:
        wt = u ** -2

    v_all = weighted_variance (x, wt)

    with phelp.get_map () as map:
        get_thetas = lambda args: np.asarray (map (_map_one_theta, args))
        thetas = get_thetas ((t, x, wt, p, nbin, nshift, v_all)
                             for p in periods)
        imin = thetas.argmin ()
        pmin = periods[imin]

        # Now do the Monte Carlo jacknifing so that the caller can have some idea
        # as to the significance of the minimal value of `thetas`.

        mc_thetas = np.empty (periods.shape)
        mc_tmins = np.empty (nsmc)

        for i in xrange (nsmc):
            shuf = np.random.permutation (x.size)
            # Note that what we do here is very MapReduce-y. I'm not aware of
            # an easy way to implement this computation in that model, though.
            mc_thetas = get_thetas ((t, x[shuf], wt[shuf], p, nbin, nshift, v_all)
                                    for p in periods)
            mc_tmins[i] = mc_thetas.min ()

        mc_tmins.sort ()
        mc_pvalue = mc_tmins.searchsorted (thetas[imin]) / nsmc

        # Now add noise to assess the uncertainty of the period.

        mc_pmins = np.empty (numc)

        for i in xrange (numc):
            noised = np.random.normal (x, u)
            mc_thetas = get_thetas ((t, noised, wt, p, nbin, nshift, v_all)
                                    for p in periods)
            mc_pmins[i] = periods[mc_thetas.argmin ()]

        mc_pmins.sort ()
        mc_puncert = mc_pmins.std ()

    # All done.

    return PDMResult (thetas=thetas, imin=imin, pmin=pmin, mc_tmins=mc_tmins,
                      mc_pvalue=mc_pvalue, mc_pmins=mc_pmins,
                      mc_puncert=mc_puncert)
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
# vim:fenc=utf-8
#
# Copyright © 2015 Christopher C. Strelioff <chris.strelioff@gmail.com>
#
# Distributed under terms of the MIT license.

"""
api.py -- the main lastfm api interface.
"""
import time
import json

from urllib.parse import urlencode

from urllib.request import urlopen
from urllib.request import Request
from urllib.request import HTTPError
from urllib.request import URLError

class client(object):

    def __init__(self, key, secret):
        self.API_URL = "http://ws.audioscrobbler.com/2.0/"
        self.API_KEY = key
        self.API_SEC = secret

        self.last_call = time.time()

        self.kwargs = {"api_key": self.API_KEY, "format": "json"}

        # set to True when user authenticated
        self.authenticated = False

    def _throttle(self):
        """Limit to minimum of 0.2 seconds per call."""
        min_time = 0.2

        now = time.time()
        time_since_last = now - self.last_call

        if time_since_last < min_time:
            time.sleep(min_time - time_since_last)

        self.last_call = now

    def _send_request(self, args, **kwargs):
        """General request to the API."""
        self._throttle()

        kwargs.update(args)
        kwargs.update(self.kwargs)

        try:
            # format data and request
            data = urlencode(kwargs).encode('utf8')
            req = Request(self.API_URL, data)

            # query lastfm
            response = urlopen(req)

            # parse response data
            response_data = json.loads(response.readall().decode('utf8'))
        except HTTPError as e:
            print("HTTP error: {:}".format(e.code))
        except URLError as e:
            print("Network error: {}".format(e.reason.args[1]))

        return response_data

    def artist_get_tags(self, artist=None, mbid=None, user=None,
                        autocorrect=1):
        """Artist: get artist tags by specific user.

        http://www.last.fm/api/show/artist.getTags

        :param artist: artist name, required unless mbid
        :type artist: str
        :param mbid: musicbrainz id for the artist, optional
        :type mbid: str
        :param user: username, required if not authenticated
        :type user: str
        :param autocorrect: autocorrect spelling, [0|1]
        :type autocorrect: int
        :rtype: list if raw=0 or dict if raw=1
        """
        if not artist and not mbid:
            raise TypeError("Must provide artist or mbid")

        if not self.authenticated and not user:
            raise TypeError("If user not authenticated, must provide user.")

        args = {"method": "artist.gettags", "autocorrect": 1}

        if artist:
            args["artist"] = artist
        elif mbid:
            args["mbid"] = mbid

        if user:
            args["user"] = user

        return self._send_request(args)
<commit_msg>add artist_get_top_albums()<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
# vim:fenc=utf-8
#
# Copyright © 2015 Christopher C. Strelioff <chris.strelioff@gmail.com>
#
# Distributed under terms of the MIT license.

"""
api.py -- the main lastfm api interface.
"""
import time
import json

from urllib.parse import urlencode

from urllib.request import urlopen
from urllib.request import Request
from urllib.request import HTTPError
from urllib.request import URLError

class client(object):

    def __init__(self, key, secret):
        self.API_URL = "http://ws.audioscrobbler.com/2.0/"
        self.API_KEY = key
        self.API_SEC = secret

        self.last_call = time.time()

        self.kwargs = {"api_key": self.API_KEY, "format": "json"}

        # set to True when user authenticated
        self.authenticated = False

    def _throttle(self):
        """Limit to minimum of 0.2 seconds per call."""
        min_time = 0.2

        now = time.time()
        time_since_last = now - self.last_call

        if time_since_last < min_time:
            time.sleep(min_time - time_since_last)

        self.last_call = now

    def _send_request(self, args, **kwargs):
        """General request to the API."""
        self._throttle()

        kwargs.update(args)
        kwargs.update(self.kwargs)

        try:
            # format data and request
            data = urlencode(kwargs).encode('utf8')
            req = Request(self.API_URL, data)

            # query lastfm
            response = urlopen(req)

            # parse response data
            response_data = json.loads(response.readall().decode('utf8'))
        except HTTPError as e:
            print("HTTP error: {:}".format(e.code))
        except URLError as e:
            print("Network error: {}".format(e.reason.args[1]))

        return response_data

    def artist_get_tags(self, artist=None, mbid=None, user=None,
                        autocorrect=1):
        """Get the tags applied by an individual user to an artist on Last.fm.

        http://www.last.fm/api/show/artist.getTags

        :param artist: artist name, required unless mbid
        :type artist: str
        :param mbid: musicbrainz id for the artist, optional
        :type mbid: str
        :param user: username, required if not authenticated
        :type user: str
        :param autocorrect: autocorrect spelling, [0|1]
        :type autocorrect: int
        :rtype: dict
        """
        if not artist and not mbid:
            raise TypeError("Must provide artist or mbid")

        if not self.authenticated and not user:
            raise TypeError("If user not authenticated, must provide user.")

        args = {'method': 'artist.gettags',
                'autocorrect': autocorrect}

        if artist:
            args["artist"] = artist
        elif mbid:
            args["mbid"] = mbid

        if user:
            args["user"] = user

        return self._send_request(args)

    def artist_get_top_albums(self, artist=None, mbid=None, autocorrect=1,
                              page=1, limit=10):
        """Get the top albums for an artist on Last.fm, ordered by popularity.

        http://www.last.fm/api/show/artist.getTopAlbums

        :param artist: artist name, required unless mbid
        :type artist: str
        :param mbid: musicbrainz id for the artist, optional
        :type mbid: str
        :param autocorrect: autocorrect spelling, [0|1]
        :type autocorrect: int
        :param page: the page number to fetch, default page=1
        :type page: int
        :param limit: the number of results to fetch per page, default limit=10
        :type limit: int
        :rtype: dict
        """
        if not artist and not mbid:
            raise TypeError("Must provide artist or mbid")

        args = {'method': 'artist.gettopalbums',
                'autocorrect': autocorrect,
                'page': page,
                'limit': limit}

        if artist:
            args["artist"] = artist
        elif mbid:
            args["mbid"] = mbid

        return self._send_request(args)
<|endoftext|>