<commit_before>"""Defines distributions from which to sample conditional data."""

import numpy as np
from pylearn2.format.target_format import OneHotFormatter
from pylearn2.space import VectorSpace
from pylearn2.utils import sharedX
import theano
import theano.tensor as T
from theano.tensor.shared_randomstreams import RandomStreams

class Distribution(object):
    def get_space__init__(self, space):
        self.space = space

    def get_space(self):
        return self.space

    def get_total_dimension(self):
        return self.space.get_total_dimension()

    def sample(self, n):
        """
        Parameters
        ----------
        n : integer
            Number of samples to generate

        Returns
        -------
        samples : batch of members of output space
        """

        raise NotImplementedError("abstract method")

class OneHotDistribution(Distribution):
    """Randomly samples from a distribution of one-hot vectors."""

    def __init__(self, space, rng=None):
        super(OneHotDistribution, self).__init__(space)

        self.dim = space.get_total_dimension()
        self.formatter = OneHotFormatter(self.dim, dtype=space.dtype)

        self.rng = RandomStreams() if rng is None else rng

    def sample(self, n):
        idxs = self.rng.random_integers((n, 1), low=0, high=self.dim - 1)
        return self.formatter.theano_expr(idxs, mode='concatenate')

class KernelDensityEstimateDistribution(Distribution):
    """Randomly samples from a kernel density estimate yielded by a set
    of training points.

    Simple sampling procedure [1]:

    1. With training points $x_1, ... x_n$, sample a point $x_i$
       uniformly
    2. From original KDE, we have a kernel defined at point $x_i$;
       sample randomly from this kernel

    [1]: http://www.stat.cmu.edu/~cshalizi/350/lectures/28/lecture-28.pdf
    """

    def __init__(self, X, bandwidth=1, space=None, rng=None):
        """
        Parameters
        ----------
        X : ndarray of shape (num_examples, num_features)
            Training examples from which to generate a kernel density
            estimate

        bandwidth : float
            Bandwidth (or h, or sigma) of the generated kernels
        """

        assert X.ndim == 2
        if space is None:
            space = VectorSpace(dim=X.shape[1], dtype=X.dtype)

        # super(KernelDensityEstimateDistribution, self).__init__(space)

        self.X = sharedX(X, name='KDE_X')

        self.bandwidth = sharedX(bandwidth, name='bandwidth')
        self.rng = RandomStreams() if rng is None else rng

    def sample(self, n):
        # Sample $n$ training examples
        training_samples = self.X[self.rng.choice(size=(n,), a=self.X.shape[0], replace=True)]

        # Sample individually from each selected associated kernel
        #
        # (not well documented within NumPy / Theano, but rng.normal
        # call samples from a multivariate normal with diagonal
        # covariance matrix)
        ret, updates = theano.scan(fn=lambda sample: self.rng.normal(size=(self.X.shape[1],),
                                                                avg=sample, std=self.bandwidth,
                                                                dtype=theano.config.floatX),
                                   sequences=[training_samples])

        return ret, updates
<commit_msg>The correct solution is beautiful. Just took a few hours to reach it (the final solution doesn't seem to be documented anywhere in NumPy / Theano docs..)<commit_after>"""Defines distributions from which to sample conditional data."""

import numpy as np
from pylearn2.format.target_format import OneHotFormatter
from pylearn2.space import VectorSpace
from pylearn2.utils import sharedX
import theano
import theano.tensor as T
from theano.tensor.shared_randomstreams import RandomStreams

class Distribution(object):
    def get_space__init__(self, space):
        self.space = space

    def get_space(self):
        return self.space

    def get_total_dimension(self):
        return self.space.get_total_dimension()

    def sample(self, n):
        """
        Parameters
        ----------
        n : integer
            Number of samples to generate

        Returns
        -------
        samples : batch of members of output space
        """

        raise NotImplementedError("abstract method")

class OneHotDistribution(Distribution):
    """Randomly samples from a distribution of one-hot vectors."""

    def __init__(self, space, rng=None):
        super(OneHotDistribution, self).__init__(space)

        self.dim = space.get_total_dimension()
        self.formatter = OneHotFormatter(self.dim, dtype=space.dtype)

        self.rng = RandomStreams() if rng is None else rng

    def sample(self, n):
        idxs = self.rng.random_integers((n, 1), low=0, high=self.dim - 1)
        return self.formatter.theano_expr(idxs, mode='concatenate')

class KernelDensityEstimateDistribution(Distribution):
    """Randomly samples from a kernel density estimate yielded by a set
    of training points.

    Simple sampling procedure [1]:

    1. With training points $x_1, ... x_n$, sample a point $x_i$
       uniformly
    2. From original KDE, we have a kernel defined at point $x_i$;
       sample randomly from this kernel

    [1]: http://www.stat.cmu.edu/~cshalizi/350/lectures/28/lecture-28.pdf
    """

    def __init__(self, X, bandwidth=1, space=None, rng=None):
        """
        Parameters
        ----------
        X : ndarray of shape (num_examples, num_features)
            Training examples from which to generate a kernel density
            estimate

        bandwidth : float
            Bandwidth (or h, or sigma) of the generated kernels
        """

        assert X.ndim == 2
        if space is None:
            space = VectorSpace(dim=X.shape[1], dtype=X.dtype)

        # super(KernelDensityEstimateDistribution, self).__init__(space)

        self.X = sharedX(X, name='KDE_X')

        self.bandwidth = sharedX(bandwidth, name='bandwidth')
        self.rng = RandomStreams() if rng is None else rng

    def sample(self, n):
        # Sample $n$ training examples
        training_samples = self.X[self.rng.choice(size=(n,), a=self.X.shape[0], replace=True)]

        # Sample individually from each selected associated kernel
        #
        # (not well documented within NumPy / Theano, but rng.normal
        # call samples from a multivariate normal with diagonal
        # covariance matrix)
        ret = self.rng.normal(size=(n, self.X.shape[1]),
                              avg=training_samples, std=self.bandwidth,
                              dtype=theano.config.floatX)

        return ret
<|endoftext|>


<commit_before>from PyQt5.QtCore import QObject, pyqtSlot
from PyQt5.QtNetwork import QNetworkReply
from ucoinpy.api import bma
from .....tools.exceptions import NoPeerAvailable
from ..... import __version__
import logging
from aiohttp.errors import ClientError
import asyncio
import random
from socket import gaierror
import jsonschema
from distutils.version import StrictVersion

class BmaAccess(QObject):
    """
    This class is used to access BMA API.
    """

    __saved_requests = [str(bma.blockchain.Block), str(bma.blockchain.Parameters)]

    def __init__(self, data, network):
        """
        Constructor of a network

        :param dict data: The data present in this cache
        :param sakia.core.net.network.Network network: The network used to connect
        """
        super().__init__()
        self._data = data
        self._rollback_to = None
        self._pending_requests = {}
        self._network = network

    @classmethod
    def create(cls, network):
        """
        Initialize a new BMAAccess object with empty data.

        :param sakia.core.net.network.Network network:
        :return: A new BmaAccess object
        :rtype: sakia.core.net.api.bma.access.BmaAccess
        """
        return cls({}, network)

    @property
    def data(self):
        return self._data.copy()

    def load_from_json(self, json_data):
        """
        Put data in the cache from json datas.

        :param dict data: The cache in json format
        """
        data = {}
        for entry in json_data['entries']:
            key = entry['key']
            cache_key = (key[0], key[1], key[2], key[3], key[4])
            data[cache_key] = entry['value']
        self._data = data
        self._rollback_to = json_data['rollback']

    def jsonify(self):
        """
        Get the cache in json format

        :return: The cache as a dict in json format
        """
        data = {k: self._data[k] for k in self._data.keys()}
        entries = []
        for d in data:
            entries.append({'key': d,
                            'value': data[d]})
        return {'rollback': self._rollback_to,
                'entries': entries}

    @staticmethod
    def _gen_cache_key(request, req_args, get_args):
        return (str(request),
                str(tuple(frozenset(sorted(req_args.keys())))),
                str(tuple(frozenset(sorted(req_args.values())))),
                str(tuple(frozenset(sorted(get_args.keys())))),
                str(tuple(frozenset(sorted(get_args.values())))))

    def _compare_json(self, first, second):
        """
        Compare two json dicts
        :param first: the first dictionnary
        :param second: the second dictionnary
        :return: True if the json dicts are the same
        :rtype: bool
        """
        def ordered(obj):
            if isinstance(obj, dict):
                try:
                    return sorted((k, ordered(v)) for k, v in obj.items())
                except TypeError:
                    return obj
            if isinstance(obj, list):
                try:
                    return sorted(ordered(x) for x in obj)
                except TypeError:
                    return obj
            else:
                return obj
        return ordered(first) == ordered(second)

    def _get_from_cache(self, request, req_args, get_args):
        """
        Get data from the cache
        :param request: The requested data
        :param cache_key: The key
        :rtype: tuple[bool, dict]
        """
        cache_key = BmaAccess._gen_cache_key(request, req_args, get_args)
        if cache_key in self._data.keys():
            cached_data = self._data[cache_key]
            need_reload = True
            # If we detected a rollback
            # We reload if we don't know if this block changed or not
            if self._rollback_to:
                if request is bma.blockchain.Block:
                    if get_args["number"] >= self._rollback_to:
                        need_reload = True
                if request is bma.blockchain.Parameters and self._rollback_to == 0:
                    need_reload = True
            elif str(request) in BmaAccess.__saved_requests \
                or cached_data['metadata']['block_hash'] == self._network.current_blockid.sha_hash:
                need_reload = False
            ret_data = cached_data['value']
        else:
            need_reload = True
            ret_data = None
        return need_reload, ret_data

    def _update_rollback(self, request, req_args, get_args, data):
        """
        Update the rollback

        If the request is a bma/blockchain/Block, we check if
        the hash answered is the same as our hash, in which case,
        we know that the rollback didn't reset blocks before this one
        :param class request: A bma request class calling for data
        :param dict req_args: Arguments to pass to the request constructor
        :param dict get_args: Arguments to pass to the request __get__ method
        :param dict data: Json data got from the blockchain
        """
        if self._rollback_to and request is bma.blockchain.Block:
            if get_args['number'] >= self._rollback_to:
                cache_key = BmaAccess._gen_cache_key(request, req_args, get_args)
                if cache_key in self._data and self._data[cache_key]['value']['hash'] == data['hash']:
                    self._rollback_to = get_args['number']

    def _update_cache(self, request, req_args, get_args, data):
        """
        Update data in cache and returns True if cached data changed
        :param class request: A bma request class calling for data
        :param dict req_args: Arguments to pass to the request constructor
        :param dict get_args: Arguments to pass to the request __get__ method
        :param dict data: Json data to save in cache
        :return: True if data changed
        :rtype: bool
        """
        self._update_rollback(request, req_args, get_args, data)

        cache_key = BmaAccess._gen_cache_key(request, req_args, get_args)
        if cache_key not in self._data:
            self._data[cache_key] = {'metadata': {},
                                     'value': {}}

        self._data[cache_key]['metadata']['block_number'] = self._network.current_blockid.number
        self._data[cache_key]['metadata']['block_hash'] = self._network.current_blockid.sha_hash
        self._data[cache_key]['metadata']['sakia_version'] = __version__
        if not self._compare_json(self._data[cache_key]['value'], data):
            self._data[cache_key]['value'] = data
            return True
        return False

    def _invalidate_cache(self, post_request):
        """
        Invalidate data depending on posted request
        :param class post_request: The posted request
        """
        invalidated = {bma.wot.Add: bma.wot.Lookup}
        if post_request in invalidated:
            invalidated_cache = self._data.copy()
            for data in self._data:
                if data[0] == str(invalidated[post_request]):
                    invalidated_cache.pop(data)
            self._data = invalidated_cache

    def rollback(self):
        """
        When a rollback is detected, we move the rollback cursor to 0
        """
        self._rollback_to = 0

    def filter_nodes(self, request, nodes):
        def compare_versions(node, version):
            if node.version and node.version != '':
                try:
                    return StrictVersion(node.version) > StrictVersion(version)
                except TypeError:
                    return False
            else:
                return False
        filters = {
            bma.ud.History: lambda n: compare_versions(n, "0.11.0"),
            bma.tx.History: lambda n: compare_versions(n, "0.11.0")
        }
        if request in filters:
            return [n for n in nodes if filters[request](n)]
        else:
            return nodes

    async def future_request(self, request, req_args={}, get_args={}):
        """
        Start a request to the network and returns a future.

        :param class request: A bma request class calling for data
        :param dict req_args: Arguments to pass to the request constructor
        :param dict get_args: Arguments to pass to the request __get__ method
        :return: The future data
        :rtype: dict
        """
        data = self._get_from_cache(request, req_args, get_args)
        need_reload = data[0]
        json_data = data[1]

        nodes = self.filter_nodes(request, self._network.synced_nodes)
        if need_reload and len(nodes) > 0:
            tries = 0
            while tries < 3:
                node = random.choice(nodes)
                conn_handler = node.endpoint.conn_handler()
                req = request(conn_handler, **req_args)
                try:
                    json_data = await req.get(**get_args)
                    self._update_cache(request, req_args, get_args, json_data)
                    return json_data
                except ValueError as e:
                    if '404' in str(e) or '400' in str(e):
                        raise
                    tries += 1
                except (ClientError, gaierror, asyncio.TimeoutError) as e:
                    tries += 1
                except jsonschema.ValidationError as e:
                    logging.debug(str(e))
                    tries += 1
        if len(nodes) == 0 or json_data is None:
            raise NoPeerAvailable("", len(nodes))
        return json_data

    async def simple_request(self, request, req_args={}, get_args={}):
        """
        Start a request to the network but don't cache its result.

        :param class request: A bma request class calling for data
        :param dict req_args: Arguments to pass to the request constructor
        :param dict get_args: Arguments to pass to the request __get__ method
        :return: The returned data if cached = True else return the QNetworkReply
        """
        nodes = self.filter_nodes(request, self._network.synced_nodes)
        if len(nodes) > 0:
            node = random.choice(nodes)
            req = request(node.endpoint.conn_handler(), **req_args)
            tries = 0
            json_data = None
            while tries < 3:
                try:
                    json_data = await req.get(**get_args)
                    return json_data
                except ValueError as e:
                    if '404' in str(e) or '400' in str(e):
                        raise
                    tries += 1
                except (ClientError, gaierror, asyncio.TimeoutError) as e:
                    tries += 1
                except jsonschema.ValidationError as e:
                    logging.debug(str(e))
                    tries += 1
        if len(nodes) == 0 or not json_data:
            raise NoPeerAvailable("", len(nodes))
        return json_data

    async def broadcast(self, request, req_args={}, post_args={}):
        """
        Broadcast data to a network.
        Sends the data to all knew nodes.

        :param request: A ucoinpy bma request class
        :param req_args: Arguments to pass to the request constructor
        :param post_args: Arguments to pass to the request __post__ method
        :return: All nodes replies
        :rtype: tuple of QNetworkReply

        .. note:: If one node accept the requests (returns 200),
        the broadcast should be considered accepted by the network.
        """
        nodes = self._network.online_nodes
        replies = []
        if len(nodes) > 0:
            for node in nodes:
                logging.debug("Trying to connect to : " + node.pubkey)
                conn_handler = node.endpoint.conn_handler()
                req = request(conn_handler, **req_args)
                try:
                    reply = await req.post(**post_args)
                    replies.append(reply)
                except ValueError as e:
                    if '404' in str(e) or '400' in str(e):
                        raise
                except (ClientError, gaierror):
                    pass
                except asyncio.TimeoutError:
                    pass
            self._invalidate_cache(request)
        else:
            raise NoPeerAvailable("", len(nodes))
        return tuple(replies)
<commit_msg>Remove QtNetwork<commit_after>from PyQt5.QtCore import QObject, pyqtSlot
from ucoinpy.api import bma
from .....tools.exceptions import NoPeerAvailable
from ..... import __version__
import logging
from aiohttp.errors import ClientError
import asyncio
import random
from socket import gaierror
import jsonschema
from distutils.version import StrictVersion

class BmaAccess(QObject):
    """
    This class is used to access BMA API.
    """

    __saved_requests = [str(bma.blockchain.Block), str(bma.blockchain.Parameters)]

    def __init__(self, data, network):
        """
        Constructor of a network

        :param dict data: The data present in this cache
        :param sakia.core.net.network.Network network: The network used to connect
        """
        super().__init__()
        self._data = data
        self._rollback_to = None
        self._pending_requests = {}
        self._network = network

    @classmethod
    def create(cls, network):
        """
        Initialize a new BMAAccess object with empty data.

        :param sakia.core.net.network.Network network:
        :return: A new BmaAccess object
        :rtype: sakia.core.net.api.bma.access.BmaAccess
        """
        return cls({}, network)

    @property
    def data(self):
        return self._data.copy()

    def load_from_json(self, json_data):
        """
        Put data in the cache from json datas.

        :param dict data: The cache in json format
        """
        data = {}
        for entry in json_data['entries']:
            key = entry['key']
            cache_key = (key[0], key[1], key[2], key[3], key[4])
            data[cache_key] = entry['value']
        self._data = data
        self._rollback_to = json_data['rollback']

    def jsonify(self):
        """
        Get the cache in json format

        :return: The cache as a dict in json format
        """
        data = {k: self._data[k] for k in self._data.keys()}
        entries = []
        for d in data:
            entries.append({'key': d,
                            'value': data[d]})
        return {'rollback': self._rollback_to,
                'entries': entries}

    @staticmethod
    def _gen_cache_key(request, req_args, get_args):
        return (str(request),
                str(tuple(frozenset(sorted(req_args.keys())))),
                str(tuple(frozenset(sorted(req_args.values())))),
                str(tuple(frozenset(sorted(get_args.keys())))),
                str(tuple(frozenset(sorted(get_args.values())))))

    def _compare_json(self, first, second):
        """
        Compare two json dicts
        :param first: the first dictionnary
        :param second: the second dictionnary
        :return: True if the json dicts are the same
        :rtype: bool
        """
        def ordered(obj):
            if isinstance(obj, dict):
                try:
                    return sorted((k, ordered(v)) for k, v in obj.items())
                except TypeError:
                    return obj
            if isinstance(obj, list):
                try:
                    return sorted(ordered(x) for x in obj)
                except TypeError:
                    return obj
            else:
                return obj
        return ordered(first) == ordered(second)

    def _get_from_cache(self, request, req_args, get_args):
        """
        Get data from the cache
        :param request: The requested data
        :param cache_key: The key
        :rtype: tuple[bool, dict]
        """
        cache_key = BmaAccess._gen_cache_key(request, req_args, get_args)
        if cache_key in self._data.keys():
            cached_data = self._data[cache_key]
            need_reload = True
            # If we detected a rollback
            # We reload if we don't know if this block changed or not
            if self._rollback_to:
                if request is bma.blockchain.Block:
                    if get_args["number"] >= self._rollback_to:
                        need_reload = True
                if request is bma.blockchain.Parameters and self._rollback_to == 0:
                    need_reload = True
            elif str(request) in BmaAccess.__saved_requests \
                or cached_data['metadata']['block_hash'] == self._network.current_blockid.sha_hash:
                need_reload = False
            ret_data = cached_data['value']
        else:
            need_reload = True
            ret_data = None
        return need_reload, ret_data

    def _update_rollback(self, request, req_args, get_args, data):
        """
        Update the rollback

        If the request is a bma/blockchain/Block, we check if
        the hash answered is the same as our hash, in which case,
        we know that the rollback didn't reset blocks before this one
        :param class request: A bma request class calling for data
        :param dict req_args: Arguments to pass to the request constructor
        :param dict get_args: Arguments to pass to the request __get__ method
        :param dict data: Json data got from the blockchain
        """
        if self._rollback_to and request is bma.blockchain.Block:
            if get_args['number'] >= self._rollback_to:
                cache_key = BmaAccess._gen_cache_key(request, req_args, get_args)
                if cache_key in self._data and self._data[cache_key]['value']['hash'] == data['hash']:
                    self._rollback_to = get_args['number']

    def _update_cache(self, request, req_args, get_args, data):
        """
        Update data in cache and returns True if cached data changed
        :param class request: A bma request class calling for data
        :param dict req_args: Arguments to pass to the request constructor
        :param dict get_args: Arguments to pass to the request __get__ method
        :param dict data: Json data to save in cache
        :return: True if data changed
        :rtype: bool
        """
        self._update_rollback(request, req_args, get_args, data)

        cache_key = BmaAccess._gen_cache_key(request, req_args, get_args)
        if cache_key not in self._data:
            self._data[cache_key] = {'metadata': {},
                                     'value': {}}

        self._data[cache_key]['metadata']['block_number'] = self._network.current_blockid.number
        self._data[cache_key]['metadata']['block_hash'] = self._network.current_blockid.sha_hash
        self._data[cache_key]['metadata']['sakia_version'] = __version__
        if not self._compare_json(self._data[cache_key]['value'], data):
            self._data[cache_key]['value'] = data
            return True
        return False

    def _invalidate_cache(self, post_request):
        """
        Invalidate data depending on posted request
        :param class post_request: The posted request
        """
        invalidated = {bma.wot.Add: bma.wot.Lookup}
        if post_request in invalidated:
            invalidated_cache = self._data.copy()
            for data in self._data:
                if data[0] == str(invalidated[post_request]):
                    invalidated_cache.pop(data)
            self._data = invalidated_cache

    def rollback(self):
        """
        When a rollback is detected, we move the rollback cursor to 0
        """
        self._rollback_to = 0

    def filter_nodes(self, request, nodes):
        def compare_versions(node, version):
            if node.version and node.version != '':
                try:
                    return StrictVersion(node.version) > StrictVersion(version)
                except TypeError:
                    return False
            else:
                return False
        filters = {
            bma.ud.History: lambda n: compare_versions(n, "0.11.0"),
            bma.tx.History: lambda n: compare_versions(n, "0.11.0")
        }
        if request in filters:
            return [n for n in nodes if filters[request](n)]
        else:
            return nodes

    async def future_request(self, request, req_args={}, get_args={}):
        """
        Start a request to the network and returns a future.

        :param class request: A bma request class calling for data
        :param dict req_args: Arguments to pass to the request constructor
        :param dict get_args: Arguments to pass to the request __get__ method
        :return: The future data
        :rtype: dict
        """
        data = self._get_from_cache(request, req_args, get_args)
        need_reload = data[0]
        json_data = data[1]

        nodes = self.filter_nodes(request, self._network.synced_nodes)
        if need_reload and len(nodes) > 0:
            tries = 0
            while tries < 3:
                node = random.choice(nodes)
                conn_handler = node.endpoint.conn_handler()
                req = request(conn_handler, **req_args)
                try:
                    json_data = await req.get(**get_args)
                    self._update_cache(request, req_args, get_args, json_data)
                    return json_data
                except ValueError as e:
                    if '404' in str(e) or '400' in str(e):
                        raise
                    tries += 1
                except (ClientError, gaierror, asyncio.TimeoutError) as e:
                    tries += 1
                except jsonschema.ValidationError as e:
                    logging.debug(str(e))
                    tries += 1
        if len(nodes) == 0 or json_data is None:
            raise NoPeerAvailable("", len(nodes))
        return json_data

    async def simple_request(self, request, req_args={}, get_args={}):
        """
        Start a request to the network but don't cache its result.

        :param class request: A bma request class calling for data
        :param dict req_args: Arguments to pass to the request constructor
        :param dict get_args: Arguments to pass to the request __get__ method
        :return: The returned data
        """
        nodes = self.filter_nodes(request, self._network.synced_nodes)
        if len(nodes) > 0:
            node = random.choice(nodes)
            req = request(node.endpoint.conn_handler(), **req_args)
            tries = 0
            json_data = None
            while tries < 3:
                try:
                    json_data = await req.get(**get_args)
                    return json_data
                except ValueError as e:
                    if '404' in str(e) or '400' in str(e):
                        raise
                    tries += 1
                except (ClientError, gaierror, asyncio.TimeoutError) as e:
                    tries += 1
                except jsonschema.ValidationError as e:
                    logging.debug(str(e))
                    tries += 1
        if len(nodes) == 0 or not json_data:
            raise NoPeerAvailable("", len(nodes))
        return json_data

    async def broadcast(self, request, req_args={}, post_args={}):
        """
        Broadcast data to a network.
        Sends the data to all knew nodes.

        :param request: A ucoinpy bma request class
        :param req_args: Arguments to pass to the request constructor
        :param post_args: Arguments to pass to the request __post__ method
        :return: All nodes replies
        :rtype: tuple of aiohttp replies

        .. note:: If one node accept the requests (returns 200),
        the broadcast should be considered accepted by the network.
        """
        nodes = self._network.online_nodes
        replies = []
        if len(nodes) > 0:
            for node in nodes:
                logging.debug("Trying to connect to : " + node.pubkey)
                conn_handler = node.endpoint.conn_handler()
                req = request(conn_handler, **req_args)
                try:
                    reply = await req.post(**post_args)
                    replies.append(reply)
                except ValueError as e:
                    if '404' in str(e) or '400' in str(e):
                        raise
                except (ClientError, gaierror):
                    pass
                except asyncio.TimeoutError:
                    pass
            self._invalidate_cache(request)
        else:
            raise NoPeerAvailable("", len(nodes))
        return tuple(replies)
<|endoftext|>


<commit_before>import modod, modod.dre
import modod.dre as ext
from modod.dre import DRE
import argparse

def main():
    parser = argparse.ArgumentParser(description="This tool will simplify an deterministic regular expression.")
    parser.add_argument("expressions",help="The expressions to simplify",nargs="*")
    parser.add_argument("-v","--verbose",help="Print the parsed expressions",action="store_true")
    parser.add_argument("--graph",metavar="DOT_FILE", help="Dump the graphs in DOT format into DOT_FILE.n.(in|out).dot")
    parser.add_argument("--letters", help="Interpret terminal symbols (abc) as letter sequences (a,b,c).", action="store_true")
    parser.add_argument("--chargroup", help="Attempt to print single-letter choices as character groups.", choices=['none', 'complete', 'all'], default='complete')
    parser.add_argument("--join", help="Remove duplicate choice expressions", action="store_true")
    parser.add_argument("--steps", help="Print individual steps.", action="store_true")
    args=parser.parse_args()
    if args.letters:
        modod._lexerExt.letters = True
        modod.dre.Concatenation._label = ''
    modod.charGroup = {
        'none': modod.CHARGROUP_NONE,
        'complete': modod.CHARGROUP_COMPLETE,
        'all': modod.CHARGROUP_PARTIAL
    }[args.chargroup]
    def lineReader():
        try:
            while True:
                yield input()
        except EOFError:
            pass
    expressions = map(DRE.fromString, args.expressions or lineReader())
    f = ['{b}', '{a} -> {b}'][args.verbose]
    for i,a in enumerate(expressions):
        b = a.simplify(args.steps)
        if args.steps:
            for j, (step, x) in enumerate(b):
                print('#{} {:10} {}'.format(j, step, x.toString()))
                if args.graph:
                    y = open('{0}.{1}.{2}.dot'.format(args.graph, i+1, j+1), 'w+')
                    y.write(x.toDOTString())
                    y.close()
            print()
            b = x
        else:
            if args.join:
                b = ext.reduceChoiceNary(b)
            print (f.format(a=a.toString(), b=b.toString()))
        if args.graph:
            x = open('{0}.{1}.in.dot'.format(args.graph, i+1), 'w+')
            y = open('{0}.{1}.out.dot'.format(args.graph, i+1), 'w+')
            x.write(a.toDOTString())
            y.write(b.toDOTString())
            x.close()
            y.close()

<commit_msg>Simplify (interactive) Abbruch ohne Fehler bei:<commit_after>import modod, modod.dre
import modod.dre as ext
from modod.dre import DRE
import argparse

def main():
    parser = argparse.ArgumentParser(description="This tool will simplify an deterministic regular expression.")
    parser.add_argument("expressions",help="The expressions to simplify",nargs="*")
    parser.add_argument("-v","--verbose",help="Print the parsed expressions",action="store_true")
    parser.add_argument("--graph",metavar="DOT_FILE", help="Dump the graphs in DOT format into DOT_FILE.n.(in|out).dot")
    parser.add_argument("--letters", help="Interpret terminal symbols (abc) as letter sequences (a,b,c).", action="store_true")
    parser.add_argument("--chargroup", help="Attempt to print single-letter choices as character groups.", choices=['none', 'complete', 'all'], default='complete')
    parser.add_argument("--join", help="Remove duplicate choice expressions", action="store_true")
    parser.add_argument("--steps", help="Print individual steps.", action="store_true")
    args=parser.parse_args()
    if args.letters:
        modod._lexerExt.letters = True
        modod.dre.Concatenation._label = ''
    modod.charGroup = {
        'none': modod.CHARGROUP_NONE,
        'complete': modod.CHARGROUP_COMPLETE,
        'all': modod.CHARGROUP_PARTIAL
    }[args.chargroup]
    def lineReader():
        try:
            line = input().strip()
            while line:
                yield line
                line = input().strip()
        except EOFError:
            pass
        except KeyboardInterrupt:
            pass
    expressions = map(DRE.fromString, args.expressions or lineReader())
    f = ['{b}', '{a} -> {b}'][args.verbose]
    for i,a in enumerate(expressions):
        b = a.simplify(args.steps)
        if args.steps:
            for j, (step, x) in enumerate(b):
                print('#{} {:10} {}'.format(j, step, x.toString()))
                if args.graph:
                    y = open('{0}.{1}.{2}.dot'.format(args.graph, i+1, j+1), 'w+')
                    y.write(x.toDOTString())
                    y.close()
            print()
            b = x
        else:
            if args.join:
                b = ext.reduceChoiceNary(b)
            print (f.format(a=a.toString(), b=b.toString()))
        if args.graph:
            x = open('{0}.{1}.in.dot'.format(args.graph, i+1), 'w+')
            y = open('{0}.{1}.out.dot'.format(args.graph, i+1), 'w+')
            x.write(a.toDOTString())
            y.write(b.toDOTString())
            x.close()
            y.close()

<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# Copyright (c) 2012 Jérémie DECOCK (http://www.jdhp.org)

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

"""
TODO...
"""

import argparse
import feedparser
import json
import os.path
import shutil
import urllib.request

DB_PATH = "podcasts.json"
OUTPUT_DIR_PATH = "./mp3"

def download_file(file_url, file_output_path, http_headers_dict={}):
    """
    TODO...
    """
    request = urllib.request.Request(file_url, data=None, headers= http_headers_dict)
    with urllib.request.urlopen(request) as response, open(file_output_path, 'wb') as out_file:
        shutil.copyfileobj(response, out_file)

def download_podcast(url):
    """
    TODO...
    """
    feed = feedparser.parse(url)

    #print("Channel URL:", feed["url"])
    #print("Channel Title:", feed["channel"]["title"])
    #print("Channel Desc:", feed["channel"]["description"])

    for item in feed["items"]:
        #print(item.keys())
        item_title = item['title']

        for link in item["links"]:
            if link['type'] == 'audio/mpeg':
                item_url = link['href']

                file_name = item_url.split('/')[-1]
                file_name = file_name.split('%')[-1]

                file_name, file_extension = os.path.splitext(file_name)

                file_name = file_name + "_" + item_title + file_extension
                file_name = '_'.join(file_name.split())                                    # replace spaces with '_'
                file_name = ''.join(c for c in file_name if (c.isalnum() or c in ".-_"))   # remove special char in file_name
                file_name = '_'.join([c for c in file_name.split('_') if c != ''])         # replace multiples '_' with simple '_'
                file_name = '.'.join(file_name.split('_.'))
                file_name = file_name.lower()

                file_path = os.path.join(OUTPUT_DIR_PATH, file_name)

                # Download the files that haven't been downloaded yet
                if not os.path.exists(file_path):
                    print("Download", item_url, " -> ", file_path)
                    download_file(item_url, file_path)

    return feed

def main():
    """
    TODO...
    """
    parser = argparse.ArgumentParser(description='An argparse snippet.')

    parser.add_argument("url", nargs=1, metavar="URL",
                        help="URL of the RSS document to parse")

    args = parser.parse_args()
    rss_url = args.url[0]

    # Load the podcast db
    try:
        with open(DB_PATH, "r") as fd:
            feed_dict = json.load(fd)
    except FileNotFoundError:
        feed_dict = {}

    # Get last feeds
    last_feeds_dict = download_podcast(rss_url)

    # Update the podcast db
    feed_dict.update(last_feeds_dict)

    # Save the podcast db
    with open(DB_PATH, "w") as fd:
        json.dump(feed_dict, fd, sort_keys=True, indent=4)  # pretty print format

if __name__ == '__main__':
    main()

<commit_msg>Improve output file names.<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# Copyright (c) 2012 Jérémie DECOCK (http://www.jdhp.org)

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

"""
TODO...
"""

import argparse
import feedparser
import json
import os.path
import shutil
import urllib.request

DB_PATH = "podcasts.json"
OUTPUT_DIR_PATH = "./mp3"

def download_file(file_url, file_output_path, http_headers_dict={}):
    """
    TODO...
    """
    request = urllib.request.Request(file_url, data=None, headers= http_headers_dict)
    with urllib.request.urlopen(request) as response, open(file_output_path, 'wb') as out_file:
        shutil.copyfileobj(response, out_file)

def download_podcast(url):
    """
    TODO...
    """
    feed = feedparser.parse(url)

    #print("Channel URL:", feed["url"])
    #print("Channel Title:", feed["channel"]["title"])
    #print("Channel Desc:", feed["channel"]["description"])

    channel_title = feed["channel"]["title"]

    for item in feed["items"]:
        #print(item.keys())

        pub_time = item["published_parsed"]
        pub_time_str = "{}-{:02d}-{:02d}_{:02d}h{:02d}".format(pub_time.tm_year, pub_time.tm_mon, pub_time.tm_mday, pub_time.tm_hour, pub_time.tm_min)

        item_title = item['title']

        for link in item["links"]:
            if link['type'] == 'audio/mpeg':
                item_url = link['href']

                file_name = item_url.split('/')[-1]
                file_name = file_name.split('%')[-1]

                file_name, file_extension = os.path.splitext(file_name)

                #file_name = channel_title + "_" + file_name + "_" + item_title + file_extension
                file_name = channel_title + "_" + pub_time_str + "_" + item_title + file_extension

                file_name = '_'.join(file_name.split())                                    # replace spaces with '_'
                file_name = ''.join(c for c in file_name if (c.isalnum() or c in ".-_"))   # remove special char in file_name
                file_name = '_'.join([c for c in file_name.split('_') if c != ''])         # replace multiples '_' with simple '_'
                file_name = '.'.join(file_name.split('_.'))
                file_name = file_name.lower()

                file_path = os.path.join(OUTPUT_DIR_PATH, file_name)

                # Download the files that haven't been downloaded yet
                if not os.path.exists(file_path):
                    print("Download", item_url, " -> ", file_path)
                    download_file(item_url, file_path)

    return feed

def main():
    """
    TODO...
    """
    parser = argparse.ArgumentParser(description='An argparse snippet.')

    parser.add_argument("url", nargs=1, metavar="URL",
                        help="URL of the RSS document to parse")

    args = parser.parse_args()
    rss_url = args.url[0]

    # Load the podcast db
    try:
        with open(DB_PATH, "r") as fd:
            feed_dict = json.load(fd)
    except FileNotFoundError:
        feed_dict = {}

    # Get last feeds
    last_feeds_dict = download_podcast(rss_url)

    # Update the podcast db
    feed_dict.update(last_feeds_dict)

    # Save the podcast db
    with open(DB_PATH, "w") as fd:
        json.dump(feed_dict, fd, sort_keys=True, indent=4)  # pretty print format

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before>"""Implementation of :class:`jukeboxcore.reftrack.ReftypeInterface` for Assets."""
from collections import defaultdict

import maya.cmds as cmds

from jukeboxcore import djadapter
from jukeboxcore.filesys import JB_File
from jukeboxcore.reftrack import ReftypeInterface
from jukeboxcore.filesys import TaskFileInfo
from jukeboxcore.gui import djitemdata
from jukeboxcore.gui.treemodel import TreeModel, TreeItem, ListItemData
from jukeboxcore.gui.filesysitemdata import TaskFileInfoItemData
from jukeboxmaya import common
from jukeboxmaya import reftrack

class AssetReftypeInterface(ReftypeInterface):
    """Interface for handling the content of released assets in the reference workflow.
    """

    def __init__(self, refobjinter):
        """Initialize a new AssetReftypeInterface that uses the given RefobjInterface

        :param refobjinter: the interface to handle the reftrack nodes
        :type refobjinter: :class:`jukeboxmaya.refobjinter.MayaRefobjInterface`
        :raises: None
        """
        super(AssetReftypeInterface, self).__init__(refobjinter)

    def is_replaceable(self, refobj):
        """Return whether the given reference of the refobject is replaceable or
        if it should just get deleted and loaded again.

        Returns True, because Maya can replace references.

        :param refobj: the refobject to query
        :type refobj: refobj
        :returns: True, if replaceable
        :rtype: bool
        :raises: NotImplementedError
        """
        return True

    def get_scenenode(self, nodes):
        """Get the scenenode in the given nodes

        There should only be one scenenode in nodes!

        :param nodes:
        :type nodes:
        :returns: None
        :rtype: None
        :raises: AssertionError
        """
        scenenodes = cmds.ls(nodes, type='jb_sceneNode')
        assert scenenodes, "Found no scene nodes!"
        assert len(scenenodes) == 1, "Found to many scene nodes! %s" % scenenodes
        return scenenodes[0]

    def reference(self, refobj, taskfileinfo):
        """Reference the given taskfileinfo into the scene and return the created reference node

        The created reference node will be used on :meth:`RefobjInterface.set_reference` to
        set the reference on a reftrack node.
        Do not call :meth:`RefobjInterface.set_reference` yourself.

        This will also create a group node and group all dagnodes under a appropriate node.

        :param refobj: the reftrack node that will be linked to the reference
        :type refobj: str
        :param taskfileinfo: The taskfileinfo that holds the information for what to reference
        :type taskfileinfo: :class:`jukeboxcore.filesys.TaskFileInfo`
        :returns: the reference node that was created and should set on the appropriate reftrack node
        :rtype: str
        :raises: None
        """
        # work in root namespace
        with common.preserve_namespace(":"):
            jbfile = JB_File(taskfileinfo)
            filepath = jbfile.get_fullpath()
            ns_suggestion = reftrack.get_namespace(taskfileinfo)
            reffile = cmds.file(filepath, reference=True, namespace=ns_suggestion)
            node = cmds.referenceQuery(reffile, referenceNode=True)  # get reference node
            ns = cmds.referenceQuery(node, namespace=True)  # query the actual new namespace
            content = cmds.namespaceInfo(ns, listOnlyDependencyNodes=True, dagPath=True)  # get the content
            # connect reftrack with scenenode
            scenenode = self.get_scenenode(content)
            self.get_refobjinter().connect_reftrack_scenenode(refobj, scenenode)
            dagcontent = cmds.ls(content, dag=True, ap=True)  # get only the dagnodes so we can group them
            if not dagcontent:
                return node  # no need for a top group if there are not dagnodes to group
            # group the dagnodes
            grpname = reftrack.get_groupname(taskfileinfo)
            reftrack.group_content(dagcontent, ns, grpname, "jb_asset")
            return node

    def load(self, refobj, reference):
        """Load the given reference

        Load in this case means, that a reference is already in the scene
        but it is not in a loaded state.
        Loading the reference means, that the actual data will be read.

        :param refobj: the reftrack node that is linked to the reference
        :type refobj: str
        :param reference: the reference node
        :type refobj: str
        :returns: None
        :rtype: None
        :raises: None
        """
        cmds.file(loadReference=reference)

    def unload(self, refobj, reference):
        """Unload the given reference

        Unload in this case means, that a reference is stays in the scene
        but it is not in a loaded state.
        So there is a reference, but data is not read from it.

        :param refobj: the refobj that is linked to the reference
        :param reference: the reference object. E.g. in Maya a reference node
        :returns: None
        :rtype: None
        :raises: None
        """
        cmds.file(unloadReference=reference)

    def replace(self, refobj, reference, taskfileinfo):
        """Replace the given reference with the given taskfileinfo

        :param refobj: the refobj that is linked to the reference
        :param reference: the reference object. E.g. in Maya a reference node
        :param taskfileinfo: the taskfileinfo that will replace the old entity
        :type taskfileinfo: :class:`jukeboxcore.filesys.TaskFileInfo`
        :returns: None
        :rtype: None
        :raises: None
        """
        jbfile = JB_File(taskfileinfo)
        filepath = jbfile.get_fullpath()
        cmds.file(filepath, loadReference=reference)
        ns = cmds.referenceQuery(reference, namespace=True)  # query the actual new namespace
        content = cmds.namespaceInfo(ns, listOnlyDependencyNodes=True)  # get the content
        scenenode = self.get_scenenode(content) # get the scene node
        self.get_refobjinter().connect_reftrack_scenenode(refobj, scenenode)

    def delete(self, refobj):
        """Delete the content of the given refobj

        :param refobj: the refobj that represents the content that should be deleted
        :type refobj: refobj
        :returns: None
        :rtype: None
        :raises: None
        """
        common.disconnect_node(refobj)
        refobjinter = self.get_refobjinter()
        reference = refobjinter.get_reference(refobj)
        if reference:
            cmds.file(reference, removeReference=True, force=True)
        else:
            ns = cmds.getAttr("%s.namespace" % refobj)
            content = cmds.namespaceInfo(ns, listNamespace=True)
            cmds.delete(content)
            cmds.namespace(removeNamespace=ns)

    def import_reference(self, refobj, reference):
        """Import the given reference

        The reference of the refobj will be set to None automatically afterwards with
        :meth:`RefobjInterface.set_reference`

        :param refobj: the refobj that is linked to the reference
        :param reference: the reference object. E.g. in Maya a reference node
        :returns: None
        :rtype: None
        :raises: None
        """
        f = cmds.referenceQuery(reference, filename=True)
        cmds.file(f, importReference=True)

    def import_taskfile(self, refobj, taskfileinfo):
        """Import the given taskfileinfo and update the refobj

        :param refobj: the refobject
        :type refobj: refobject
        :param taskfileinfo: the taskfileinfo to reference
        :type taskfileinfo: :class:`jukeboxcore.filesys.TaskFileInfo`
        :returns: None
        :rtype: None
        :raises: None
        """
        # work in root namespace
        with common.preserve_namespace(":"):
            jbfile = JB_File(taskfileinfo)
            filepath = jbfile.get_fullpath()
            ns_suggestion = reftrack.get_namespace(taskfileinfo)
            nodes = cmds.file(filepath, i=True, namespace=ns_suggestion, returnNewNodes=True)  # import
            assert nodes, 'Nothing was imported! this is unusual!'
            ns = common.get_top_namespace(nodes[0])  # get the actual namespace
            cmds.setAttr("%s.namespace" % refobj, ns, type="string")
            nscontent = cmds.namespaceInfo(ns, listOnlyDependencyNodes=True)  # get the content
            scenenode = self.get_scenenode(nscontent)
            self.get_refobjinter().connect_reftrack_scenenode(refobj, scenenode)
            dagcontent = cmds.ls(nodes, dag=True, ap=True)  # get only the dagnodes so we can group them
            if not dagcontent:
                return  # no need for a top group if there are not dagnodes to group
            # group the dagnodes in the new namespace
            grpname = reftrack.get_groupname(taskfileinfo)
            reftrack.group_content(dagcontent, ns, grpname, "jb_asset")
            return

    def fetch_option_taskfileinfos(self, element):
        """Fetch the options for possible files to load, replace etc for the given element.

        Options from which to choose a file to load or replace.

        :param element: The element for which the options should be fetched.
        :type element: :class:`jukeboxcore.djadapter.models.Asset` | :class:`jukeboxcore.djadapter.models.Shot`
        :returns: The options
        :rtype: list of :class:`TaskFileInfo`
        :raises: None
        """
        tfs = []
        for task in element.tasks.all():
            taskfiles = list(task.taskfile_set.filter(releasetype=djadapter.RELEASETYPES['release'],
                                                      typ=djadapter.FILETYPES['mayamainscene']))
            tfs.extend(taskfiles)
        tfis = [TaskFileInfo.create_from_taskfile(tf) for tf in tfs]
        return tfis

    def create_options_model(self, taskfileinfos):
        """Create a new treemodel that has the taskfileinfos as internal_data of the leaves.

        I recommend using :class:`jukeboxcore.gui.filesysitemdata.TaskFileInfoItemData` for the leaves.
        So a valid root item would be something like::

          rootdata = jukeboxcore.gui.treemodel.ListItemData(["Asset/Shot", "Task", "Descriptor", "Version", "Releasetype"])
          rootitem = jukeboxcore.gui.treemodel.TreeItem(rootdata)

        :returns: the option model with :class:`TaskFileInfo` as internal_data of the leaves.
        :rtype: :class:`jukeboxcore.gui.treemodel.TreeModel`
        :raises: None
        """
        rootdata = ListItemData(["Asset/Shot", "Task", "Descriptor", "Version", "Releasetype"])
        rootitem = TreeItem(rootdata)
        tasks = defaultdict(list)
        for tfi in taskfileinfos:
            tasks[tfi.task].append(tfi)
        for task, tfis in tasks.iteritems():
            taskdata = djitemdata.TaskItemData(task)
            taskitem = TreeItem(taskdata, rootitem)
            for tfi in tfis:
                tfidata = TaskFileInfoItemData(tfi)
                TreeItem(tfidata, taskitem)
        return TreeModel(rootitem)

    def get_option_labels(self, element):
        """Return labels for each level of the option model.

        The options returned by :meth:`RefobjInterface.fetch_options` is a treemodel
        with ``n`` levels. Each level should get a label to describe what is displays.

        Assets are organized in tasks and versions.

        :param element: The element for which the options should be fetched.
        :type element: :class:`jukeboxcore.djadapter.models.Asset` | :class:`jukeboxcore.djadapter.models.Shot`
        :returns: label strings for all levels
        :rtype: list
        :raises: None
        """
        return ["Task", "Version"]

    def get_suggestions(self, reftrack):
        """Return a list with possible children for this reftrack

        Each Reftrack may want different children. E.g. a Asset wants
        to suggest a shader for itself and all assets that are linked in
        to it in the database. Suggestions only apply for enities with status
        other than None.

        A suggestion is a tuple of typ and element. It will be used to create a newlen
        :class:`Reftrack`. The parent will be this instance, root and interface will
        of course be the same.

        This will delegate the call to the  appropriate :class:`ReftypeInterface`.
        So suggestions may vary for every typ and might depend on the
        status of the reftrack.

        :param reftrack: the reftrack which needs suggestions
        :type reftrack: :class:`Reftrack`
        :returns: list of suggestions, tuples of type and element.
        :rtype: list
        :raises: None
        """
        return []

    def get_scene_suggestions(self, current):
        """Return a list with elements for reftracks for the current scene with this type.

        For every element returned, the reftrack system will create a :class:`Reftrack` with the type
        of this interface, if it is not already in the scene.

        E.g. if you have a type that references whole scenes, you might suggest all
        linked assets for shots, and all liked assets plus the current element itself for assets.
        If you have a type like shader, that usually need a parent, you would return an empty list.
        Cameras might only make sense for shots and not for assets etc.

        Do not confuse this with :meth:`ReftypeInterface.get_suggestions`. It will gather suggestions
        for children of a :class:`Reftrack`.

        The standard implementation only returns an empty list!

        :param reftrack: the reftrack which needs suggestions
        :type reftrack: :class:`Reftrack`
        :returns: list of suggestions, tuples of type and element.
        :rtype: list
        :raises: None
        """
        l = []
        if isinstance(current, djadapter.models.Asset):
            l.append(current)
        l.extend(list(current.assets.all()))
        return l
<commit_msg>Implement get_typ_color for asset. Assets are blue.<commit_after>"""Implementation of :class:`jukeboxcore.reftrack.ReftypeInterface` for Assets."""
from collections import defaultdict

import maya.cmds as cmds
from PySide import QtGui

from jukeboxcore import djadapter
from jukeboxcore.filesys import JB_File
from jukeboxcore.reftrack import ReftypeInterface
from jukeboxcore.filesys import TaskFileInfo
from jukeboxcore.gui import djitemdata
from jukeboxcore.gui.treemodel import TreeModel, TreeItem, ListItemData
from jukeboxcore.gui.filesysitemdata import TaskFileInfoItemData
from jukeboxmaya import common
from jukeboxmaya import reftrack

class AssetReftypeInterface(ReftypeInterface):
    """Interface for handling the content of released assets in the reference workflow.
    """

    def __init__(self, refobjinter):
        """Initialize a new AssetReftypeInterface that uses the given RefobjInterface

        :param refobjinter: the interface to handle the reftrack nodes
        :type refobjinter: :class:`jukeboxmaya.refobjinter.MayaRefobjInterface`
        :raises: None
        """
        super(AssetReftypeInterface, self).__init__(refobjinter)

    def is_replaceable(self, refobj):
        """Return whether the given reference of the refobject is replaceable or
        if it should just get deleted and loaded again.

        Returns True, because Maya can replace references.

        :param refobj: the refobject to query
        :type refobj: refobj
        :returns: True, if replaceable
        :rtype: bool
        :raises: NotImplementedError
        """
        return True

    def get_scenenode(self, nodes):
        """Get the scenenode in the given nodes

        There should only be one scenenode in nodes!

        :param nodes:
        :type nodes:
        :returns: None
        :rtype: None
        :raises: AssertionError
        """
        scenenodes = cmds.ls(nodes, type='jb_sceneNode')
        assert scenenodes, "Found no scene nodes!"
        assert len(scenenodes) == 1, "Found to many scene nodes! %s" % scenenodes
        return scenenodes[0]

    def reference(self, refobj, taskfileinfo):
        """Reference the given taskfileinfo into the scene and return the created reference node

        The created reference node will be used on :meth:`RefobjInterface.set_reference` to
        set the reference on a reftrack node.
        Do not call :meth:`RefobjInterface.set_reference` yourself.

        This will also create a group node and group all dagnodes under a appropriate node.

        :param refobj: the reftrack node that will be linked to the reference
        :type refobj: str
        :param taskfileinfo: The taskfileinfo that holds the information for what to reference
        :type taskfileinfo: :class:`jukeboxcore.filesys.TaskFileInfo`
        :returns: the reference node that was created and should set on the appropriate reftrack node
        :rtype: str
        :raises: None
        """
        # work in root namespace
        with common.preserve_namespace(":"):
            jbfile = JB_File(taskfileinfo)
            filepath = jbfile.get_fullpath()
            ns_suggestion = reftrack.get_namespace(taskfileinfo)
            reffile = cmds.file(filepath, reference=True, namespace=ns_suggestion)
            node = cmds.referenceQuery(reffile, referenceNode=True)  # get reference node
            ns = cmds.referenceQuery(node, namespace=True)  # query the actual new namespace
            content = cmds.namespaceInfo(ns, listOnlyDependencyNodes=True, dagPath=True)  # get the content
            # connect reftrack with scenenode
            scenenode = self.get_scenenode(content)
            self.get_refobjinter().connect_reftrack_scenenode(refobj, scenenode)
            dagcontent = cmds.ls(content, dag=True, ap=True)  # get only the dagnodes so we can group them
            if not dagcontent:
                return node  # no need for a top group if there are not dagnodes to group
            # group the dagnodes
            grpname = reftrack.get_groupname(taskfileinfo)
            reftrack.group_content(dagcontent, ns, grpname, "jb_asset")
            return node

    def load(self, refobj, reference):
        """Load the given reference

        Load in this case means, that a reference is already in the scene
        but it is not in a loaded state.
        Loading the reference means, that the actual data will be read.

        :param refobj: the reftrack node that is linked to the reference
        :type refobj: str
        :param reference: the reference node
        :type refobj: str
        :returns: None
        :rtype: None
        :raises: None
        """
        cmds.file(loadReference=reference)

    def unload(self, refobj, reference):
        """Unload the given reference

        Unload in this case means, that a reference is stays in the scene
        but it is not in a loaded state.
        So there is a reference, but data is not read from it.

        :param refobj: the refobj that is linked to the reference
        :param reference: the reference object. E.g. in Maya a reference node
        :returns: None
        :rtype: None
        :raises: None
        """
        cmds.file(unloadReference=reference)

    def replace(self, refobj, reference, taskfileinfo):
        """Replace the given reference with the given taskfileinfo

        :param refobj: the refobj that is linked to the reference
        :param reference: the reference object. E.g. in Maya a reference node
        :param taskfileinfo: the taskfileinfo that will replace the old entity
        :type taskfileinfo: :class:`jukeboxcore.filesys.TaskFileInfo`
        :returns: None
        :rtype: None
        :raises: None
        """
        jbfile = JB_File(taskfileinfo)
        filepath = jbfile.get_fullpath()
        cmds.file(filepath, loadReference=reference)
        ns = cmds.referenceQuery(reference, namespace=True)  # query the actual new namespace
        content = cmds.namespaceInfo(ns, listOnlyDependencyNodes=True)  # get the content
        scenenode = self.get_scenenode(content) # get the scene node
        self.get_refobjinter().connect_reftrack_scenenode(refobj, scenenode)

    def delete(self, refobj):
        """Delete the content of the given refobj

        :param refobj: the refobj that represents the content that should be deleted
        :type refobj: refobj
        :returns: None
        :rtype: None
        :raises: None
        """
        common.disconnect_node(refobj)
        refobjinter = self.get_refobjinter()
        reference = refobjinter.get_reference(refobj)
        if reference:
            cmds.file(reference, removeReference=True, force=True)
        else:
            ns = cmds.getAttr("%s.namespace" % refobj)
            content = cmds.namespaceInfo(ns, listNamespace=True)
            cmds.delete(content)
            cmds.namespace(removeNamespace=ns)

    def import_reference(self, refobj, reference):
        """Import the given reference

        The reference of the refobj will be set to None automatically afterwards with
        :meth:`RefobjInterface.set_reference`

        :param refobj: the refobj that is linked to the reference
        :param reference: the reference object. E.g. in Maya a reference node
        :returns: None
        :rtype: None
        :raises: None
        """
        f = cmds.referenceQuery(reference, filename=True)
        cmds.file(f, importReference=True)

    def import_taskfile(self, refobj, taskfileinfo):
        """Import the given taskfileinfo and update the refobj

        :param refobj: the refobject
        :type refobj: refobject
        :param taskfileinfo: the taskfileinfo to reference
        :type taskfileinfo: :class:`jukeboxcore.filesys.TaskFileInfo`
        :returns: None
        :rtype: None
        :raises: None
        """
        # work in root namespace
        with common.preserve_namespace(":"):
            jbfile = JB_File(taskfileinfo)
            filepath = jbfile.get_fullpath()
            ns_suggestion = reftrack.get_namespace(taskfileinfo)
            nodes = cmds.file(filepath, i=True, namespace=ns_suggestion, returnNewNodes=True)  # import
            assert nodes, 'Nothing was imported! this is unusual!'
            ns = common.get_top_namespace(nodes[0])  # get the actual namespace
            cmds.setAttr("%s.namespace" % refobj, ns, type="string")
            nscontent = cmds.namespaceInfo(ns, listOnlyDependencyNodes=True)  # get the content
            scenenode = self.get_scenenode(nscontent)
            self.get_refobjinter().connect_reftrack_scenenode(refobj, scenenode)
            dagcontent = cmds.ls(nodes, dag=True, ap=True)  # get only the dagnodes so we can group them
            if not dagcontent:
                return  # no need for a top group if there are not dagnodes to group
            # group the dagnodes in the new namespace
            grpname = reftrack.get_groupname(taskfileinfo)
            reftrack.group_content(dagcontent, ns, grpname, "jb_asset")
            return

    def fetch_option_taskfileinfos(self, element):
        """Fetch the options for possible files to load, replace etc for the given element.

        Options from which to choose a file to load or replace.

        :param element: The element for which the options should be fetched.
        :type element: :class:`jukeboxcore.djadapter.models.Asset` | :class:`jukeboxcore.djadapter.models.Shot`
        :returns: The options
        :rtype: list of :class:`TaskFileInfo`
        :raises: None
        """
        tfs = []
        for task in element.tasks.all():
            taskfiles = list(task.taskfile_set.filter(releasetype=djadapter.RELEASETYPES['release'],
                                                      typ=djadapter.FILETYPES['mayamainscene']))
            tfs.extend(taskfiles)
        tfis = [TaskFileInfo.create_from_taskfile(tf) for tf in tfs]
        return tfis

    def create_options_model(self, taskfileinfos):
        """Create a new treemodel that has the taskfileinfos as internal_data of the leaves.

        I recommend using :class:`jukeboxcore.gui.filesysitemdata.TaskFileInfoItemData` for the leaves.
        So a valid root item would be something like::

          rootdata = jukeboxcore.gui.treemodel.ListItemData(["Asset/Shot", "Task", "Descriptor", "Version", "Releasetype"])
          rootitem = jukeboxcore.gui.treemodel.TreeItem(rootdata)

        :returns: the option model with :class:`TaskFileInfo` as internal_data of the leaves.
        :rtype: :class:`jukeboxcore.gui.treemodel.TreeModel`
        :raises: None
        """
        rootdata = ListItemData(["Asset/Shot", "Task", "Descriptor", "Version", "Releasetype"])
        rootitem = TreeItem(rootdata)
        tasks = defaultdict(list)
        for tfi in taskfileinfos:
            tasks[tfi.task].append(tfi)
        for task, tfis in tasks.iteritems():
            taskdata = djitemdata.TaskItemData(task)
            taskitem = TreeItem(taskdata, rootitem)
            for tfi in tfis:
                tfidata = TaskFileInfoItemData(tfi)
                TreeItem(tfidata, taskitem)
        return TreeModel(rootitem)

    def get_option_labels(self, element):
        """Return labels for each level of the option model.

        The options returned by :meth:`RefobjInterface.fetch_options` is a treemodel
        with ``n`` levels. Each level should get a label to describe what is displays.

        Assets are organized in tasks and versions.

        :param element: The element for which the options should be fetched.
        :type element: :class:`jukeboxcore.djadapter.models.Asset` | :class:`jukeboxcore.djadapter.models.Shot`
        :returns: label strings for all levels
        :rtype: list
        :raises: None
        """
        return ["Task", "Version"]

    def get_suggestions(self, reftrack):
        """Return a list with possible children for this reftrack

        Each Reftrack may want different children. E.g. a Asset wants
        to suggest a shader for itself and all assets that are linked in
        to it in the database. Suggestions only apply for enities with status
        other than None.

        A suggestion is a tuple of typ and element. It will be used to create a newlen
        :class:`Reftrack`. The parent will be this instance, root and interface will
        of course be the same.

        This will delegate the call to the  appropriate :class:`ReftypeInterface`.
        So suggestions may vary for every typ and might depend on the
        status of the reftrack.

        :param reftrack: the reftrack which needs suggestions
        :type reftrack: :class:`Reftrack`
        :returns: list of suggestions, tuples of type and element.
        :rtype: list
        :raises: None
        """
        return []

    def get_scene_suggestions(self, current):
        """Return a list with elements for reftracks for the current scene with this type.

        For every element returned, the reftrack system will create a :class:`Reftrack` with the type
        of this interface, if it is not already in the scene.

        E.g. if you have a type that references whole scenes, you might suggest all
        linked assets for shots, and all liked assets plus the current element itself for assets.
        If you have a type like shader, that usually need a parent, you would return an empty list.
        Cameras might only make sense for shots and not for assets etc.

        Do not confuse this with :meth:`ReftypeInterface.get_suggestions`. It will gather suggestions
        for children of a :class:`Reftrack`.

        The standard implementation only returns an empty list!

        :param reftrack: the reftrack which needs suggestions
        :type reftrack: :class:`Reftrack`
        :returns: list of suggestions, tuples of type and element.
        :rtype: list
        :raises: None
        """
        l = []
        if isinstance(current, djadapter.models.Asset):
            l.append(current)
        l.extend(list(current.assets.all()))
        return l

    def get_typ_color(self, ):
        """Return a color that should be used to identify the type in an UI

        :returns: a color for this type
        :rtype: :class:`QtGui.QColor`
        :raises: NotImplementedError
        """
        return QtGui.QColor.fromHsv(210, 115, 230)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

import logging
import functools
from datetime import datetime

from dateutil import parser

from framework.mongo import database
from framework.postcommit_tasks.handlers import run_postcommit
from framework.sessions import session
from framework.celery_tasks import app
from website import settings

from flask import request

logger = logging.getLogger(__name__)

collection = None
if database._get_current_object() is not None:
    collection = database['pagecounters']
elif database._get_current_object() is None and settings.DEBUG_MODE:
    logger.warn('Cannot connect to database. Analytics will be unavailable')
else:
    raise RuntimeError('Cannot connect to database')

@run_postcommit(once_per_request=False, celery=True)
@app.task(max_retries=5, default_retry_delay=60)
def increment_user_activity_counters(user_id, action, date_string, db=None):
    db = db or database  # default to local proxy
    if not db and settings.DEBUG_MODE:
        logger.warn('Cannot connect to database. Analytics will be unavailable')
        return
    collection = database['useractivitycounters']
    date = parser.parse(date_string).strftime('%Y/%m/%d')
    query = {
        '$inc': {
            'total': 1,
            'date.{0}.total'.format(date): 1,
            'action.{0}.total'.format(action): 1,
            'action.{0}.date.{1}'.format(action, date): 1,
        }
    }
    collection.update(
        {'_id': user_id},
        query,
        upsert=True,
        manipulate=False,
    )
    return True

def get_total_activity_count(user_id, db=None):
    db = db or database
    if not db and settings.DEBUG_MODE:
        logger.warn('Cannot connect to database. Analytics will be unavailable')
        return
    collection = database['useractivitycounters']
    result = collection.find_one(
        {'_id': user_id}, {'total': 1}
    )
    if result and 'total' in result:
        return result['total']
    return 0

def clean_page(page):
    return page.replace(
        '.', '_'
    ).replace(
        '$', '_'
    )

def build_page(rex, kwargs):
    """Build page key from format pattern and request data.

    :param str: Format string (e.g. `'{node}:{file}'`)
    :param dict kwargs: Data used to render format string
    """
    target_node = kwargs.get('node') or kwargs.get('project')
    target_id = target_node._id
    data = {
        'target_id': target_id,
    }
    data.update(kwargs)
    data.update(request.args.to_dict())
    try:
        return rex.format(**data)
    except KeyError:
        return None

def update_counter(page, node_info=None, db=None):
    """Update counters for page.

    :param str page: Colon-delimited page key in analytics collection
    :param db: MongoDB database or `None`
    """
    db = db or database
    if not db and settings.DEBUG_MODE:
        logger.warn('Cannot connect to database. Analytics will be unavailable')
        return
    collection = db['pagecounters']

    date = datetime.utcnow()
    date = date.strftime('%Y/%m/%d')

    page = clean_page(page)

    d = {'$inc': {}}

    visited_by_date = session.data.get('visited_by_date')
    if not visited_by_date:
        visited_by_date = {'date': date, 'pages': []}

    if date == visited_by_date['date']:
        if page not in visited_by_date['pages']:
            d['$inc']['date.%s.unique' % date] = 1
            visited_by_date['pages'].append(page)
            session.data['visited_by_date'] = visited_by_date
    else:
        visited_by_date['date'] = date
        visited_by_date['pages'] = []
        d['$inc']['date.%s.unique' % date] = 1
        visited_by_date['pages'].append(page)
        session.data['visited_by_date'] = visited_by_date

    d['$inc']['date.%s.total' % date] = 1

    visited = session.data.get('visited')  # '/project/x/, project/y/'
    if not visited:
        visited = []
    if page not in visited:
        d['$inc']['unique'] = 1
        visited.append(page)
        session.data['visited'] = visited
    d['$inc']['total'] = 1

    # If a download counter is being updated, only perform the update
    # if the user who is downloading isn't a contributor to the project
    page_type = page.split(':')[0]
    if page_type == 'download' and node_info:
        contributors = node_info['contributors']
        current_user = session.data.get('auth_user_id')
        if current_user and current_user in contributors:
            d['$inc']['unique'] = 0
            d['$inc']['total'] = 0

    collection.update({'_id': page}, d, True, False)

def update_counters(rex, node_info=None, db=None):
    """Create a decorator that updates analytics in `pagecounters` when the
    decorated function is called. Note: call inner function before incrementing
    counters so that counters are not changed if inner function fails.

    :param rex: Pattern for building page key from keyword arguments of
        decorated function
    """
    def wrapper(func):
        @functools.wraps(func)
        def wrapped(*args, **kwargs):
            ret = func(*args, **kwargs)
            page = build_page(rex, kwargs)
            update_counter(page, node_info, db)
            return ret
        return wrapped
    return wrapper

def get_basic_counters(page, db=None):
    db = db or database
    if not db and settings.DEBUG_MODE:
        logger.warn('Cannot connect to database. Analytics will be unavailable')
        return
    collection = db['pagecounters']
    unique = 0
    total = 0
    collection = database['pagecounters']
    result = collection.find_one(
        {'_id': clean_page(page)},
        {'total': 1, 'unique': 1}
    )
    if result:
        if 'unique' in result:
            unique = result['unique']
        if 'total' in result:
            total = result['total']
        return unique, total
    else:
        return None, None
<commit_msg>Disable import that is killing our deployment.<commit_after>#!/usr/bin/env python
# encoding: utf-8

import logging
import functools
from datetime import datetime

from dateutil import parser

from framework.mongo import database
from framework.postcommit_tasks.handlers import run_postcommit
from framework.sessions import session
from framework.celery_tasks import app
from website import settings

from flask import request

logger = logging.getLogger(__name__)

# TODO Find out why this is here and fix it.
collection = None
# if database._get_current_object() is not None:
#     collection = database['pagecounters']
# elif database._get_current_object() is None and settings.DEBUG_MODE:
#     logger.warn('Cannot connect to database. Analytics will be unavailable')
# else:
#     raise RuntimeError('Cannot connect to database')

@run_postcommit(once_per_request=False, celery=True)
@app.task(max_retries=5, default_retry_delay=60)
def increment_user_activity_counters(user_id, action, date_string, db=None):
    db = db or database  # default to local proxy
    if not db and settings.DEBUG_MODE:
        logger.warn('Cannot connect to database. Analytics will be unavailable')
        return
    collection = database['useractivitycounters']
    date = parser.parse(date_string).strftime('%Y/%m/%d')
    query = {
        '$inc': {
            'total': 1,
            'date.{0}.total'.format(date): 1,
            'action.{0}.total'.format(action): 1,
            'action.{0}.date.{1}'.format(action, date): 1,
        }
    }
    collection.update(
        {'_id': user_id},
        query,
        upsert=True,
        manipulate=False,
    )
    return True

def get_total_activity_count(user_id, db=None):
    db = db or database
    if not db and settings.DEBUG_MODE:
        logger.warn('Cannot connect to database. Analytics will be unavailable')
        return
    collection = database['useractivitycounters']
    result = collection.find_one(
        {'_id': user_id}, {'total': 1}
    )
    if result and 'total' in result:
        return result['total']
    return 0

def clean_page(page):
    return page.replace(
        '.', '_'
    ).replace(
        '$', '_'
    )

def build_page(rex, kwargs):
    """Build page key from format pattern and request data.

    :param str: Format string (e.g. `'{node}:{file}'`)
    :param dict kwargs: Data used to render format string
    """
    target_node = kwargs.get('node') or kwargs.get('project')
    target_id = target_node._id
    data = {
        'target_id': target_id,
    }
    data.update(kwargs)
    data.update(request.args.to_dict())
    try:
        return rex.format(**data)
    except KeyError:
        return None

def update_counter(page, node_info=None, db=None):
    """Update counters for page.

    :param str page: Colon-delimited page key in analytics collection
    :param db: MongoDB database or `None`
    """
    db = db or database
    if not db and settings.DEBUG_MODE:
        logger.warn('Cannot connect to database. Analytics will be unavailable')
        return
    collection = db['pagecounters']

    date = datetime.utcnow()
    date = date.strftime('%Y/%m/%d')

    page = clean_page(page)

    d = {'$inc': {}}

    visited_by_date = session.data.get('visited_by_date')
    if not visited_by_date:
        visited_by_date = {'date': date, 'pages': []}

    if date == visited_by_date['date']:
        if page not in visited_by_date['pages']:
            d['$inc']['date.%s.unique' % date] = 1
            visited_by_date['pages'].append(page)
            session.data['visited_by_date'] = visited_by_date
    else:
        visited_by_date['date'] = date
        visited_by_date['pages'] = []
        d['$inc']['date.%s.unique' % date] = 1
        visited_by_date['pages'].append(page)
        session.data['visited_by_date'] = visited_by_date

    d['$inc']['date.%s.total' % date] = 1

    visited = session.data.get('visited')  # '/project/x/, project/y/'
    if not visited:
        visited = []
    if page not in visited:
        d['$inc']['unique'] = 1
        visited.append(page)
        session.data['visited'] = visited
    d['$inc']['total'] = 1

    # If a download counter is being updated, only perform the update
    # if the user who is downloading isn't a contributor to the project
    page_type = page.split(':')[0]
    if page_type == 'download' and node_info:
        contributors = node_info['contributors']
        current_user = session.data.get('auth_user_id')
        if current_user and current_user in contributors:
            d['$inc']['unique'] = 0
            d['$inc']['total'] = 0

    collection.update({'_id': page}, d, True, False)

def update_counters(rex, node_info=None, db=None):
    """Create a decorator that updates analytics in `pagecounters` when the
    decorated function is called. Note: call inner function before incrementing
    counters so that counters are not changed if inner function fails.

    :param rex: Pattern for building page key from keyword arguments of
        decorated function
    """
    def wrapper(func):
        @functools.wraps(func)
        def wrapped(*args, **kwargs):
            ret = func(*args, **kwargs)
            page = build_page(rex, kwargs)
            update_counter(page, node_info, db)
            return ret
        return wrapped
    return wrapper

def get_basic_counters(page, db=None):
    db = db or database
    if not db and settings.DEBUG_MODE:
        logger.warn('Cannot connect to database. Analytics will be unavailable')
        return
    collection = db['pagecounters']
    unique = 0
    total = 0
    collection = database['pagecounters']
    result = collection.find_one(
        {'_id': clean_page(page)},
        {'total': 1, 'unique': 1}
    )
    if result:
        if 'unique' in result:
            unique = result['unique']
        if 'total' in result:
            total = result['total']
        return unique, total
    else:
        return None, None
<|endoftext|>


<commit_before>from __future__ import division
import numpy as np
import cv2
import warnings

from menpo.base import Vectorizable
from menpo.transform import Transform, Translation, Rotation

def align_2d_3d(points_3d, points_image, image_shape, focal_length=None,
                distortion_coeffs=None):
    height, width = image_shape
    # Create camera matrix
    focal_length = (max(height, width)
                    if focal_length is None else focal_length)
    c_x = width / 2.0
    c_y = height / 2.0
    camera_matrix = np.array([[focal_length, 0, c_x],
                              [0, focal_length, c_y],
                              [0, 0, 1.0]])

    # If distortion coefficients are None, set them to zero
    if distortion_coeffs is None:
        distortion_coeffs = np.zeros(4)
    # Estimate the camera pose given the 3D sparse pointcloud on the
    # mesh, its 2D projection on the image, the camera matrix and the
    # distortion coefficients
    lm2d = points_image.points[:, ::-1].copy()
    converged, r_vec, t_vec = cv2.solvePnP(points_3d.points,
                                           lm2d,
                                           camera_matrix, distortion_coeffs)

    if not converged:
        warnings.warn('cv2.SolvePnP did not converge to a solution')

    # Create rotation and translation transform objects from the vectors
    # acquired at the previous step
    rotation_matrix = cv2.Rodrigues(r_vec)[0]
    r = Rotation(rotation_matrix)
    t = Translation(t_vec.ravel())

    return r, t, focal_length

class OrthographicProjection(Transform, Vectorizable):

    def __init__(self, focal_length, image_shape):
        self.focal_length = focal_length
        self.height = image_shape[0]
        self.width = image_shape[1]

    @property
    def n_dims(self):
        return 3

    @property
    def n_parameters(self):
        return 1

    def as_vector(self):
        return np.array([self.focal_length])

    def _from_vector_inplace(self, vector):
        self.focal_length = vector[0]

    def _apply(self, x, **kwargs):
        f = self.focal_length
        c_x = self.width / 2
        c_y = self.height / 2

        output = np.empty_like(x)
        output[:, 0] = (f * x[:, 1]) + c_y
        output[:, 1] = (f * x[:, 0]) + c_x
        output[:, 2] = x[:, 2]

        return output

class PerspectiveProjection(OrthographicProjection):
    def _apply(self, x, **kwargs):
        f = self.focal_length
        c_x = self.width / 2
        c_y = self.height / 2

        output = np.empty_like(x)
        output[:, 0] = (f * x[:, 1]) / x[:, 2] + c_y
        output[:, 1] = (f * x[:, 0]) / x[:, 2] + c_x
        output[:, 2] = x[:, 2]

        return output

class OrthographicCamera(Vectorizable):
    def __init__(self, rotation, translation, projection):
        self.rotation_transform = rotation
        self.translation_transform = translation
        self.projection_transform = projection

    @property
    def focal_length(self):
        return self.projection_transform.focal_length

    @classmethod
    def init_from_image_shape_and_vector(cls, image_shape, vector):
        r = Rotation.init_identity(n_dims=3)
        t = Translation.init_identity(n_dims=3)
        p = OrthographicProjection(focal_length=1, image_shape=image_shape)
        return cls(r, t, p).from_vector(vector)

    @classmethod
    def init_from_2d_projected_shape(cls, points_3d, points_image,
                                     image_shape, focal_length=None,
                                     distortion_coeffs=None):
        r, t, focal_length = align_2d_3d(
            points_3d, points_image, image_shape, focal_length=focal_length,
            distortion_coeffs=distortion_coeffs)
        return OrthographicCamera(r, t, OrthographicProjection(focal_length,
                                                               image_shape))

    def apply(self, instance, **kwargs):
        return self.camera_transform.apply(instance)

    @property
    def n_parameters(self):
        return (self.projection_transform.n_parameters +
                self.rotation_transform.n_parameters +
                self.translation_transform.n_parameters)

    def as_vector(self):
        # focal_length, q_w, q_x, q_y, q_z, t_x, t_y, t_z
        params = np.zeros(self.n_parameters)

        # focal length
        params[:1] = self.projection_transform.as_vector()

        # 4 parameters: q_w, q_x, q_y, q_z
        params[1:5] = self.rotation_transform.as_vector()

        # 3 parameters: t_x, t_y, t_z
        params[5:] = self.translation_transform.as_vector()
        return params

    def _from_vector_inplace(self, vector):
        self.projection_transform._from_vector_inplace(vector[:1])
        self.rotation_transform._from_vector_inplace(vector[1:5])
        self.translation_transform._from_vector_inplace(vector[5:])

    @property
    def view_transform(self):
        return self.rotation_transform.compose_before(self.translation_transform)

    @property
    def camera_transform(self):
        return self.view_transform.compose_before(self.projection_transform)

class PerspectiveCamera(OrthographicCamera):
    @classmethod
    def init_from_image_shape_and_vector(cls, image_shape, vector):
        r = Rotation.init_identity(n_dims=3)
        t = Translation.init_identity(n_dims=3)
        p = PerspectiveProjection(focal_length=1, image_shape=image_shape)
        return cls(r, t, p).from_vector(vector)

    @classmethod
    def init_from_2d_projected_shape(cls, points_3d, points_image,
                                     image_shape, focal_length=None,
                                     distortion_coeffs=None):
        r, t, focal_length = align_2d_3d(
            points_3d, points_image, image_shape, focal_length=focal_length,
            distortion_coeffs=distortion_coeffs)
        return PerspectiveCamera(r, t, PerspectiveProjection(focal_length,
                                                             image_shape))
<commit_msg>make CV2 optional<commit_after>from __future__ import division
import numpy as np
import warnings

from menpo.base import Vectorizable, MenpoMissingDependencyError
from menpo.transform import Transform, Translation, Rotation

def align_2d_3d(points_3d, points_image, image_shape, focal_length=None,
                distortion_coeffs=None):
    try:
        import cv2
    except ImportError:
        raise MenpoMissingDependencyError('opencv3')
    height, width = image_shape
    # Create camera matrix
    focal_length = (max(height, width)
                    if focal_length is None else focal_length)
    c_x = width / 2.0
    c_y = height / 2.0
    camera_matrix = np.array([[focal_length, 0, c_x],
                              [0, focal_length, c_y],
                              [0, 0, 1.0]])

    # If distortion coefficients are None, set them to zero
    if distortion_coeffs is None:
        distortion_coeffs = np.zeros(4)
    # Estimate the camera pose given the 3D sparse pointcloud on the
    # mesh, its 2D projection on the image, the camera matrix and the
    # distortion coefficients
    lm2d = points_image.points[:, ::-1].copy()
    converged, r_vec, t_vec = cv2.solvePnP(points_3d.points,
                                           lm2d,
                                           camera_matrix, distortion_coeffs)

    if not converged:
        warnings.warn('cv2.SolvePnP did not converge to a solution')

    # Create rotation and translation transform objects from the vectors
    # acquired at the previous step
    rotation_matrix = cv2.Rodrigues(r_vec)[0]
    r = Rotation(rotation_matrix)
    t = Translation(t_vec.ravel())

    return r, t, focal_length

class OrthographicProjection(Transform, Vectorizable):

    def __init__(self, focal_length, image_shape):
        self.focal_length = focal_length
        self.height = image_shape[0]
        self.width = image_shape[1]

    @property
    def n_dims(self):
        return 3

    @property
    def n_parameters(self):
        return 1

    def as_vector(self):
        return np.array([self.focal_length])

    def _from_vector_inplace(self, vector):
        self.focal_length = vector[0]

    def _apply(self, x, **kwargs):
        f = self.focal_length
        c_x = self.width / 2
        c_y = self.height / 2

        output = np.empty_like(x)
        output[:, 0] = (f * x[:, 1]) + c_y
        output[:, 1] = (f * x[:, 0]) + c_x
        output[:, 2] = x[:, 2]

        return output

class PerspectiveProjection(OrthographicProjection):
    def _apply(self, x, **kwargs):
        f = self.focal_length
        c_x = self.width / 2
        c_y = self.height / 2

        output = np.empty_like(x)
        output[:, 0] = (f * x[:, 1]) / x[:, 2] + c_y
        output[:, 1] = (f * x[:, 0]) / x[:, 2] + c_x
        output[:, 2] = x[:, 2]

        return output

class OrthographicCamera(Vectorizable):
    def __init__(self, rotation, translation, projection):
        self.rotation_transform = rotation
        self.translation_transform = translation
        self.projection_transform = projection

    @property
    def focal_length(self):
        return self.projection_transform.focal_length

    @classmethod
    def init_from_image_shape_and_vector(cls, image_shape, vector):
        r = Rotation.init_identity(n_dims=3)
        t = Translation.init_identity(n_dims=3)
        p = OrthographicProjection(focal_length=1, image_shape=image_shape)
        return cls(r, t, p).from_vector(vector)

    @classmethod
    def init_from_2d_projected_shape(cls, points_3d, points_image,
                                     image_shape, focal_length=None,
                                     distortion_coeffs=None):
        r, t, focal_length = align_2d_3d(
            points_3d, points_image, image_shape, focal_length=focal_length,
            distortion_coeffs=distortion_coeffs)
        return OrthographicCamera(r, t, OrthographicProjection(focal_length,
                                                               image_shape))

    def apply(self, instance, **kwargs):
        return self.camera_transform.apply(instance)

    @property
    def n_parameters(self):
        return (self.projection_transform.n_parameters +
                self.rotation_transform.n_parameters +
                self.translation_transform.n_parameters)

    def as_vector(self):
        # focal_length, q_w, q_x, q_y, q_z, t_x, t_y, t_z
        params = np.zeros(self.n_parameters)

        # focal length
        params[:1] = self.projection_transform.as_vector()

        # 4 parameters: q_w, q_x, q_y, q_z
        params[1:5] = self.rotation_transform.as_vector()

        # 3 parameters: t_x, t_y, t_z
        params[5:] = self.translation_transform.as_vector()
        return params

    def _from_vector_inplace(self, vector):
        self.projection_transform._from_vector_inplace(vector[:1])
        self.rotation_transform._from_vector_inplace(vector[1:5])
        self.translation_transform._from_vector_inplace(vector[5:])

    @property
    def view_transform(self):
        return self.rotation_transform.compose_before(self.translation_transform)

    @property
    def camera_transform(self):
        return self.view_transform.compose_before(self.projection_transform)

class PerspectiveCamera(OrthographicCamera):
    @classmethod
    def init_from_image_shape_and_vector(cls, image_shape, vector):
        r = Rotation.init_identity(n_dims=3)
        t = Translation.init_identity(n_dims=3)
        p = PerspectiveProjection(focal_length=1, image_shape=image_shape)
        return cls(r, t, p).from_vector(vector)

    @classmethod
    def init_from_2d_projected_shape(cls, points_3d, points_image,
                                     image_shape, focal_length=None,
                                     distortion_coeffs=None):
        r, t, focal_length = align_2d_3d(
            points_3d, points_image, image_shape, focal_length=focal_length,
            distortion_coeffs=distortion_coeffs)
        return PerspectiveCamera(r, t, PerspectiveProjection(focal_length,
                                                             image_shape))
<|endoftext|>


<commit_before><commit_msg>Example Tunnelling<commit_after><|endoftext|>


<commit_before>"""
Brume
"""

VERSION = '1.2.2'
<commit_msg>Update __init__.py<commit_after>"""
Brume
"""

VERSION = '1.3.0'
<|endoftext|>