<commit_before>import pyinotify
import threading
import time
from pydispatch import dispatcher

from os.path                 import normpath
from media.monitor.events    import PathChannel
from media.monitor.log       import Loggable
from media.monitor.listeners import StoreWatchListener, OrganizeListener
from media.monitor.handler   import ProblemFileHandler
from media.monitor.organizer import Organizer
import media.monitor.pure as mmp

class ManagerTimeout(threading.Thread,Loggable):
    def __init__(self, manager):
        threading.Thread.__init__(self)
        self.manager = manager
    def run(self):
        while True:
            time.sleep(3)
            self.manager.flush_organize()
            #self.logger.info("Force flushed organize...")

class Manager(Loggable):
    """
    An abstraction over media monitors core pyinotify functions. These
    include adding watched,store, organize directories, etc. Basically
    composes over WatchManager from pyinotify
    """
    global_inst = None
    all_signals = set(['add_watch', 'remove_watch'])
    def __init__(self):
        self.wm = pyinotify.WatchManager()
        # These two instance variables are assumed to be constant
        self.watch_channel    = 'watch'
        self.organize_channel = 'organize'
        self.watch_listener   = StoreWatchListener(signal = self.watch_channel)
        # TODO : change this to  a weak ref
        # TODO : get rid of this hack once cc-4235 is fixed
        self.__timeout_thread = ManagerTimeout(self)
        self.__timeout_thread.daemon = True
        self.__timeout_thread.start()
        self.organize = {
            'organize_path'      : None,
            'imported_path'      : None,
            'recorded_path'      : None,
            'problem_files_path' : None,
            'organizer'          : None,
            'problem_handler'    : None,
            'organize_listener'  : OrganizeListener(signal=
                self.organize_channel),
        }
        def dummy(sender, event): self.watch_move( event.path, sender=sender )
        dispatcher.connect(dummy, signal='watch_move', sender=dispatcher.Any,
                weak=False)
        def subwatch_add(sender, directory):
            self.__add_watch(directory, self.watch_listener)
        dispatcher.connect(subwatch_add, signal='add_subwatch',
                sender=dispatcher.Any, weak=False)
        # A private mapping path => watch_descriptor
        # we use the same dictionary for organize, watch, store wd events.
        # this is a little hacky because we are unable to have multiple wd's
        # on the same path.
        self.__wd_path = {}
        # The following set isn't really necessary anymore. Should be
        # removed...
        self.watched_directories = set([])
        Manager.global_inst = self

    # This is the only event that we are unable to process "normally". I.e.
    # through dedicated handler objects. Because we must have access to a
    # manager instance. Hence we must slightly break encapsulation.
    def watch_move(self, watch_dir, sender=None):
        """
        handle 'watch move' events directly sent from listener
        """
        self.logger.info("Watch dir '%s' has been renamed (hence removed)" %
                watch_dir)
        self.remove_watch_directory(normpath(watch_dir))

    def watch_signal(self): return self.watch_listener.signal

    def __remove_watch(self,path):
        # only delete if dir is actually being watched
        if path in self.__wd_path:
            wd = self.__wd_path[path]
            self.wm.rm_watch(wd, rec=True)
            del(self.__wd_path[path])

    def __add_watch(self,path,listener):
        self.logger.info("Adding listener '%s' to '%s'" %
                         ( listener.__class__.__name__, path) )
        wd = self.wm.add_watch(path, pyinotify.ALL_EVENTS, rec=True,
                auto_add=True, proc_fun=listener)
        self.__wd_path[path] = wd.values()[0]

    def __create_organizer(self, target_path, recorded_path):
        """
        creates an organizer at new destination path or modifies the old one
        """
        # TODO : find a proper fix for the following hack
        # We avoid creating new instances of organize because of the way
        # it interacts with pydispatch. We must be careful to never have
        # more than one instance of OrganizeListener but this is not so
        # easy. (The singleton hack in Organizer) doesn't work. This is
        # the only thing that seems to work.
        if self.organize['organizer']:
            o               = self.organize['organizer']
            o.channel       = self.organize_channel
            o.target_path   = target_path
            o.recorded_path = recorded_path
        else:
            self.organize['organizer'] = Organizer(channel=
                    self.organize_channel, target_path=target_path,
                    recorded_path=recorded_path)

    def get_problem_files_path(self):
        """
        returns the path where problem files should go
        """
        return self.organize['problem_files_path']

    def set_problem_files_path(self, new_path):
        """
        Set the path where problem files should go
        """
        self.organize['problem_files_path'] = new_path
        self.organize['problem_handler'] = \
            ProblemFileHandler( PathChannel(signal='badfile',path=new_path) )

    def get_recorded_path(self):
        """
        returns the path of the recorded directory
        """
        return self.organize['recorded_path']

    def set_recorded_path(self, new_path):
        self.__remove_watch(self.organize['recorded_path'])
        self.organize['recorded_path'] = new_path
        self.__create_organizer( self.organize['imported_path'], new_path)
        self.__add_watch(new_path, self.watch_listener)

    def get_organize_path(self):
        """
        returns the current path that is being watched for organization
        """
        return self.organize['organize_path']

    def set_organize_path(self, new_path):
        """
        sets the organize path to be new_path. Under the current scheme there
        is only one organize path but there is no reason why more cannot be
        supported
        """
        # if we are already organizing a particular directory we remove the
        # watch from it first before organizing another directory
        self.__remove_watch(self.organize['organize_path'])
        self.organize['organize_path'] = new_path
        # the OrganizeListener instance will walk path and dispatch an organize
        # event for every file in that directory
        self.organize['organize_listener'].flush_events(new_path)
        self.__add_watch(new_path, self.organize['organize_listener'])

    def flush_organize(self):
        path = self.organize['organize_path']
        self.organize['organize_listener'].flush_events(path)

    def get_imported_path(self):
        return self.organize['imported_path']

    def set_imported_path(self,new_path):
        """
        set the directory where organized files go to.
        """
        self.__remove_watch(self.organize['imported_path'])
        self.organize['imported_path'] = new_path
        self.__create_organizer( new_path, self.organize['recorded_path'])
        self.__add_watch(new_path, self.watch_listener)

    def change_storage_root(self, store):
        """
        hooks up all the directories for you. Problem, recorded, imported,
        organize.
        """
        store_paths = mmp.expand_storage(store)
        self.set_problem_files_path(store_paths['problem_files'])
        self.set_imported_path(store_paths['imported'])
        self.set_recorded_path(store_paths['recorded'])
        self.set_organize_path(store_paths['organize'])
        mmp.create_dir(store)
        for p in store_paths.values():
            mmp.create_dir(p)

    def has_watch(self, path):
        """
        returns true if the path is being watched or not. Any kind of watch:
        organize, store, watched.
        """
        return path in self.__wd_path

    def add_watch_directory(self, new_dir):
        """
        adds a directory to be "watched". "watched" directories are
        those that are being monitored by media monitor for airtime in
        this context and not directories pyinotify calls watched
        """
        if self.has_watch(new_dir):
            self.logger.info("Cannot add '%s' to watched directories. It's \
                    already being watched" % new_dir)
        else:
            self.logger.info("Adding watched directory: '%s'" % new_dir)
            self.__add_watch(new_dir, self.watch_listener)

    def remove_watch_directory(self, watch_dir):
        """
        removes a directory from being "watched". Undoes add_watch_directory
        """
        if self.has_watch(watch_dir):
            self.logger.info("Removing watched directory: '%s'", watch_dir)
            self.__remove_watch(watch_dir)
        else:
            self.logger.info("'%s' is not being watched, hence cannot be \
                    removed" % watch_dir)

    def loop(self):
        """
        block until we receive pyinotify events
        """
        notifier = pyinotify.Notifier(self.wm)
        notifier.coalesce_events()
        notifier.loop()
        # Experiments with running notifier in different modes
        # There are 3 options: normal, async, threaded.
        #import asyncore
        #pyinotify.AsyncNotifier(self.wm).loop()
        #asyncore.loop()
<commit_msg>mm2: checking for null value<commit_after>import pyinotify
import threading
import time
from pydispatch import dispatcher

from os.path                 import normpath
from media.monitor.events    import PathChannel
from media.monitor.log       import Loggable
from media.monitor.listeners import StoreWatchListener, OrganizeListener
from media.monitor.handler   import ProblemFileHandler
from media.monitor.organizer import Organizer
import media.monitor.pure as mmp

class ManagerTimeout(threading.Thread,Loggable):
    def __init__(self, manager):
        threading.Thread.__init__(self)
        self.manager = manager
    def run(self):
        while True:
            time.sleep(3)
            self.manager.flush_organize()
            #self.logger.info("Force flushed organize...")

class Manager(Loggable):
    """
    An abstraction over media monitors core pyinotify functions. These
    include adding watched,store, organize directories, etc. Basically
    composes over WatchManager from pyinotify
    """
    global_inst = None
    all_signals = set(['add_watch', 'remove_watch'])
    def __init__(self):
        self.wm = pyinotify.WatchManager()
        # These two instance variables are assumed to be constant
        self.watch_channel    = 'watch'
        self.organize_channel = 'organize'
        self.watch_listener   = StoreWatchListener(signal = self.watch_channel)
        # TODO : change this to  a weak ref
        # TODO : get rid of this hack once cc-4235 is fixed
        self.__timeout_thread = ManagerTimeout(self)
        self.__timeout_thread.daemon = True
        self.__timeout_thread.start()
        self.organize = {
            'organize_path'      : None,
            'imported_path'      : None,
            'recorded_path'      : None,
            'problem_files_path' : None,
            'organizer'          : None,
            'problem_handler'    : None,
            'organize_listener'  : OrganizeListener(signal=
                self.organize_channel),
        }
        def dummy(sender, event): self.watch_move( event.path, sender=sender )
        dispatcher.connect(dummy, signal='watch_move', sender=dispatcher.Any,
                weak=False)
        def subwatch_add(sender, directory):
            self.__add_watch(directory, self.watch_listener)
        dispatcher.connect(subwatch_add, signal='add_subwatch',
                sender=dispatcher.Any, weak=False)
        # A private mapping path => watch_descriptor
        # we use the same dictionary for organize, watch, store wd events.
        # this is a little hacky because we are unable to have multiple wd's
        # on the same path.
        self.__wd_path = {}
        # The following set isn't really necessary anymore. Should be
        # removed...
        self.watched_directories = set([])
        Manager.global_inst = self

    # This is the only event that we are unable to process "normally". I.e.
    # through dedicated handler objects. Because we must have access to a
    # manager instance. Hence we must slightly break encapsulation.
    def watch_move(self, watch_dir, sender=None):
        """
        handle 'watch move' events directly sent from listener
        """
        self.logger.info("Watch dir '%s' has been renamed (hence removed)" %
                watch_dir)
        self.remove_watch_directory(normpath(watch_dir))

    def watch_signal(self): return self.watch_listener.signal

    def __remove_watch(self,path):
        # only delete if dir is actually being watched
        if path in self.__wd_path:
            wd = self.__wd_path[path]
            self.wm.rm_watch(wd, rec=True)
            del(self.__wd_path[path])

    def __add_watch(self,path,listener):
        self.logger.info("Adding listener '%s' to '%s'" %
                         ( listener.__class__.__name__, path) )
        wd = self.wm.add_watch(path, pyinotify.ALL_EVENTS, rec=True,
                auto_add=True, proc_fun=listener)
        if wd: self.__wd_path[path] = wd.values()[0]

    def __create_organizer(self, target_path, recorded_path):
        """
        creates an organizer at new destination path or modifies the old one
        """
        # TODO : find a proper fix for the following hack
        # We avoid creating new instances of organize because of the way
        # it interacts with pydispatch. We must be careful to never have
        # more than one instance of OrganizeListener but this is not so
        # easy. (The singleton hack in Organizer) doesn't work. This is
        # the only thing that seems to work.
        if self.organize['organizer']:
            o               = self.organize['organizer']
            o.channel       = self.organize_channel
            o.target_path   = target_path
            o.recorded_path = recorded_path
        else:
            self.organize['organizer'] = Organizer(channel=
                    self.organize_channel, target_path=target_path,
                    recorded_path=recorded_path)

    def get_problem_files_path(self):
        """
        returns the path where problem files should go
        """
        return self.organize['problem_files_path']

    def set_problem_files_path(self, new_path):
        """
        Set the path where problem files should go
        """
        self.organize['problem_files_path'] = new_path
        self.organize['problem_handler'] = \
            ProblemFileHandler( PathChannel(signal='badfile',path=new_path) )

    def get_recorded_path(self):
        """
        returns the path of the recorded directory
        """
        return self.organize['recorded_path']

    def set_recorded_path(self, new_path):
        self.__remove_watch(self.organize['recorded_path'])
        self.organize['recorded_path'] = new_path
        self.__create_organizer( self.organize['imported_path'], new_path)
        self.__add_watch(new_path, self.watch_listener)

    def get_organize_path(self):
        """
        returns the current path that is being watched for organization
        """
        return self.organize['organize_path']

    def set_organize_path(self, new_path):
        """
        sets the organize path to be new_path. Under the current scheme there
        is only one organize path but there is no reason why more cannot be
        supported
        """
        # if we are already organizing a particular directory we remove the
        # watch from it first before organizing another directory
        self.__remove_watch(self.organize['organize_path'])
        self.organize['organize_path'] = new_path
        # the OrganizeListener instance will walk path and dispatch an organize
        # event for every file in that directory
        self.organize['organize_listener'].flush_events(new_path)
        self.__add_watch(new_path, self.organize['organize_listener'])

    def flush_organize(self):
        path = self.organize['organize_path']
        self.organize['organize_listener'].flush_events(path)

    def get_imported_path(self):
        return self.organize['imported_path']

    def set_imported_path(self,new_path):
        """
        set the directory where organized files go to.
        """
        self.__remove_watch(self.organize['imported_path'])
        self.organize['imported_path'] = new_path
        self.__create_organizer( new_path, self.organize['recorded_path'])
        self.__add_watch(new_path, self.watch_listener)

    def change_storage_root(self, store):
        """
        hooks up all the directories for you. Problem, recorded, imported,
        organize.
        """
        store_paths = mmp.expand_storage(store)
        self.set_problem_files_path(store_paths['problem_files'])
        self.set_imported_path(store_paths['imported'])
        self.set_recorded_path(store_paths['recorded'])
        self.set_organize_path(store_paths['organize'])
        mmp.create_dir(store)
        for p in store_paths.values():
            mmp.create_dir(p)

    def has_watch(self, path):
        """
        returns true if the path is being watched or not. Any kind of watch:
        organize, store, watched.
        """
        return path in self.__wd_path

    def add_watch_directory(self, new_dir):
        """
        adds a directory to be "watched". "watched" directories are
        those that are being monitored by media monitor for airtime in
        this context and not directories pyinotify calls watched
        """
        if self.has_watch(new_dir):
            self.logger.info("Cannot add '%s' to watched directories. It's \
                    already being watched" % new_dir)
        else:
            self.logger.info("Adding watched directory: '%s'" % new_dir)
            self.__add_watch(new_dir, self.watch_listener)

    def remove_watch_directory(self, watch_dir):
        """
        removes a directory from being "watched". Undoes add_watch_directory
        """
        if self.has_watch(watch_dir):
            self.logger.info("Removing watched directory: '%s'", watch_dir)
            self.__remove_watch(watch_dir)
        else:
            self.logger.info("'%s' is not being watched, hence cannot be \
                    removed" % watch_dir)

    def loop(self):
        """
        block until we receive pyinotify events
        """
        notifier = pyinotify.Notifier(self.wm)
        notifier.coalesce_events()
        notifier.loop()
        # Experiments with running notifier in different modes
        # There are 3 options: normal, async, threaded.
        #import asyncore
        #pyinotify.AsyncNotifier(self.wm).loop()
        #asyncore.loop()
<|endoftext|>


<commit_before>"""
A set of utility functions to help the Collections
"""

import math
from typing import Any, Dict, List

__registered_collections = {}

def nCr(n: int, r: int) -> int:
    """
    Compute the binomial coefficient n! / (k! * (n-k)!)

    Parameters
    ----------
    n : int
        Number of samples
    r : int
        Denominator

    Returns
    -------
    ret : int
        Value
    """
    return math.factorial(n) / math.factorial(r) / math.factorial(n - r)

def register_collection(collection: "Collection") -> None:
    """Registers a collection for use by the factory.

    Parameters
    ----------
    collection : Collection
        The Collection class to be registered

    """

    class_name = collection.__name__.lower()
    # if class_name in __registered_collections:
    #     raise KeyError("Collection type '{}' already registered".format(class_name))
    __registered_collections[class_name] = collection

def collection_factory(data: Dict[str, Any], client: "FractalClient" = None) -> "Collection":
    """Creates a new Collection class from a JSON blob.

    Parameters
    ----------
    data : Dict[str, Any]
        The JSON blob to create a new class from.
    client : FractalClient, optional
        A FractalClient connected to a server

    Returns
    -------
    Collection
        A ODM of the data.

    """
    if "collection" not in data:
        raise KeyError("Attempted to create Collection from JSON, but no `collection` field found.")

    if data["collection"].lower() not in __registered_collections:
        raise KeyError("Attempted to create Collection of unknown type '{}'.".format(data["collection"]))

    return __registered_collections[data["collection"].lower()].from_json(data, client=client)

def collections_name_map() -> Dict[str, str]:
    """
    Returns a map of internal name to external Collection name.

    Returns
    -------
    Dict[str, str]
        Map of {'internal': 'user fiendly name'}
    """
    return {k: v.__name__ for k, v in __registered_collections.items()}

def list_known_collections() -> List[str]:
    """
    Returns the case sensitive list of Collection names.

    Returns
    -------
    List[str]
        A list of registered collections
    """
    return list(collection_name_map.values())

def composition_planner(program=None, method=None, basis=None, driver=None, keywords=None):
    """
    Plans out a given query into multiple pieces
    """

    base = {"program": program, "method": method, "basis": basis, "driver": driver, "keywords": keywords}

    if ("-d3" in method.lower()) and ("dftd3" != program.lower()) and ("hessian" != driver.lower()):
        dftd3keys = {"program": "dftd3", "method": method, "basis": None, "driver": driver, "keywords": None}
        base["method"] = method.lower().split("-d3")[0]

        return [dftd3keys, base]

    else:
        return [base]
<commit_msg>Hack fix for d3 calculations which shouldn't be split in the planner<commit_after>"""
A set of utility functions to help the Collections
"""

import math
from typing import Any, Dict, List

__registered_collections = {}

def nCr(n: int, r: int) -> int:
    """
    Compute the binomial coefficient n! / (k! * (n-k)!)

    Parameters
    ----------
    n : int
        Number of samples
    r : int
        Denominator

    Returns
    -------
    ret : int
        Value
    """
    return math.factorial(n) / math.factorial(r) / math.factorial(n - r)

def register_collection(collection: "Collection") -> None:
    """Registers a collection for use by the factory.

    Parameters
    ----------
    collection : Collection
        The Collection class to be registered

    """

    class_name = collection.__name__.lower()
    # if class_name in __registered_collections:
    #     raise KeyError("Collection type '{}' already registered".format(class_name))
    __registered_collections[class_name] = collection

def collection_factory(data: Dict[str, Any], client: "FractalClient" = None) -> "Collection":
    """Creates a new Collection class from a JSON blob.

    Parameters
    ----------
    data : Dict[str, Any]
        The JSON blob to create a new class from.
    client : FractalClient, optional
        A FractalClient connected to a server

    Returns
    -------
    Collection
        A ODM of the data.

    """
    if "collection" not in data:
        raise KeyError("Attempted to create Collection from JSON, but no `collection` field found.")

    if data["collection"].lower() not in __registered_collections:
        raise KeyError("Attempted to create Collection of unknown type '{}'.".format(data["collection"]))

    return __registered_collections[data["collection"].lower()].from_json(data, client=client)

def collections_name_map() -> Dict[str, str]:
    """
    Returns a map of internal name to external Collection name.

    Returns
    -------
    Dict[str, str]
        Map of {'internal': 'user fiendly name'}
    """
    return {k: v.__name__ for k, v in __registered_collections.items()}

def list_known_collections() -> List[str]:
    """
    Returns the case sensitive list of Collection names.

    Returns
    -------
    List[str]
        A list of registered collections
    """
    return list(collection_name_map.values())

def composition_planner(program=None, method=None, basis=None, driver=None, keywords=None):
    """
    Plans out a given query into multiple pieces
    """

    base = {"program": program, "method": method, "basis": basis, "driver": driver, "keywords": keywords}

    # HACK: shortcut these. See issue #688
    if method.lower() in ["b97-d3bj", "b97m-d3bj", "wb97m-d3bj"]:
        return [base]

    if ("-d3" in method.lower()) and ("dftd3" != program.lower()) and ("hessian" != driver.lower()):
        dftd3keys = {"program": "dftd3", "method": method, "basis": None, "driver": driver, "keywords": None}
        base["method"] = method.lower().split("-d3")[0]

        return [dftd3keys, base]

    else:
        return [base]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# This code is part of Qiskit.
#
# (C) Copyright IBM 2020.
#
# This code is licensed under the Apache License, Version 2.0. You may
# obtain a copy of this license in the LICENSE.txt file in the root directory
# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
#
# Any modifications or derivative works of this code must retain this
# copyright notice, and modified files need to carry a notice indicating
# that they have been altered from the originals.

"""AbelianGrouper Class"""

import itertools
from collections import defaultdict
from typing import List, Tuple, Dict, cast

import networkx as nx
import numpy as np

from qiskit.aqua import AquaError
from .converter_base import ConverterBase
from ..list_ops.list_op import ListOp
from ..list_ops.summed_op import SummedOp
from ..operator_base import OperatorBase
from ..primitive_ops.pauli_op import PauliOp
from ..state_fns.operator_state_fn import OperatorStateFn

class AbelianGrouper(ConverterBase):
    """The AbelianGrouper converts SummedOps into a sum of Abelian sums.

    Meaning, it will traverse the Operator, and when it finds a SummedOp, it will evaluate which of
    the summed sub-Operators commute with one another. It will then convert each of the groups of
    commuting Operators into their own SummedOps, and return the sum-of-commuting-SummedOps.
    This is particularly useful for cases where mutually commuting groups can be handled
    similarly, as in the case of Pauli Expectations, where commuting Paulis have the same
    diagonalizing circuit rotation, or Pauli Evolutions, where commuting Paulis can be
    diagonalized together.
    """

    def __init__(self, traverse: bool = True) -> None:
        """
        Args:
            traverse: Whether to convert only the Operator passed to ``convert``, or traverse
                down that Operator.
        """
        self._traverse = traverse

    def convert(self, operator: OperatorBase) -> OperatorBase:
        """Check if operator is a SummedOp, in which case covert it into a sum of mutually
        commuting sums, or if the Operator contains sub-Operators and ``traverse`` is True,
        attempt to convert any sub-Operators.

        Args:
            operator: The Operator to attempt to convert.

        Returns:
            The converted Operator.
        """
        # pylint: disable=cyclic-import,import-outside-toplevel
        from ..evolutions.evolved_op import EvolvedOp

        if isinstance(operator, ListOp):
            if isinstance(operator, SummedOp) and all(isinstance(op, PauliOp)
                                                      for op in operator.oplist):
                # For now, we only support graphs over Paulis.
                return self.group_subops(operator)
            elif self._traverse:
                return operator.traverse(self.convert)
            else:
                return operator
        elif isinstance(operator, OperatorStateFn) and self._traverse:
            return OperatorStateFn(self.convert(operator.primitive),
                                   is_measurement=operator.is_measurement,
                                   coeff=operator.coeff)
        elif isinstance(operator, EvolvedOp) and self._traverse:
            return EvolvedOp(self.convert(operator.primitive), coeff=operator.coeff)  # type: ignore
        else:
            return operator

    @classmethod
    def group_subops(cls, list_op: ListOp, fast: bool = True, use_nx: bool = False) -> ListOp:
        """Given a ListOp, attempt to group into Abelian ListOps of the same type.

        Args:
            list_op: The Operator to group into Abelian groups
            fast: Enable the fast commutation graph generation if all operators are Pauli operators
            use_nx: Enable networkx.coloring.greedy_color instead of the numpy-based coloring

        Returns:
            The grouped Operator.

        Raises:
            AquaError: Any of list_op's sub-ops do not have a ``commutes`` method.
        """
        if any(not hasattr(op, 'commutes') for op in list_op.oplist):
            raise AquaError('Cannot determine Abelian groups if an Operator in list_op does not '
                            'contain a `commutes` method'.format())

        if fast and all(isinstance(op, PauliOp) for op in list_op.oplist):
            edges = cls._commutation_graph_fast(list_op)
        else:
            edges = cls._commutation_graph(list_op)
        nodes = range(len(list_op))

        # Keys in coloring_dict are nodes, values are colors
        if use_nx:
            coloring_dict = cls._networkx_coloring(nodes, edges)
        else:
            coloring_dict = cls._largest_degree_first_coloring(nodes, edges)

        groups = {}  # type: Dict
        # sort items so that the output is consistent with all options (fast and use_nx)
        for idx, color in sorted(coloring_dict.items()):
            groups.setdefault(color, []).append(list_op[idx])

        group_ops = [list_op.__class__(group, abelian=True) for group in groups.values()]
        if len(group_ops) == 1:
            return group_ops[0] * list_op.coeff  # type: ignore
        return list_op.__class__(group_ops, coeff=list_op.coeff)  # type: ignore

    @staticmethod
    def _commutation_graph(list_op: ListOp) -> List[Tuple[int, int]]:
        """Create edges (i, j) if i and j are not commutable.

        Args:
            list_op: list_op

        Returns:
            A list of pairs of indices of the operators that are not commutable
        """
        indices = range(len(list_op))
        return [(i, j) for i, j in itertools.combinations(indices, 2)
                if not list_op[i].commutes(list_op[j])]  # type: ignore

    @staticmethod
    def _commutation_graph_fast(list_op: ListOp) -> List[Tuple[int, int]]:
        """Create edges (i, j) if i and j are not commutable.

        Note:
            This method is applicable to only PauliOps.

        Args:
            list_op: list_op

        Returns:
            A list of pairs of indices of the operators that are not commutable
        """
        # convert a Pauli operator into int vector where {I: 0, X: 2, Y: 3, Z: 1}
        mat1 = np.array([op.primitive.z + 2 * op.primitive.x for op in list_op], dtype=np.int8)
        mat2 = mat1[:, None]
        # mat3[i, j] is True if i and j are commutable with TPB
        mat3 = (((mat1 * mat2) * (mat1 - mat2)) == 0).all(axis=2)
        # return [(i, j) if mat3[i, j] is False and i < j]
        return cast(List[Tuple[int, int]], zip(*np.where(np.triu(np.logical_not(mat3), k=1))))

    @staticmethod
    def _networkx_coloring(nodes: range, edges: List[Tuple[int, int]], strategy='largest_first') \
            -> Dict[int, List[int]]:
        graph = nx.Graph()
        graph.add_nodes_from(nodes)
        graph.add_edges_from(edges)
        # pylint: disable=no-member
        return nx.coloring.greedy_color(graph, strategy=strategy)

    @staticmethod
    def _largest_degree_first_coloring(nodes: range, edges: List[Tuple[int, int]]) \
            -> Dict[int, List[int]]:
        adj = defaultdict(list)
        for i, j in edges:
            adj[i].append(j)
            adj[j].append(i)
        color = np.array([-1] * (max(nodes) + 1))
        all_colors = np.arange(len(nodes))
        for i in sorted(nodes, key=lambda x: len(adj[x]), reverse=True):
            neighbors = adj[i]
            color_neighbors = color[neighbors]
            color_neighbors = color_neighbors[color_neighbors >= 0]
            mask = np.ones(len(nodes), dtype=bool)
            mask[color_neighbors] = False
            color[i] = np.min(all_colors[mask])
        if np.min(color[nodes]) == -1:
            # never reach here if the input graph is valid
            raise AquaError('Uncolored nodes are left')
        return {i: color[i] for i in nodes}
<commit_msg>Simplify AbelianGrouper using retworkx (Qiskit/qiskit-aqua#1217)<commit_after># -*- coding: utf-8 -*-

# This code is part of Qiskit.
#
# (C) Copyright IBM 2020.
#
# This code is licensed under the Apache License, Version 2.0. You may
# obtain a copy of this license in the LICENSE.txt file in the root directory
# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
#
# Any modifications or derivative works of this code must retain this
# copyright notice, and modified files need to carry a notice indicating
# that they have been altered from the originals.

"""AbelianGrouper Class"""

import warnings
from typing import List, Tuple, Dict, cast, Optional

import numpy as np
import retworkx as rx

from qiskit.aqua import AquaError
from .converter_base import ConverterBase
from ..list_ops.list_op import ListOp
from ..list_ops.summed_op import SummedOp
from ..operator_base import OperatorBase
from ..primitive_ops.pauli_op import PauliOp
from ..state_fns.operator_state_fn import OperatorStateFn

class AbelianGrouper(ConverterBase):
    """The AbelianGrouper converts SummedOps into a sum of Abelian sums.

    Meaning, it will traverse the Operator, and when it finds a SummedOp, it will evaluate which of
    the summed sub-Operators commute with one another. It will then convert each of the groups of
    commuting Operators into their own SummedOps, and return the sum-of-commuting-SummedOps.
    This is particularly useful for cases where mutually commuting groups can be handled
    similarly, as in the case of Pauli Expectations, where commuting Paulis have the same
    diagonalizing circuit rotation, or Pauli Evolutions, where commuting Paulis can be
    diagonalized together.
    """

    def __init__(self, traverse: bool = True) -> None:
        """
        Args:
            traverse: Whether to convert only the Operator passed to ``convert``, or traverse
                down that Operator.
        """
        self._traverse = traverse

    def convert(self, operator: OperatorBase) -> OperatorBase:
        """Check if operator is a SummedOp, in which case covert it into a sum of mutually
        commuting sums, or if the Operator contains sub-Operators and ``traverse`` is True,
        attempt to convert any sub-Operators.

        Args:
            operator: The Operator to attempt to convert.

        Returns:
            The converted Operator.
        """
        # pylint: disable=cyclic-import,import-outside-toplevel
        from ..evolutions.evolved_op import EvolvedOp

        if isinstance(operator, ListOp):
            if isinstance(operator, SummedOp) and all(isinstance(op, PauliOp)
                                                      for op in operator.oplist):
                # For now, we only support graphs over Paulis.
                return self.group_subops(operator)
            elif self._traverse:
                return operator.traverse(self.convert)
            else:
                return operator
        elif isinstance(operator, OperatorStateFn) and self._traverse:
            return OperatorStateFn(self.convert(operator.primitive),
                                   is_measurement=operator.is_measurement,
                                   coeff=operator.coeff)
        elif isinstance(operator, EvolvedOp) and self._traverse:
            return EvolvedOp(self.convert(operator.primitive), coeff=operator.coeff)  # type: ignore
        else:
            return operator

    @classmethod
    def group_subops(cls, list_op: ListOp, fast: Optional[bool] = None,
                     use_nx: Optional[bool] = None) -> ListOp:
        """Given a ListOp, attempt to group into Abelian ListOps of the same type.

        Args:
            list_op: The Operator to group into Abelian groups
            fast: Ignored - parameter will be removed in future release
            use_nx: Ignored - parameter will be removed in future release

        Returns:
            The grouped Operator.

        Raises:
            AquaError: If any of list_op's sub-ops is not ``PauliOp``.
        """
        if fast is not None or use_nx is not None:
            warnings.warn('Options `fast` and `use_nx` of `AbelianGrouper.group_subops` are '
                          'no longer used and are now deprecated and will be removed no '
                          'sooner than 3 months following the 0.8.0 release.')

        for op in list_op.oplist:
            if not isinstance(op, PauliOp):
                raise AquaError(
                    'Cannot determine Abelian groups if any Operator in list_op is not '
                    '`PauliOp`. E.g., {} ({})'.format(op, type(op)))

        edges = cls._commutation_graph(list_op)
        nodes = range(len(list_op))

        graph = rx.PyGraph()
        graph.add_nodes_from(nodes)
        graph.add_edges_from_no_data(edges)
        # Keys in coloring_dict are nodes, values are colors
        coloring_dict = rx.graph_greedy_color(graph)

        groups = {}  # type: Dict
        # sort items so that the output is consistent with all options (fast and use_nx)
        for idx, color in sorted(coloring_dict.items()):
            groups.setdefault(color, []).append(list_op[idx])

        group_ops = [list_op.__class__(group, abelian=True) for group in groups.values()]
        if len(group_ops) == 1:
            return group_ops[0] * list_op.coeff  # type: ignore
        return list_op.__class__(group_ops, coeff=list_op.coeff)  # type: ignore

    @staticmethod
    def _commutation_graph(list_op: ListOp) -> List[Tuple[int, int]]:
        """Create edges (i, j) if i and j are not commutable.

        Note:
            This method is applicable to only PauliOps.

        Args:
            list_op: list_op

        Returns:
            A list of pairs of indices of the operators that are not commutable
        """
        # convert a Pauli operator into int vector where {I: 0, X: 2, Y: 3, Z: 1}
        mat1 = np.array([op.primitive.z + 2 * op.primitive.x for op in list_op], dtype=np.int8)
        mat2 = mat1[:, None]
        # mat3[i, j] is True if i and j are commutable with TPB
        mat3 = (((mat1 * mat2) * (mat1 - mat2)) == 0).all(axis=2)
        # return [(i, j) if mat3[i, j] is False and i < j]
        return cast(List[Tuple[int, int]], list(zip(*np.where(np.triu(np.logical_not(mat3), k=1)))))
<|endoftext|>


<commit_before>import os
import errno
import shutil
import fnmatch
from glob import has_magic

def makedirs(path, mode=0o777, exist_ok=False):
    """Create a directory, and ancestors if necessary.

    If exists_ok is True, no exception is raised when path exists.
    Mimics behaviour of os.makedirs in python 3.2.
    """
    try:
        os.makedirs(path, mode)
    except OSError as e:
        if not exist_ok or e.errno != errno.EEXIST:
            raise

def rmtree_onerror(func, path, exc):
    """Handle deleting read-only files on windows."""
    import stat
    excvalue = exc[1]
    if func in (os.rmdir, os.remove):
        os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # 0777
        func(path)
    else:
        raise

def remove(path):
    """Remove a directory or a file."""
    if os.path.isdir(path):
        shutil.rmtree(path, onerror=rmtree_onerror)
    else:
        os.remove(path)

def copy(src, dest, parents=False, replace=False, ignore=None):
    """Copy a directory or file to dest.

    If parents is True, create ancestor directories if they do not exist.

    If replace is True and src is a directory, first remove dest if it exists.
    Files will always be overwritten.

    ignore should be function that can be used as shutil.copytree() ignore argument
    """
    if parents:
        makedirs(os.path.dirname(dest), exist_ok=True)

    if os.path.isdir(src):
        if replace:
            if os.path.exists(dest):
                remove(dest)

        shutil.copytree(src, dest, symlinks=True, ignore=ignore)
    else:
        if os.path.islink(src):
            if os.path.isdir(dest):
                dest = os.path.join(dest, os.path.basename(src))
            if os.path.exists(dest):
                remove(dest)
            os.symlink(os.readlink(src), dest)
        else:
            shutil.copy(src, dest)

def copy_ignore(patterns):
    """Return a function that can be used as shutil.copytree() ignore argument.

    All names that match one of the patterns according to match(), will be ignored.
    Note: the full path has to match, not just the name.
    """
    def _ignore(path, names):
        ignored_names = []
        for n in names:
            if match_list(os.path.join(path, n), patterns):
                ignored_names.append(n)
        return set(ignored_names)
    return _ignore

def separate_path(path):
    """Separate a path into its components"""
    path = path.rstrip("/").rstrip("\\")

    components = []
    #don't use regular string split so that we don't have to be strict on the format
    #of the path
    dirname, basename = os.path.split(path)
    while basename:
        components.append(basename)
        dirname, basename = os.path.split(dirname)
    components.reverse()

    return components

def match(path, pattern):
    """
    Check if path matches pattern.

    Pattern matching behaves the same way as it does in Apache Ant.
    That is, it is done per-component. For example the pattern *.h will match
    file.h but not dir/file.h. Use two asterisks to match any number of components.

    Examples:
        lib/*.a will match lib/file.a, but not lib/debug/file.a
        lib/*/* will match lib/a/b, lib/sub1/b.c, etc.
        lib/**/*.a will match lib/file.a, lib/a/file.a, lib/a/b/file.a, etc.
        **/test/** will match test, a/test, test/a, a/b/c/test/d/e, etc.
        dir/file_???.txt will match dir/file_abc.txt, but not dir/file_abcd.txt

    fnmatch.fnmatch() is used for the actual matching, so character ranges can also be used.
    """
    path_segments = separate_path(path)
    pattern_segments = separate_path(pattern)
    path_length = len(path_segments)
    pattern_length = len(pattern_segments)

    i = 0
    j = 0
    while i < pattern_length and j < path_length:
        if pattern_segments[i] == "**":
            if i + 1 < pattern_length:
                i += 1
            else:
                #the '**' is at the end, so any path segments after will be matched
                return True

            #increment j until a path segment matches or we come to the end
            while j < path_length:
                #in case there are multiple path segments that would match,
                #make sure we find a match with the correct number of segments remaining
                equal_remaining = path_length - j == pattern_length - i
                allow_different = pattern_segments[i:].count("**")
                if (fnmatch.fnmatch(path_segments[j], pattern_segments[i]) and
                    (equal_remaining or allow_different)):
                    break
                else:
                    j += 1
        else:
            #normal segment by segment matching
            if not fnmatch.fnmatch(path_segments[j], pattern_segments[i]):
                return False
            i += 1
            j += 1

            #handle the special case of '**/x/**' when x is the last path segment.
            #when i points to the last '**', j will already be one past the end,
            #so simply increment i again
            if i == pattern_length - 1 and pattern_segments[i] == "**":
                if j == path_length:
                    i += 1

    #if we have reached here, it should mean that we have successfully
    #matched all segments
    if i == pattern_length and j == path_length:
        return True

    return False

def match_list(path, patterns):
    """Check if path matches any pattern in a list"""
    for p in patterns:
        if match(path, p):
            return True
    return False

def walk(path, depth_limit=None):
    """Walk directory tree rooted at path

    For each directory, it yields (dirpath, dirnames, filenames).
    dirpath is the path of the directory being visited, dirnames is the list
    of directories in dirpath and filenames is the list of files in dirpath.

    If depth_limit is not None, the tree is only walked up to depth depth_limit.
    For example, if depth_limit is 2, the contents of the root directory is returned,
    and then the contents of the subdirectories of the root directory.
    """
    def _walk_recursive(path, depth):
        if depth_limit is None or depth < depth_limit:
            try:
                contents = os.listdir(path)
            except OSError:
                return

            dirs = []
            files = []
            for name in contents:
                subpath = os.path.join(path, name)
                if os.path.isdir(subpath):
                    dirs.append(name)
                else:
                    files.append(name)

            yield path, dirs, files

            for name in dirs:
                for x in _walk_recursive(os.path.join(path, name), depth + 1):
                    yield x

    for x in _walk_recursive(path, 0):
        yield x

def glob(pattern):
    """Return a generator that yields paths matching pattern

    The matching is done with match(), so glob('src/**/*.h') will return
    all .h files in the directory tree 'src'
    """
    if os.path.exists(pattern):
        yield pattern
    else:
        path = pattern
        depth_limit = 0
        while has_magic(path):
            path, basename = os.path.split(path)
            depth_limit += 1
        if not path:
            path = "."
        if pattern.count("**"):
            depth_limit = None

        for parent, dirs, files in walk(path, depth_limit):
            for p in dirs + files:
                subpath = os.path.join(parent, p)
                if match(subpath, pattern):
                    yield subpath
<commit_msg>Handle special case of deleting folders on windows<commit_after>import os
import errno
import shutil
import time
import fnmatch
from glob import has_magic

def makedirs(path, mode=0o777, exist_ok=False):
    """Create a directory, and ancestors if necessary.

    If exists_ok is True, no exception is raised when path exists.
    Mimics behaviour of os.makedirs in python 3.2.
    """
    try:
        os.makedirs(path, mode)
    except OSError as e:
        if not exist_ok or e.errno != errno.EEXIST:
            raise

def rmtree_onerror(func, path, exc):
    """Handle deleting things on windows."""
    import stat
    excvalue = exc[1]
    if func in (os.rmdir, os.remove) and excvalue.errno == errno.ENOTEMPTY:
        #Because windows sometimes keeps a directory alive for a little
        #after it has been removed, wait and try again.
        elapsed = 0
        while elapsed < 0.1:
            try:
                func(path)
            except WindowsError:
                time.sleep(0.01)
                elapsed += 0.01
            else:
                break
    elif func in (os.rmdir, os.remove) and excvalue.errno == errno.EACCES:
        #Read only directory
        os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # 0777
        func(path)
    else:
        raise

def remove(path):
    """Remove a directory or a file."""
    if os.path.isdir(path):
        shutil.rmtree(path, onerror=rmtree_onerror)
    else:
        os.remove(path)

def copy(src, dest, parents=False, replace=False, ignore=None):
    """Copy a directory or file to dest.

    If parents is True, create ancestor directories if they do not exist.

    If replace is True and src is a directory, first remove dest if it exists.
    Files will always be overwritten.

    ignore should be function that can be used as shutil.copytree() ignore argument
    """
    if parents:
        makedirs(os.path.dirname(dest), exist_ok=True)

    if os.path.isdir(src):
        if replace:
            if os.path.exists(dest):
                remove(dest)

        shutil.copytree(src, dest, symlinks=True, ignore=ignore)
    else:
        if os.path.islink(src):
            if os.path.isdir(dest):
                dest = os.path.join(dest, os.path.basename(src))
            if os.path.exists(dest):
                remove(dest)
            os.symlink(os.readlink(src), dest)
        else:
            shutil.copy(src, dest)

def copy_ignore(patterns):
    """Return a function that can be used as shutil.copytree() ignore argument.

    All names that match one of the patterns according to match(), will be ignored.
    Note: the full path has to match, not just the name.
    """
    def _ignore(path, names):
        ignored_names = []
        for n in names:
            if match_list(os.path.join(path, n), patterns):
                ignored_names.append(n)
        return set(ignored_names)
    return _ignore

def separate_path(path):
    """Separate a path into its components"""
    path = path.rstrip("/").rstrip("\\")

    components = []
    #don't use regular string split so that we don't have to be strict on the format
    #of the path
    dirname, basename = os.path.split(path)
    while basename:
        components.append(basename)
        dirname, basename = os.path.split(dirname)
    components.reverse()

    return components

def match(path, pattern):
    """
    Check if path matches pattern.

    Pattern matching behaves the same way as it does in Apache Ant.
    That is, it is done per-component. For example the pattern *.h will match
    file.h but not dir/file.h. Use two asterisks to match any number of components.

    Examples:
        lib/*.a will match lib/file.a, but not lib/debug/file.a
        lib/*/* will match lib/a/b, lib/sub1/b.c, etc.
        lib/**/*.a will match lib/file.a, lib/a/file.a, lib/a/b/file.a, etc.
        **/test/** will match test, a/test, test/a, a/b/c/test/d/e, etc.
        dir/file_???.txt will match dir/file_abc.txt, but not dir/file_abcd.txt

    fnmatch.fnmatch() is used for the actual matching, so character ranges can also be used.
    """
    path_segments = separate_path(path)
    pattern_segments = separate_path(pattern)
    path_length = len(path_segments)
    pattern_length = len(pattern_segments)

    i = 0
    j = 0
    while i < pattern_length and j < path_length:
        if pattern_segments[i] == "**":
            if i + 1 < pattern_length:
                i += 1
            else:
                #the '**' is at the end, so any path segments after will be matched
                return True

            #increment j until a path segment matches or we come to the end
            while j < path_length:
                #in case there are multiple path segments that would match,
                #make sure we find a match with the correct number of segments remaining
                equal_remaining = path_length - j == pattern_length - i
                allow_different = pattern_segments[i:].count("**")
                if (fnmatch.fnmatch(path_segments[j], pattern_segments[i]) and
                    (equal_remaining or allow_different)):
                    break
                else:
                    j += 1
        else:
            #normal segment by segment matching
            if not fnmatch.fnmatch(path_segments[j], pattern_segments[i]):
                return False
            i += 1
            j += 1

            #handle the special case of '**/x/**' when x is the last path segment.
            #when i points to the last '**', j will already be one past the end,
            #so simply increment i again
            if i == pattern_length - 1 and pattern_segments[i] == "**":
                if j == path_length:
                    i += 1

    #if we have reached here, it should mean that we have successfully
    #matched all segments
    if i == pattern_length and j == path_length:
        return True

    return False

def match_list(path, patterns):
    """Check if path matches any pattern in a list"""
    for p in patterns:
        if match(path, p):
            return True
    return False

def walk(path, depth_limit=None):
    """Walk directory tree rooted at path

    For each directory, it yields (dirpath, dirnames, filenames).
    dirpath is the path of the directory being visited, dirnames is the list
    of directories in dirpath and filenames is the list of files in dirpath.

    If depth_limit is not None, the tree is only walked up to depth depth_limit.
    For example, if depth_limit is 2, the contents of the root directory is returned,
    and then the contents of the subdirectories of the root directory.
    """
    def _walk_recursive(path, depth):
        if depth_limit is None or depth < depth_limit:
            try:
                contents = os.listdir(path)
            except OSError:
                return

            dirs = []
            files = []
            for name in contents:
                subpath = os.path.join(path, name)
                if os.path.isdir(subpath):
                    dirs.append(name)
                else:
                    files.append(name)

            yield path, dirs, files

            for name in dirs:
                for x in _walk_recursive(os.path.join(path, name), depth + 1):
                    yield x

    for x in _walk_recursive(path, 0):
        yield x

def glob(pattern):
    """Return a generator that yields paths matching pattern

    The matching is done with match(), so glob('src/**/*.h') will return
    all .h files in the directory tree 'src'
    """
    if os.path.exists(pattern):
        yield pattern
    else:
        path = pattern
        depth_limit = 0
        while has_magic(path):
            path, basename = os.path.split(path)
            depth_limit += 1
        if not path:
            path = "."
        if pattern.count("**"):
            depth_limit = None

        for parent, dirs, files in walk(path, depth_limit):
            for p in dirs + files:
                subpath = os.path.join(parent, p)
                if match(subpath, pattern):
                    yield subpath
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Hive Appier Framework
# Copyright (c) 2008-2016 Hive Solutions Lda.
#
# This file is part of Hive Appier Framework.
#
# Hive Appier Framework is free software: you can redistribute it and/or modify
# it under the terms of the Apache License as published by the Apache
# Foundation, either version 2.0 of the License, or (at your option) any
# later version.
#
# Hive Appier Framework is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Apache License for more details.
#
# You should have received a copy of the Apache License along with
# Hive Appier Framework. If not, see <http://www.apache.org/licenses/>.

__author__ = "João Magalhães <joamag@hive.pt>"
""" The author(s) of the module """

__version__ = "1.0.0"
""" The version of the module """

__revision__ = "$LastChangedRevision$"
""" The revision number of the module """

__date__ = "$LastChangedDate$"
""" The last change date of the module """

__copyright__ = "Copyright (c) 2008-2016 Hive Solutions Lda."
""" The copyright for the module """

__license__ = "Apache License, Version 2.0"
""" The license for the module """

import threading

class AsyncManager(object):

    def __init__(self, owner):
        object.__init__(self)
        self.owner = owner

    def start(self):
        pass

    def stop(self):
        pass

    def add(self, method, args, kwargs, request = None, mid = None):
        pass

class SimpleManager(AsyncManager):

    def add(self, method, args, kwargs, request = None, mid = None):
        if request: kwargs["request"] = request
        if mid: kwargs["mid"] = mid
        thread = threading.Thread(
            target = method,
            args = args,
            kwargs = kwargs
        )
        thread.start()

class QueueManager(AsyncManager):

    def start(self):
        self.thread = threading.Thread(target = self.handler)
        self.queue = []
        self.condition = threading.Condition()
        self.running = True
        self.thread.daemon = True
        self.thread.start()

    def stop(self):
        self.running = False

    def add(self, method, args, kwargs, request = None, mid = None):
        if request: kwargs["request"] = request
        if mid: kwargs["mid"] = mid
        item = (method, args, kwargs)
        self.condition.acquire()
        try:
            self.queue.append(item)
            self.condition.notify()
        finally:
            self.condition.release()

    def handler(self):
        while self.running:
            self.condition.acquire()
            while not self.queue: self.condition.wait()
            try: item = self.queue.pop(0)
            finally: self.condition.release()
            method, args, kwargs = item
            try: method(*args, **kwargs)
            except BaseException as exception:
                self.owner.log_error(
                    exception,
                    message = "Problem handling async item: %s"
                )
<commit_msg>new async values<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Hive Appier Framework
# Copyright (c) 2008-2016 Hive Solutions Lda.
#
# This file is part of Hive Appier Framework.
#
# Hive Appier Framework is free software: you can redistribute it and/or modify
# it under the terms of the Apache License as published by the Apache
# Foundation, either version 2.0 of the License, or (at your option) any
# later version.
#
# Hive Appier Framework is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Apache License for more details.
#
# You should have received a copy of the Apache License along with
# Hive Appier Framework. If not, see <http://www.apache.org/licenses/>.

__author__ = "João Magalhães <joamag@hive.pt>"
""" The author(s) of the module """

__version__ = "1.0.0"
""" The version of the module """

__revision__ = "$LastChangedRevision$"
""" The revision number of the module """

__date__ = "$LastChangedDate$"
""" The last change date of the module """

__copyright__ = "Copyright (c) 2008-2016 Hive Solutions Lda."
""" The copyright for the module """

__license__ = "Apache License, Version 2.0"
""" The license for the module """

import threading

class AsyncManager(object):

    def __init__(self, owner):
        object.__init__(self)
        self.owner = owner

    def start(self):
        pass

    def stop(self):
        pass

    def add(self, method, args, kwargs, request = None, mid = None):
        pass

class SimpleManager(AsyncManager):

    def add(self, method, args, kwargs, request = None, mid = None):
        AsyncManager.add(self, method, args, kwargs, request = request, mid = mid)
        if request: kwargs["request"] = request
        if mid: kwargs["mid"] = mid
        thread = threading.Thread(
            target = method,
            args = args,
            kwargs = kwargs
        )
        thread.start()

class QueueManager(AsyncManager):

    def start(self):
        AsyncManager.start(self)
        self.thread = threading.Thread(target = self.handler)
        self.queue = []
        self.condition = threading.Condition()
        self.running = True
        self.thread.daemon = True
        self.thread.start()

    def stop(self):
        AsyncManager.stop(self)
        self.running = False

    def add(self, method, args, kwargs, request = None, mid = None):
        AsyncManager.add(self, method, args, kwargs, request = request, mid = mid)
        if request: kwargs["request"] = request
        if mid: kwargs["mid"] = mid
        item = (method, args, kwargs)
        self.condition.acquire()
        try:
            self.queue.append(item)
            self.condition.notify()
        finally:
            self.condition.release()

    def handler(self):
        while self.running:
            self.condition.acquire()
            while not self.queue: self.condition.wait()
            try: item = self.queue.pop(0)
            finally: self.condition.release()
            method, args, kwargs = item
            try: method(*args, **kwargs)
            except BaseException as exception:
                self.owner.log_error(
                    exception,
                    message = "Problem handling async item: %s"
                )
<|endoftext|>


<commit_before>from datetime import datetime
from hashlib import sha1
from PIL import Image
import errno
import logging
import os
import traceback
from urllib import unquote

from django.conf import settings
from django import template
from django.utils import timezone
from watermarker import utils
from watermarker.models import Watermark

register = template.Library()

# determine the quality of the image after the watermark is applied
QUALITY = getattr(settings, 'WATERMARKING_QUALITY', 85)
OBSCURE = getattr(settings, 'WATERMARK_OBSCURE_ORIGINAL', True)
RANDOM_POS_ONCE = getattr(settings, 'WATERMARK_RANDOM_POSITION_ONCE', True)

log = logging.getLogger('watermarker')

class Watermarker(object):

    def __call__(self, url, name, position=None, opacity=0.5, tile=False,
                 scale=1.0, greyscale=False, rotation=0, obscure=OBSCURE,
                 quality=QUALITY, random_position_once=RANDOM_POS_ONCE):
        """
        Creates a watermarked copy of an image.

        * ``name``: This is the name of the Watermark object that you wish to
          apply to the image.
        * ``position``:  There are several options.

            * ``R``: random placement, which is the default behavior.
            * ``C``: center the watermark
            * ``XxY`` where ``X`` is either a specific pixel position on the
              x-axis or a percentage of the total width of the target image and
              ``Y`` is a specific pixel position on the y-axis of the image or
              a percentage of the total height of the target image.  These
              values represent the location of the top and left edges of the
              watermark.  If either ``X`` or ``Y`` is a percentage, you must
              use a percent sign.  This is not used if either one of the
              ``tiled`` or ``scale`` parameters are true.

              Examples:

                * ``50%x50%``: positions the watermark at the center of the
                  image.
                * ``50%x100``: positions the watermark at the midpoint of the
                  total width of the image and 100 pixels from the top of the
                  image
                * ``100x50%``: positions the watermark at the midpoint of the
                  total height of the image and 100 pixels from the left edge
                  of the image
                * ``100x100``: positions the top-left corner of the watermark
                  at 100 pixels from the top of the image and 100 pixels from
                  the left edge of the image.

            * ``br``, ``bl``, ``tr``, ``tl`` where ``b`` means "bottom", ``t``
              means "top", ``l`` means "left", and ``r`` means "right".  This
              will position the watermark at the extreme edge of the original
              image with just enough room for the watermark to "fully show".
              This assumes the watermark is not as big as the original image.

        * ``opacity``: an integer from 0 to 100.  This value represents the
          transparency level of the watermark when it is applied.  A value of
          100 means the watermark is completely opaque while a value of 0 means
          the watermark will be invisible.
        * ``tile``: ``True`` or ``False`` to specify whether or not the
          watermark shall be tiled across the entire image.
        * ``scale``: a floating-point number above 0 to specify the scaling for
          the watermark.  If you want the watermark to be scaled to its maximum
          without falling off the edge of the target image, use ``F``.  By
          default, scale is set to ``1.0``, or 1:1 scaling, meaning the
          watermark will be placed on the target image at its original size.
        * ``greyscale``: ``True`` or ``False`` to specify whether or not the
          watermark should be converted to a greyscale image before applying it
          to the target image.  Default is ``False``.
        * ``rotation``: 0 to 359 to specify the number of degrees to rotate the
          watermark before applying it to the target image.  Alternatively, you
          may set ``rotation=R`` for a random rotation value.
        * ``obscure``: set to ``False`` if you wish to expose the original
          image's filename.  Defaults to ``True``.
        * ``quality``: the quality of the resulting watermarked image.  Default
          is 85.
        """

        # look for the specified watermark by name.  If it's not there, go no
        # further
        try:
            watermark = Watermark.objects.get(name=name, is_active=True)
        except Watermark.DoesNotExist:
            log.error('Watermark "%s" does not exist... Bailing out.' % name)
            return url

        # make sure URL is a string
        url = str(url)

        basedir = '%s/watermarked' % os.path.dirname(url)
        base, ext = os.path.splitext(os.path.basename(url))

        # open the target image file along with the watermark image
        target_path = self.get_url_path(url)
        target = Image.open(target_path)
        mark = Image.open(watermark.image.path)

        # determine the actual value that the parameters provided will render
        random_position = bool(position is None or str(position).lower() == 'r')
        scale = utils.determine_scale(scale, target, mark)
        mark = mark.resize(scale, resample=Image.ANTIALIAS)
        rotation = utils.determine_rotation(rotation, mark)
        pos = utils.determine_position(position, target, mark)

        # see if we need to create only one randomly positioned watermarked
        # image
        if not random_position or \
            (not random_position_once and random_position):
            log.debug('Generating random position for watermark each time')
            position = pos
        else:
            log.debug('Random positioning watermark once')

        params = {
            'position':  position,
            'opacity':   opacity,
            'scale':     scale,
            'tile':      tile,
            'greyscale': greyscale,
            'rotation':  rotation,
            'base':      base,
            'ext':       ext,
            'quality':   quality,
            'watermark': watermark.id,
            'opacity_int': int(opacity * 100),
            'left':      pos[0],
            'top':       pos[1],
        }
        log.debug('Params: %s' % params)

        wm_name = self.watermark_name(mark, **params)
        wm_url = self.watermark_path(basedir, base, ext, wm_name, obscure)
        wm_path = self.get_url_path(wm_url)
        log.debug('Watermark name: %s; URL: %s; Path: %s' % (
            wm_name, wm_url, wm_path
        ))

        # see if the image already exists on the filesystem.  If it does, use
        # it.
        if os.access(wm_path, os.R_OK):
            # see if the Watermark object was modified since the file was
            # created
            modified = datetime.fromtimestamp(os.path.getmtime(wm_path))
            # Django 1.4+ timezone support
            modified = timezone.make_aware(modified, timezone.get_default_timezone())

            # only return the old file if things appear to be the same
            if modified >= watermark.date_updated:
                log.info('Watermark exists and has not changed.  Bailing out.')
                return wm_url

        # make sure the position is in our params for the watermark
        params['position'] = pos

        self.create_watermark(target, mark, wm_path, **params)

        # send back the URL to the new, watermarked image
        return wm_url

    def get_url_path(self, url, root=settings.MEDIA_ROOT,
        url_root=settings.MEDIA_URL):
        """Makes a filesystem path from the specified URL"""

        if url.startswith(url_root):
            url = url[len(url_root):] # strip media root url

        return os.path.normpath(os.path.join(root, url))

    def watermark_name(self, mark, **kwargs):
        """Comes up with a good filename for the watermarked image"""

        params = [
            '%(base)s',
            'wm',
            'w%(watermark)i',
            'o%(opacity_int)i',
            'gs%(greyscale)i',
            'r%(rotation)i',
            '_p%(position)s',
        ]

        scale = kwargs.get('scale', None)
        if scale and scale != mark.size:
            params.append('_s%i' % (float(kwargs['scale'][0]) / mark.size[0] * 100))

        if kwargs.get('tile', None):
            params.append('_tiled')

        # make thumbnail filename
        name = '%s%s' % ('_'.join(params), kwargs['ext'])
        return name % kwargs

    def watermark_path(self, basedir, base, ext, wm_name, obscure=True):
        """Determines an appropriate watermark path"""

        hash = sha1(wm_name).hexdigest()

        # figure out where the watermark would be saved on the filesystem
        if obscure:
            log.debug('Obscuring original image name: %s => %s' % (wm_name, hash))
            new_file = os.path.join(basedir, hash + ext)
        else:
            log.debug('Not obscuring original image name.')
            new_file = os.path.join(basedir, hash, base + ext)

        # make sure the destination directory exists
        try:
            root = self.get_url_path(new_file)
            os.makedirs(os.path.dirname(root))
        except OSError, exc:
            if exc.errno == errno.EEXIST:
                # not to worry, directory exists
                pass
            else:
                log.error('Error creating path: %s' % traceback.format_exc())
                raise
        else:
            log.debug('Created directory: %s' % root)

        return new_file

    def create_watermark(self, target, mark, path, quality=QUALITY, **kwargs):
        """Create the watermarked image on the filesystem"""

        im = utils.watermark(target, mark, **kwargs)
        im.save(path, quality=quality)

        return im

def watermark(url, args=''):
    """Returns the URL to a watermarked copy of the image specified."""

    # initialize some variables
    args = args.split(',')
    name = args.pop(0)
    opacity = 0.5
    tile = False
    scale = 1.0
    greyscale = False
    rotation = 0
    position = None
    obscure = OBSCURE
    quality = QUALITY
    random_position_once = RANDOM_POS_ONCE

    url = unquote(url)

    # iterate over all parameters to see what we need to do
    for arg in args:
        key, value = arg.split('=')
        key = key.strip()
        value = value.strip()
        if key == 'position':
            position = value
        elif key == 'opacity':
            opacity = utils._percent(value)
        elif key == 'tile':
            tile = bool(int(value))
        elif key == 'scale':
            scale = value
        elif key == 'greyscale':
            greyscale = bool(int(value))
        elif key == 'rotation':
            rotation = value
        elif key == 'obscure':
            obscure = bool(int(value))
        elif key == 'quality':
            quality = int(value)
        elif key == 'random_position_once':
            random_position_once = bool(int(value))

    mark = Watermarker()
    return mark(url, name, position, opacity, tile, scale, greyscale,
                  rotation, obscure, quality, random_position_once)

register.filter(watermark)
<commit_msg>Python 3 syntax compatible<commit_after>from datetime import datetime
from hashlib import sha1
from PIL import Image
import errno
import logging
import os
import traceback
from urllib import unquote

from django.conf import settings
from django import template
from django.utils import timezone
from watermarker import utils
from watermarker.models import Watermark

register = template.Library()

# determine the quality of the image after the watermark is applied
QUALITY = getattr(settings, 'WATERMARKING_QUALITY', 85)
OBSCURE = getattr(settings, 'WATERMARK_OBSCURE_ORIGINAL', True)
RANDOM_POS_ONCE = getattr(settings, 'WATERMARK_RANDOM_POSITION_ONCE', True)

log = logging.getLogger('watermarker')

class Watermarker(object):

    def __call__(self, url, name, position=None, opacity=0.5, tile=False,
                 scale=1.0, greyscale=False, rotation=0, obscure=OBSCURE,
                 quality=QUALITY, random_position_once=RANDOM_POS_ONCE):
        """
        Creates a watermarked copy of an image.

        * ``name``: This is the name of the Watermark object that you wish to
          apply to the image.
        * ``position``:  There are several options.

            * ``R``: random placement, which is the default behavior.
            * ``C``: center the watermark
            * ``XxY`` where ``X`` is either a specific pixel position on the
              x-axis or a percentage of the total width of the target image and
              ``Y`` is a specific pixel position on the y-axis of the image or
              a percentage of the total height of the target image.  These
              values represent the location of the top and left edges of the
              watermark.  If either ``X`` or ``Y`` is a percentage, you must
              use a percent sign.  This is not used if either one of the
              ``tiled`` or ``scale`` parameters are true.

              Examples:

                * ``50%x50%``: positions the watermark at the center of the
                  image.
                * ``50%x100``: positions the watermark at the midpoint of the
                  total width of the image and 100 pixels from the top of the
                  image
                * ``100x50%``: positions the watermark at the midpoint of the
                  total height of the image and 100 pixels from the left edge
                  of the image
                * ``100x100``: positions the top-left corner of the watermark
                  at 100 pixels from the top of the image and 100 pixels from
                  the left edge of the image.

            * ``br``, ``bl``, ``tr``, ``tl`` where ``b`` means "bottom", ``t``
              means "top", ``l`` means "left", and ``r`` means "right".  This
              will position the watermark at the extreme edge of the original
              image with just enough room for the watermark to "fully show".
              This assumes the watermark is not as big as the original image.

        * ``opacity``: an integer from 0 to 100.  This value represents the
          transparency level of the watermark when it is applied.  A value of
          100 means the watermark is completely opaque while a value of 0 means
          the watermark will be invisible.
        * ``tile``: ``True`` or ``False`` to specify whether or not the
          watermark shall be tiled across the entire image.
        * ``scale``: a floating-point number above 0 to specify the scaling for
          the watermark.  If you want the watermark to be scaled to its maximum
          without falling off the edge of the target image, use ``F``.  By
          default, scale is set to ``1.0``, or 1:1 scaling, meaning the
          watermark will be placed on the target image at its original size.
        * ``greyscale``: ``True`` or ``False`` to specify whether or not the
          watermark should be converted to a greyscale image before applying it
          to the target image.  Default is ``False``.
        * ``rotation``: 0 to 359 to specify the number of degrees to rotate the
          watermark before applying it to the target image.  Alternatively, you
          may set ``rotation=R`` for a random rotation value.
        * ``obscure``: set to ``False`` if you wish to expose the original
          image's filename.  Defaults to ``True``.
        * ``quality``: the quality of the resulting watermarked image.  Default
          is 85.
        """

        # look for the specified watermark by name.  If it's not there, go no
        # further
        try:
            watermark = Watermark.objects.get(name=name, is_active=True)
        except Watermark.DoesNotExist:
            log.error('Watermark "%s" does not exist... Bailing out.' % name)
            return url

        # make sure URL is a string
        url = str(url)

        basedir = '%s/watermarked' % os.path.dirname(url)
        base, ext = os.path.splitext(os.path.basename(url))

        # open the target image file along with the watermark image
        target_path = self.get_url_path(url)
        target = Image.open(target_path)
        mark = Image.open(watermark.image.path)

        # determine the actual value that the parameters provided will render
        random_position = bool(position is None or str(position).lower() == 'r')
        scale = utils.determine_scale(scale, target, mark)
        mark = mark.resize(scale, resample=Image.ANTIALIAS)
        rotation = utils.determine_rotation(rotation, mark)
        pos = utils.determine_position(position, target, mark)

        # see if we need to create only one randomly positioned watermarked
        # image
        if not random_position or \
            (not random_position_once and random_position):
            log.debug('Generating random position for watermark each time')
            position = pos
        else:
            log.debug('Random positioning watermark once')

        params = {
            'position':  position,
            'opacity':   opacity,
            'scale':     scale,
            'tile':      tile,
            'greyscale': greyscale,
            'rotation':  rotation,
            'base':      base,
            'ext':       ext,
            'quality':   quality,
            'watermark': watermark.id,
            'opacity_int': int(opacity * 100),
            'left':      pos[0],
            'top':       pos[1],
        }
        log.debug('Params: %s' % params)

        wm_name = self.watermark_name(mark, **params)
        wm_url = self.watermark_path(basedir, base, ext, wm_name, obscure)
        wm_path = self.get_url_path(wm_url)
        log.debug('Watermark name: %s; URL: %s; Path: %s' % (
            wm_name, wm_url, wm_path
        ))

        # see if the image already exists on the filesystem.  If it does, use
        # it.
        if os.access(wm_path, os.R_OK):
            # see if the Watermark object was modified since the file was
            # created
            modified = datetime.fromtimestamp(os.path.getmtime(wm_path))
            # Django 1.4+ timezone support
            modified = timezone.make_aware(modified, timezone.get_default_timezone())

            # only return the old file if things appear to be the same
            if modified >= watermark.date_updated:
                log.info('Watermark exists and has not changed.  Bailing out.')
                return wm_url

        # make sure the position is in our params for the watermark
        params['position'] = pos

        self.create_watermark(target, mark, wm_path, **params)

        # send back the URL to the new, watermarked image
        return wm_url

    def get_url_path(self, url, root=settings.MEDIA_ROOT,
        url_root=settings.MEDIA_URL):
        """Makes a filesystem path from the specified URL"""

        if url.startswith(url_root):
            url = url[len(url_root):] # strip media root url

        return os.path.normpath(os.path.join(root, url))

    def watermark_name(self, mark, **kwargs):
        """Comes up with a good filename for the watermarked image"""

        params = [
            '%(base)s',
            'wm',
            'w%(watermark)i',
            'o%(opacity_int)i',
            'gs%(greyscale)i',
            'r%(rotation)i',
            '_p%(position)s',
        ]

        scale = kwargs.get('scale', None)
        if scale and scale != mark.size:
            params.append('_s%i' % (float(kwargs['scale'][0]) / mark.size[0] * 100))

        if kwargs.get('tile', None):
            params.append('_tiled')

        # make thumbnail filename
        name = '%s%s' % ('_'.join(params), kwargs['ext'])
        return name % kwargs

    def watermark_path(self, basedir, base, ext, wm_name, obscure=True):
        """Determines an appropriate watermark path"""

        hash = sha1(wm_name).hexdigest()

        # figure out where the watermark would be saved on the filesystem
        if obscure:
            log.debug('Obscuring original image name: %s => %s' % (wm_name, hash))
            new_file = os.path.join(basedir, hash + ext)
        else:
            log.debug('Not obscuring original image name.')
            new_file = os.path.join(basedir, hash, base + ext)

        # make sure the destination directory exists
        try:
            root = self.get_url_path(new_file)
            os.makedirs(os.path.dirname(root))
        except OSError as exc:
            if exc.errno == errno.EEXIST:
                # not to worry, directory exists
                pass
            else:
                log.error('Error creating path: %s' % traceback.format_exc())
                raise
        else:
            log.debug('Created directory: %s' % root)

        return new_file

    def create_watermark(self, target, mark, path, quality=QUALITY, **kwargs):
        """Create the watermarked image on the filesystem"""

        im = utils.watermark(target, mark, **kwargs)
        im.save(path, quality=quality)

        return im

def watermark(url, args=''):
    """Returns the URL to a watermarked copy of the image specified."""

    # initialize some variables
    args = args.split(',')
    name = args.pop(0)
    opacity = 0.5
    tile = False
    scale = 1.0
    greyscale = False
    rotation = 0
    position = None
    obscure = OBSCURE
    quality = QUALITY
    random_position_once = RANDOM_POS_ONCE

    url = unquote(url)

    # iterate over all parameters to see what we need to do
    for arg in args:
        key, value = arg.split('=')
        key = key.strip()
        value = value.strip()
        if key == 'position':
            position = value
        elif key == 'opacity':
            opacity = utils._percent(value)
        elif key == 'tile':
            tile = bool(int(value))
        elif key == 'scale':
            scale = value
        elif key == 'greyscale':
            greyscale = bool(int(value))
        elif key == 'rotation':
            rotation = value
        elif key == 'obscure':
            obscure = bool(int(value))
        elif key == 'quality':
            quality = int(value)
        elif key == 'random_position_once':
            random_position_once = bool(int(value))

    mark = Watermarker()
    return mark(url, name, position, opacity, tile, scale, greyscale,
                  rotation, obscure, quality, random_position_once)

register.filter(watermark)
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Hive Appier Framework
# Copyright (c) 2008-2017 Hive Solutions Lda.
#
# This file is part of Hive Appier Framework.
#
# Hive Appier Framework is free software: you can redistribute it and/or modify
# it under the terms of the Apache License as published by the Apache
# Foundation, either version 2.0 of the License, or (at your option) any
# later version.
#
# Hive Appier Framework is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Apache License for more details.
#
# You should have received a copy of the Apache License along with
# Hive Appier Framework. If not, see <http://www.apache.org/licenses/>.

__author__ = "João Magalhães <joamag@hive.pt>"
""" The author(s) of the module """

__version__ = "1.0.0"
""" The version of the module """

__revision__ = "$LastChangedRevision$"
""" The revision number of the module """

__date__ = "$LastChangedDate$"
""" The last change date of the module """

__copyright__ = "Copyright (c) 2008-2017 Hive Solutions Lda."
""" The copyright for the module """

__license__ = "Apache License, Version 2.0"
""" The license for the module """

import sys
import threading

from . import config
from . import legacy
from . import exceptions

class AsyncManager(object):

    def __init__(self, owner):
        object.__init__(self)
        self.owner = owner

    def start(self):
        pass

    def stop(self):
        pass

    def add(self, method, args, kwargs, request = None, mid = None):
        pass

class SimpleManager(AsyncManager):

    def add(self, method, args, kwargs, request = None, mid = None):
        if request: kwargs["request"] = request
        if mid: kwargs["mid"] = mid
        thread = threading.Thread(
            target = method,
            args = args,
            kwargs = kwargs
        )
        thread.start()

class QueueManager(AsyncManager):

    def __init__(self, owner):
        AsyncManager.__init__(self, owner)
        self.queue = []
        self.condition = threading.Condition()

    def start(self):
        self.thread = threading.Thread(target = self.handler)
        self.running = True
        self.thread.daemon = True
        self.thread.start()

    def stop(self):
        self.running = False

    def add(self, method, args, kwargs, request = None, mid = None):
        if request: kwargs["request"] = request
        if mid: kwargs["mid"] = mid
        item = (method, args, kwargs)
        self.condition.acquire()
        try:
            self.queue.append(item)
            self.condition.notify()
        finally:
            self.condition.release()

    def handler(self):
        while self.running:
            self.condition.acquire()
            while not self.queue: self.condition.wait()
            try: item = self.queue.pop(0)
            finally: self.condition.release()
            method, args, kwargs = item
            try: method(*args, **kwargs)
            except BaseException as exception:
                self.owner.log_error(
                    exception,
                    message = "Problem handling async item: %s"
                )

class AwaitWrapper(object):
    pass

class AyncWrapper(object):
    pass

class CoroutineWrapper(object):
    pass

def await_wrap(generator):
    return generator

def await_yield(value):
    yield value

def try_generator(value):
    is_generator = legacy.is_generator(value)
    if is_generator: return True, value
    return False, value

def to_coroutine(callable, *args, **kwargs):
    """
    Converts the provided (callback based) callable into a coroutine
    like object/function that is able to yield a future to be notified
    by a wrapper around the "original" callback.

    The parameters to be used in the calling should be passed as list
    and keyword based arguments.

    :type callable: Callable
    :param callable: The callable that is going to be converted into
    a coroutine.
    :rtype: Future
    :return: The future object that is going to be used in the control
    of the coroutine execution.
    """

    # tries to retrieve both the future and a callback from
    # the provided key based arguments in case thre's no future
    # a new one is created for the current context as for the
    # callback an unset one is used for invalid situations
    future = kwargs.pop("future", None) or Future()
    callback = kwargs.get("callback", None)

    def callback_wrap(result, *args, **kwargs):
        # sets the final result in the associated future
        # this should contain the contents coming from
        # the callback operation (payload)
        future.set_result(result)

        # in case the "original" callback is set calls it
        # with the provided arguments as expected
        callback and callback(result, *args, **kwargs)

        # runs the wake up operation so that the main loop
        # associated with the current execution environment
        # is able to process the new future result
        wakeup()

    # sets the wrapped callback in the key based arguments and
    # then runs the callback with the new arguments, yielding
    # the future that has just been created, effectively creating
    # a coroutine interface from a callback one
    kwargs["callback"] = callback_wrap
    callable(*args, **kwargs)
    yield future

def unavailable(*args, **kwargs):
    raise exceptions.AppierException(
        message = "No support for async available"
    )

# determines the target service configuration that is
# going to be used, this is going to be used to create
# the proper adaptation for the async models
server = config.conf("SERVER", None)
if server == "netius":
    import netius
    Future = netius.Future
    ensure_async = netius.ensure
    coroutine = netius.coroutine
    wakeup = netius.wakeup
    sleep = netius.sleep
    wait = netius.wait
    notify = netius.notify
else:
    Future = unavailable
    ensure_async = unavailable
    coroutine = unavailable
    wakeup = unavailable
    sleep = unavailable
    wait = unavailable
    notify = unavailable

is_neo = sys.version_info[0] >= 3 and sys.version_info[1] >= 3
if is_neo: from .async_neo import * #@UnusedWildImport

def _header_a():
    yield -1

def header_a():
    return await_wrap(_header_a())

def _ensure_a(*args, **kwargs):
    yield ensure_async(*args, **kwargs)

def ensure_a(*args, **kwargs):
    return await_wrap(_ensure_a(*args, **kwargs))
<commit_msg>small silent removal<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Hive Appier Framework
# Copyright (c) 2008-2017 Hive Solutions Lda.
#
# This file is part of Hive Appier Framework.
#
# Hive Appier Framework is free software: you can redistribute it and/or modify
# it under the terms of the Apache License as published by the Apache
# Foundation, either version 2.0 of the License, or (at your option) any
# later version.
#
# Hive Appier Framework is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Apache License for more details.
#
# You should have received a copy of the Apache License along with
# Hive Appier Framework. If not, see <http://www.apache.org/licenses/>.

__author__ = "João Magalhães <joamag@hive.pt>"
""" The author(s) of the module """

__version__ = "1.0.0"
""" The version of the module """

__revision__ = "$LastChangedRevision$"
""" The revision number of the module """

__date__ = "$LastChangedDate$"
""" The last change date of the module """

__copyright__ = "Copyright (c) 2008-2017 Hive Solutions Lda."
""" The copyright for the module """

__license__ = "Apache License, Version 2.0"
""" The license for the module """

import sys
import threading

from . import config
from . import legacy
from . import exceptions

class AsyncManager(object):

    def __init__(self, owner):
        object.__init__(self)
        self.owner = owner

    def start(self):
        pass

    def stop(self):
        pass

    def add(self, method, args, kwargs, request = None, mid = None):
        pass

class SimpleManager(AsyncManager):

    def add(self, method, args, kwargs, request = None, mid = None):
        if request: kwargs["request"] = request
        if mid: kwargs["mid"] = mid
        thread = threading.Thread(
            target = method,
            args = args,
            kwargs = kwargs
        )
        thread.start()

class QueueManager(AsyncManager):

    def __init__(self, owner):
        AsyncManager.__init__(self, owner)
        self.queue = []
        self.condition = threading.Condition()

    def start(self):
        self.thread = threading.Thread(target = self.handler)
        self.running = True
        self.thread.daemon = True
        self.thread.start()

    def stop(self):
        self.running = False

    def add(self, method, args, kwargs, request = None, mid = None):
        if request: kwargs["request"] = request
        if mid: kwargs["mid"] = mid
        item = (method, args, kwargs)
        self.condition.acquire()
        try:
            self.queue.append(item)
            self.condition.notify()
        finally:
            self.condition.release()

    def handler(self):
        while self.running:
            self.condition.acquire()
            while not self.queue: self.condition.wait()
            try: item = self.queue.pop(0)
            finally: self.condition.release()
            method, args, kwargs = item
            try: method(*args, **kwargs)
            except BaseException as exception:
                self.owner.log_error(
                    exception,
                    message = "Problem handling async item: %s"
                )

class AwaitWrapper(object):
    pass

class AyncWrapper(object):
    pass

class CoroutineWrapper(object):
    pass

def await_wrap(generator):
    return generator

def await_yield(value):
    yield value

def try_generator(value):
    is_generator = legacy.is_generator(value)
    if is_generator: return True, value
    return False, value

def to_coroutine(callable, *args, **kwargs):
    """
    Converts the provided (callback based) callable into a coroutine
    like object/function that is able to yield a future to be notified
    by a wrapper around the "original" callback.

    The parameters to be used in the calling should be passed as list
    and keyword based arguments.

    :type callable: Callable
    :param callable: The callable that is going to be converted into
    a coroutine.
    :rtype: Future
    :return: The future object that is going to be used in the control
    of the coroutine execution.
    """

    # tries to retrieve both the future and a callback from
    # the provided key based arguments in case thre's no future
    # a new one is created for the current context as for the
    # callback an unset one is used for invalid situations
    future = kwargs.pop("future", None) or Future()
    callback = kwargs.get("callback", None)

    def callback_wrap(result, *args, **kwargs):
        # sets the final result in the associated future
        # this should contain the contents coming from
        # the callback operation (payload)
        future.set_result(result)

        # in case the "original" callback is set calls it
        # with the provided arguments as expected
        callback and callback(result, *args, **kwargs)

        # runs the wake up operation so that the main loop
        # associated with the current execution environment
        # is able to process the new future result
        wakeup()

    # sets the wrapped callback in the key based arguments and
    # then runs the callback with the new arguments, yielding
    # the future that has just been created, effectively creating
    # a coroutine interface from a callback one
    kwargs["callback"] = callback_wrap
    callable(*args, **kwargs)
    yield future

def wrap_silent(function):
    return function

def unavailable(*args, **kwargs):
    raise exceptions.AppierException(
        message = "No support for async available"
    )

# determines the target service configuration that is
# going to be used, this is going to be used to create
# the proper adaptation for the async models
server = config.conf("SERVER", None)
if server == "netius":
    import netius
    Future = netius.Future
    coroutine = netius.coroutine
    ensure_async = netius.ensure
    wakeup = netius.wakeup
    sleep = netius.sleep
    wait = netius.wait
    notify = netius.notify
else:
    Future = unavailable
    coroutine = wrap_silent
    ensure_async = unavailable
    wakeup = unavailable
    sleep = unavailable
    wait = unavailable
    notify = unavailable

is_neo = sys.version_info[0] >= 3 and sys.version_info[1] >= 3
if is_neo: from .async_neo import * #@UnusedWildImport

def _header_a():
    yield -1

def header_a():
    return await_wrap(_header_a())

def _ensure_a(*args, **kwargs):
    yield ensure_async(*args, **kwargs)

def ensure_a(*args, **kwargs):
    return await_wrap(_ensure_a(*args, **kwargs))
<|endoftext|>