<commit_before>"""
General matchers.

These don't belong to any broader category, and include matchers for common
Python objects, like functions or classes.
"""
try:
    import asyncio
except ImportError:
    asyncio = None
import inspect
from itertools import starmap
from operator import itemgetter

from callee._compat import IS_PY3, STRING_TYPES
from callee.base import BaseMatcher, Eq

__all__ = [
    'Any', 'Matching', 'ArgThat', 'Captor',
    'Callable', 'Function', 'GeneratorFunction',
    'Coroutine', 'CoroutineFunction',
    'InstanceOf', 'IsA', 'SubclassOf', 'Inherits', 'Type', 'Class',
    'Attrs', 'Attr', 'HasAttrs', 'HasAttr',
]

# TODO: introduce custom exception types rather than using built-ins

class Any(BaseMatcher):
    """Matches any object."""

    def match(self, value):
        return True

    def __repr__(self):
        return "<Any>"

class Matching(BaseMatcher):
    """Matches an object that satisfies given predicate."""

    MAX_DESC_LENGTH = 32

    def __init__(self, predicate, desc=None):
        """
        :param predicate: Callable taking a single argument
                          and returning True or False
        :param desc: Optional description of the predicate.
                     This will be displayed as a part of the error message
                     on failed assertion.
        """
        if not callable(predicate):
            raise TypeError(
                "Matching requires a predicate, got %r" % (predicate,))

        self.predicate = predicate
        self.desc = self._validate_desc(desc)

    def _validate_desc(self, desc):
        """Validate the predicate description."""
        if desc is None:
            return desc

        if not isinstance(desc, STRING_TYPES):
            raise TypeError(
                "predicate description for Matching must be a string, "
                "got %r" % (type(desc),))

        # Python 2 mandates __repr__ to be an ASCII string,
        # so if Unicode is passed (usually due to unicode_literals),
        # it should be ASCII-encodable.
        if not IS_PY3 and isinstance(desc, unicode):
            try:
                desc = desc.encode('ascii', errors='strict')
            except UnicodeEncodeError:
                raise TypeError("predicate description must be "
                                "an ASCII string in Python 2")

        return desc

    def match(self, value):
        # Note that any possible exceptions from ``predicate``
        # are intentionally let through, to make it easier to diagnose errors
        # than a plain "no match" response would.
        return bool(self.predicate(value))
        # TODO: translate exceptions from the predicate into our own
        # exception type to not clutter user-visible stracktraces with our code

    def __repr__(self):
        """Return a representation of the matcher."""
        name = getattr(self.predicate, '__name__', None)
        desc = self.desc

        # When no user-provided description is available,
        # use function's own name or even its repr().
        if desc is None:
            # If not a lambda function, we can probably make the representation
            # more readable by showing just the function's own name.
            if name and name != '<lambda>':
                # Where possible, make it a fully qualified name, including
                # the module path. This is either on Python 3.3+
                # (via __qualname__), or when the predicate is
                # a standalone function (not a method).
                qualname = getattr(self.predicate, '__qualname__', name)
                is_method = inspect.ismethod(self.predicate) or \
                    isinstance(self.predicate, staticmethod)
                if qualname != name or not is_method:
                    # Note that this shows inner functions (those defined
                    # locally inside other functions) as if they were global
                    # to the module.
                    # This is why we use colon (:) as separator here, as to not
                    # suggest this is an evaluatable identifier.
                    name = '%s:%s' % (self.predicate.__module__, qualname)
            else:
                # For lambdas and other callable objects,
                # we'll just default to the Python repr().
                name = None
        else:
            # Quote and possibly ellipsize the provided description.
            if len(desc) > self.MAX_DESC_LENGTH:
                ellipsis = '...'
                desc = desc[:self.MAX_DESC_LENGTH - len(ellipsis)] + ellipsis
            desc = '"%s"' % desc

        return "<Matching %s>" % (desc or name or repr(self.predicate))

ArgThat = Matching

class Captor(BaseMatcher):
    """Argument captor.

    You can use :class:`Captor` to "capture" the original argument
    that the mock was called with, and perform custom assertions on it.

    Example::

        captor = Captor()
        mock_foo.assert_called_with(captor)

        # captured value is available as `arg` attribute
        self.assertEquals(captor.arg.some_method(), 42)
        self.assertEquals(captor.arg.some_other_method(), "foo")

    .. versionadded:: 0.2
    """
    __slots__ = ('matcher', 'value')

    def __init__(self, matcher=None):
        """
        :param matcher: Optional matcher to validate the argument against
                        before it's captured
        """
        if matcher is None:
            matcher = Any()

        if not isinstance(matcher, BaseMatcher):
            raise TypeError("expected a matcher, got %r" % (type(matcher),))
        if isinstance(matcher, Captor):
            raise TypeError("cannot pass a captor to another captor")

        self.matcher = matcher

    def has_value(self):
        """Returns whether the :class:`Captor` has captured a value."""
        return hasattr(self, 'value')

    @property
    def arg(self):
        """The captured argument value."""
        if not self.has_value():
            raise ValueError("no value captured")
        return self.value

    def match(self, value):
        if self.has_value():
            raise ValueError("a value has already been captured")

        if not self.matcher.match(value):
            return False
        self.value = value
        return True

    def __repr__(self):
        """Return a representation of the captor."""
        return "<Captor %r%s>" % (self.matcher,
                                  " (*)" if self.has_value() else "")

# Function-related matchers

class FunctionMatcher(BaseMatcher):
    """Matches values of callable types.
    This class shouldn't be used directly.
    """
    def __repr__(self):
        return "<%s>" % (self.__class__.__name__,)

class Callable(FunctionMatcher):
    """Matches any callable object (as per the :func:`callable` function)."""

    def match(self, value):
        return callable(value)

class Function(FunctionMatcher):
    """Matches any Python function."""

    def match(self, value):
        return inspect.isfunction(value)

class GeneratorFunction(FunctionMatcher):
    """Matches a generator function, i.e. one that uses ``yield`` in its body.

    .. note::

        This is distinct from matching a *generator*,
        i.e. an iterable result of calling the generator function,
        or a generator comprehension (``(... for x in ...)``).
        The :class:`~callee.collections.Generator` matcher
        should be used for those objects instead.
    """
    def match(self, value):
        return inspect.isgeneratorfunction(value)

# TODO: this needs a better place, since it's basically what Generator is
# to GeneratorFunction; perhaps we need a dedicated module for matchers for
# async-related objects (coroutines, futures, tasks, etc.)
class Coroutine(FunctionMatcher):
    """Matches an asynchronous coroutine.

    A coroutine is a result of an asynchronous function call, where the async
    function has been defined using ``@asyncio.coroutine`` or the ``async def``
    syntax.

    These are only available in Python 3.4 and above.
    On previous versions of Python, no object will match this matcher.
    """
    def match(self, value):
        return asyncio and asyncio.iscoroutine(value)

class CoroutineFunction(FunctionMatcher):
    """Matches a coroutine function.

    A coroutine function is an asynchronous function defined using the
    ``@asyncio.coroutine`` or the ``async def`` syntax.

    These are only available in Python 3.4 and above.
    On previous versions of Python, no object will match this matcher.
    """
    def match(self, value):
        return asyncio and asyncio.iscoroutinefunction(value)

# Type-related matchers

class TypeMatcher(BaseMatcher):
    """Matches an object to a type.
    This class shouldn't be used directly.
    """
    def __init__(self, type_):
        """:param type\ _: Type to match against"""
        if not isinstance(type_, type):
            raise TypeError("%s requires a type, got %r" % (
                self.__class__.__name__, type_))
        self.type_ = type_

    def __repr__(self):
        return "<%s %r>" % (self.__class__.__name__, self.type_)

class InstanceOf(TypeMatcher):
    """Matches an object that's an instance of given type
    (as per `isinstance`).
    """
    def match(self, value):
        return isinstance(value, self.type_)

IsA = InstanceOf

# TODO: reverse of this matcher (SuperclassOf)
class SubclassOf(TypeMatcher):
    """Matches a class that's a subclass of given type
    (as per `issubclass`).
    """
    def __init__(self, type_):
        """:param type\ _: Type to match against"""
        # TODO: strict= argument
        super(SubclassOf, self).__init__(type_)

    def match(self, value):
        return isinstance(value, type) and issubclass(value, self.type_)

Inherits = SubclassOf

class Type(BaseMatcher):
    """Matches any Python type object."""

    def match(self, value):
        return isinstance(value, type)

    def __repr__(self):
        return "<Type>"

class Class(BaseMatcher):
    """Matches a class (but not any other type object)."""

    def match(self, value):
        return inspect.isclass(value)

    def __repr__(self):
        return "<Class>"

# TODO: Module() matcher

# Attribute-based matchers

class Attrs(BaseMatcher):
    """Matches objects based on their attributes.

    To match successfully, the object needs to:

        * have all the attributes whose names were passed
          as positional arguments (regardless of their values)
        * have the attribute names/values that correspond exactly
          to keyword arguments' names and values

    Examples::

        Attrs('foo')  # `foo` attribute with any value
        Attrs('foo', 'bar')  # `foo` and `bar` attributes with any values
        Attrs(foo=42)  # `foo` attribute with value of 42
        Attrs(bar=Integer())  # `bar` attribute whose value is an integer
        Attrs('foo', bar='x')  # `foo` with any value, `bar` with value of 'x'
    """
    def __init__(self, *args, **kwargs):
        if not (args or kwargs):
            raise TypeError("%s() requires at least one argument" % (
                self.__class__.__name__,))

        self.attr_names = list(args)
        self.attr_dict = dict((k, v if isinstance(v, BaseMatcher) else Eq(v))
                              for k, v in kwargs.items())

    def match(self, value):
        for name in self.attr_names:
            # Can't use hasattr() here because it swallows *all* exceptions
            # from attribute access in Python 2.x, not just AttributeError.
            # More details: https://hynek.me/articles/hasattr/
            try:
                getattr(value, name)
            except AttributeError:
                return False

        for name, matcher in self.attr_dict.items():
            # Separately handle retrieving of the attribute value,
            # so that any stray AttributeErrors from the matcher itself
            # are correctly propagated.
            try:
                attrvalue = getattr(value, name)
            except AttributeError:
                return False
            if not matcher.match(attrvalue):
                return False

        return True

    def __repr__(self):
        """Return a representation of the matcher."""
        # get both the names-only and valued attributes and sort them by name
        sentinel = object()
        attrs = [(name, sentinel)
                 for name in self.attr_names] + list(self.attr_dict.items())
        attrs.sort(key=itemgetter(0))

        def attr_repr(name, value):
            # include the value with attribute name whenever necessary
            if value is sentinel:
                return name
            value = value.value if isinstance(value, Eq) else value
            return "%s=%r" % (name, value)

        return "<%s %s>" % (self.__class__.__name__,
                            " ".join(starmap(attr_repr, attrs)))

class Attr(Attrs):
    """Matches objects that have an attribute with given name and value,
    as given by a keyword argument.
    """
    def __init__(self, **kwargs):
        if not len(kwargs) == 1:
            raise TypeError("Attr() requires exactly one keyword argument")
        super(Attr, self).__init__(**kwargs)

class HasAttrs(Attrs):
    """Matches objects that have all of the specified attribute names,
    regardless of their values.
    """
    def __init__(self, *args):
        super(HasAttrs, self).__init__(*args)

class HasAttr(HasAttrs):
    """Matches object that have an attribute with given name,
    regardless of its value.
    """
    def __init__(self, name):
        super(HasAttr, self).__init__(name)
<commit_msg>Some TODOs<commit_after>"""
General matchers.

These don't belong to any broader category, and include matchers for common
Python objects, like functions or classes.
"""
try:
    import asyncio
except ImportError:
    asyncio = None
import inspect
from itertools import starmap
from operator import itemgetter

from callee._compat import IS_PY3, STRING_TYPES
from callee.base import BaseMatcher, Eq

__all__ = [
    'Any', 'Matching', 'ArgThat', 'Captor',
    'Callable', 'Function', 'GeneratorFunction',
    'Coroutine', 'CoroutineFunction',
    'InstanceOf', 'IsA', 'SubclassOf', 'Inherits', 'Type', 'Class',
    'Attrs', 'Attr', 'HasAttrs', 'HasAttr',
]

# TODO: introduce custom exception types rather than using built-ins

class Any(BaseMatcher):
    """Matches any object."""

    def match(self, value):
        return True

    def __repr__(self):
        return "<Any>"

class Matching(BaseMatcher):
    """Matches an object that satisfies given predicate."""

    MAX_DESC_LENGTH = 32

    def __init__(self, predicate, desc=None):
        """
        :param predicate: Callable taking a single argument
                          and returning True or False
        :param desc: Optional description of the predicate.
                     This will be displayed as a part of the error message
                     on failed assertion.
        """
        if not callable(predicate):
            raise TypeError(
                "Matching requires a predicate, got %r" % (predicate,))

        self.predicate = predicate
        self.desc = self._validate_desc(desc)

    def _validate_desc(self, desc):
        """Validate the predicate description."""
        if desc is None:
            return desc

        if not isinstance(desc, STRING_TYPES):
            raise TypeError(
                "predicate description for Matching must be a string, "
                "got %r" % (type(desc),))

        # Python 2 mandates __repr__ to be an ASCII string,
        # so if Unicode is passed (usually due to unicode_literals),
        # it should be ASCII-encodable.
        if not IS_PY3 and isinstance(desc, unicode):
            try:
                desc = desc.encode('ascii', errors='strict')
            except UnicodeEncodeError:
                raise TypeError("predicate description must be "
                                "an ASCII string in Python 2")

        return desc

    def match(self, value):
        # Note that any possible exceptions from ``predicate``
        # are intentionally let through, to make it easier to diagnose errors
        # than a plain "no match" response would.
        return bool(self.predicate(value))
        # TODO: translate exceptions from the predicate into our own
        # exception type to not clutter user-visible stracktraces with our code

    def __repr__(self):
        """Return a representation of the matcher."""
        name = getattr(self.predicate, '__name__', None)
        desc = self.desc

        # When no user-provided description is available,
        # use function's own name or even its repr().
        if desc is None:
            # If not a lambda function, we can probably make the representation
            # more readable by showing just the function's own name.
            if name and name != '<lambda>':
                # Where possible, make it a fully qualified name, including
                # the module path. This is either on Python 3.3+
                # (via __qualname__), or when the predicate is
                # a standalone function (not a method).
                qualname = getattr(self.predicate, '__qualname__', name)
                is_method = inspect.ismethod(self.predicate) or \
                    isinstance(self.predicate, staticmethod)
                if qualname != name or not is_method:
                    # Note that this shows inner functions (those defined
                    # locally inside other functions) as if they were global
                    # to the module.
                    # This is why we use colon (:) as separator here, as to not
                    # suggest this is an evaluatable identifier.
                    name = '%s:%s' % (self.predicate.__module__, qualname)
            else:
                # For lambdas and other callable objects,
                # we'll just default to the Python repr().
                name = None
        else:
            # Quote and possibly ellipsize the provided description.
            if len(desc) > self.MAX_DESC_LENGTH:
                ellipsis = '...'
                desc = desc[:self.MAX_DESC_LENGTH - len(ellipsis)] + ellipsis
            desc = '"%s"' % desc

        return "<Matching %s>" % (desc or name or repr(self.predicate))

ArgThat = Matching

class Captor(BaseMatcher):
    """Argument captor.

    You can use :class:`Captor` to "capture" the original argument
    that the mock was called with, and perform custom assertions on it.

    Example::

        captor = Captor()
        mock_foo.assert_called_with(captor)

        # captured value is available as `arg` attribute
        self.assertEquals(captor.arg.some_method(), 42)
        self.assertEquals(captor.arg.some_other_method(), "foo")

    .. versionadded:: 0.2
    """
    __slots__ = ('matcher', 'value')

    def __init__(self, matcher=None):
        """
        :param matcher: Optional matcher to validate the argument against
                        before it's captured
        """
        if matcher is None:
            matcher = Any()

        if not isinstance(matcher, BaseMatcher):
            raise TypeError("expected a matcher, got %r" % (type(matcher),))
        if isinstance(matcher, Captor):
            raise TypeError("cannot pass a captor to another captor")

        self.matcher = matcher

    def has_value(self):
        """Returns whether the :class:`Captor` has captured a value."""
        return hasattr(self, 'value')

    @property
    def arg(self):
        """The captured argument value."""
        if not self.has_value():
            raise ValueError("no value captured")
        return self.value

    def match(self, value):
        if self.has_value():
            raise ValueError("a value has already been captured")

        if not self.matcher.match(value):
            return False
        self.value = value
        return True

    def __repr__(self):
        """Return a representation of the captor."""
        return "<Captor %r%s>" % (self.matcher,
                                  " (*)" if self.has_value() else "")

# Function-related matchers

class FunctionMatcher(BaseMatcher):
    """Matches values of callable types.
    This class shouldn't be used directly.
    """
    def __repr__(self):
        return "<%s>" % (self.__class__.__name__,)

class Callable(FunctionMatcher):
    """Matches any callable object (as per the :func:`callable` function)."""

    def match(self, value):
        return callable(value)

class Function(FunctionMatcher):
    """Matches any Python function."""

    def match(self, value):
        return inspect.isfunction(value)

class GeneratorFunction(FunctionMatcher):
    """Matches a generator function, i.e. one that uses ``yield`` in its body.

    .. note::

        This is distinct from matching a *generator*,
        i.e. an iterable result of calling the generator function,
        or a generator comprehension (``(... for x in ...)``).
        The :class:`~callee.collections.Generator` matcher
        should be used for those objects instead.
    """
    def match(self, value):
        return inspect.isgeneratorfunction(value)

# TODO: this needs a better place, since it's basically what Generator is
# to GeneratorFunction; perhaps we need a dedicated module for matchers for
# async-related objects (coroutines, futures, tasks, etc.)
class Coroutine(FunctionMatcher):
    """Matches an asynchronous coroutine.

    A coroutine is a result of an asynchronous function call, where the async
    function has been defined using ``@asyncio.coroutine`` or the ``async def``
    syntax.

    These are only available in Python 3.4 and above.
    On previous versions of Python, no object will match this matcher.
    """
    def match(self, value):
        return asyncio and asyncio.iscoroutine(value)

class CoroutineFunction(FunctionMatcher):
    """Matches a coroutine function.

    A coroutine function is an asynchronous function defined using the
    ``@asyncio.coroutine`` or the ``async def`` syntax.

    These are only available in Python 3.4 and above.
    On previous versions of Python, no object will match this matcher.
    """
    def match(self, value):
        return asyncio and asyncio.iscoroutinefunction(value)

# Type-related matchers

class TypeMatcher(BaseMatcher):
    """Matches an object to a type.
    This class shouldn't be used directly.
    """
    def __init__(self, type_):
        """:param type\ _: Type to match against"""
        if not isinstance(type_, type):
            raise TypeError("%s requires a type, got %r" % (
                self.__class__.__name__, type_))
        self.type_ = type_

    def __repr__(self):
        return "<%s %r>" % (self.__class__.__name__, self.type_)

# TODO: reverse of this matcher (TypeOf / ClassOf)
class InstanceOf(TypeMatcher):
    """Matches an object that's an instance of given type
    (as per `isinstance`).
    """
    def __init__(self, type_):
        """:param type\ _; Type to match against"""
        # TODO: strict= argument
        super(InstanceOf, self).__init__(type_)

    def match(self, value):
        return isinstance(value, self.type_)

IsA = InstanceOf

# TODO: reverse of this matcher (SuperclassOf)
class SubclassOf(TypeMatcher):
    """Matches a class that's a subclass of given type
    (as per `issubclass`).
    """
    def __init__(self, type_):
        """:param type\ _: Type to match against"""
        # TODO: strict= argument
        super(SubclassOf, self).__init__(type_)

    def match(self, value):
        return isinstance(value, type) and issubclass(value, self.type_)

Inherits = SubclassOf

class Type(BaseMatcher):
    """Matches any Python type object."""

    def match(self, value):
        return isinstance(value, type)

    def __repr__(self):
        return "<Type>"

class Class(BaseMatcher):
    """Matches a class (but not any other type object)."""

    def match(self, value):
        return inspect.isclass(value)

    def __repr__(self):
        return "<Class>"

# TODO: Module() matcher

# Attribute-based matchers

class Attrs(BaseMatcher):
    """Matches objects based on their attributes.

    To match successfully, the object needs to:

        * have all the attributes whose names were passed
          as positional arguments (regardless of their values)
        * have the attribute names/values that correspond exactly
          to keyword arguments' names and values

    Examples::

        Attrs('foo')  # `foo` attribute with any value
        Attrs('foo', 'bar')  # `foo` and `bar` attributes with any values
        Attrs(foo=42)  # `foo` attribute with value of 42
        Attrs(bar=Integer())  # `bar` attribute whose value is an integer
        Attrs('foo', bar='x')  # `foo` with any value, `bar` with value of 'x'
    """
    def __init__(self, *args, **kwargs):
        if not (args or kwargs):
            raise TypeError("%s() requires at least one argument" % (
                self.__class__.__name__,))

        self.attr_names = list(args)
        self.attr_dict = dict((k, v if isinstance(v, BaseMatcher) else Eq(v))
                              for k, v in kwargs.items())

    def match(self, value):
        for name in self.attr_names:
            # Can't use hasattr() here because it swallows *all* exceptions
            # from attribute access in Python 2.x, not just AttributeError.
            # More details: https://hynek.me/articles/hasattr/
            try:
                getattr(value, name)
            except AttributeError:
                return False

        for name, matcher in self.attr_dict.items():
            # Separately handle retrieving of the attribute value,
            # so that any stray AttributeErrors from the matcher itself
            # are correctly propagated.
            try:
                attrvalue = getattr(value, name)
            except AttributeError:
                return False
            if not matcher.match(attrvalue):
                return False

        return True

    def __repr__(self):
        """Return a representation of the matcher."""
        # get both the names-only and valued attributes and sort them by name
        sentinel = object()
        attrs = [(name, sentinel)
                 for name in self.attr_names] + list(self.attr_dict.items())
        attrs.sort(key=itemgetter(0))

        def attr_repr(name, value):
            # include the value with attribute name whenever necessary
            if value is sentinel:
                return name
            value = value.value if isinstance(value, Eq) else value
            return "%s=%r" % (name, value)

        return "<%s %s>" % (self.__class__.__name__,
                            " ".join(starmap(attr_repr, attrs)))

class Attr(Attrs):
    """Matches objects that have an attribute with given name and value,
    as given by a keyword argument.
    """
    def __init__(self, **kwargs):
        if not len(kwargs) == 1:
            raise TypeError("Attr() requires exactly one keyword argument")
        super(Attr, self).__init__(**kwargs)

class HasAttrs(Attrs):
    """Matches objects that have all of the specified attribute names,
    regardless of their values.
    """
    def __init__(self, *args):
        super(HasAttrs, self).__init__(*args)

class HasAttr(HasAttrs):
    """Matches object that have an attribute with given name,
    regardless of its value.
    """
    def __init__(self, name):
        super(HasAttr, self).__init__(name)
<|endoftext|>


<commit_before># -*- Mode: Python -*-

import regular
import automata
from pprint import pprint as pp
import sys
W = sys.stdout.write

# two approaches for generating a lexer:
#  1) generate tables and dispatch through them at run-time.
#  2) generate range-test code from the dfa.

# #2 should be faster, especially if we try a little harder.
# idea: feed a bunch of sample data through the lexer, collecting
#   probabilities at each transition, use this to reorder them.
#   [a branch predictor will do the same thing, is it worth it?]

# I think tables are better once the lexer hits a certain size.

class lexer:

    def __init__ (self, lexicon):
        # combine all the regexps into one, with each sub-expression wrapped with TOKEN
        tokens = regular.n_or ([ (regular.TOKEN, regular.parse (x), y) for (x, y) in lexicon ])
        #pp (tokens)
        c0 = automata.regexp_to_nfa_converter()
        nfa, start, end = c0.go (tokens)
        #pp (nfa)
        c1 = automata.nfa_dfa_converter (nfa, start, end)
        dfa, finals, actions = c1.go (c0.tokens)
        flat = c1.coalesce()
        #print 'dfa='
        #pp (flat)
        #print 'finals=', finals
        self.dfa = flat
        # XXX finals + actions == redundant
        self.finals = finals
        self.actions = {}
        for state, action in actions:
            self.actions[state] = action

    def sort_transitions (self, trans):
        # sort transitions in a way that generates the most efficient set of
        #   range tests.
        tests = [ (sym.as_ranges()[1], sym, ts) for (sym, ts) in trans ]
        tests.sort (lambda a,b: cmp (len(a[0]), len(b[0])))
        return [ (sym, ts) for (ranges, sym, ts) in tests ]

    def generate (self):
        print     "    def step (self, ch):"
        for i in range (len (self.dfa)):
            if i == 0:
                print "        if self.state == %d:" % i
            else:
                print "        elif self.state == %d:" % i
            state_len = len (self.dfa[i])
            trans = self.sort_transitions (self.dfa[i])
            for j in range (state_len):
                sym, ts = trans[j]
                if state_len == 1:
                    print "            self.state = %d" % (ts,)
                elif j == state_len - 1:
                    # last range is always a catch-all
                    print "            else:"
                    print "                self.state = %d" % (ts,)
                else:
                    invert, ranges = sym.as_ranges()
                    test = []
                    for start, end in ranges:
                        if start == end:
                            test.append ("ch == %d" % (start,))
                        else:
                            test.append ("%d <= ch <= %d" % (start, end))
                    test = ' or '.join (test)
                    if invert:
                        test = 'not (%s)' % (test,)
                    if j == 0:
                        print "            if %s:" % (test,)
                    else:
                        print "            elif %s:" % (test,)
                    print "                self.state = %d" % (ts,)
            if self.actions.has_key (i):
                print "                # TOKEN %s" % (self.actions[i],)

    def gen_c (self):
        print "int step (int state, char ch)"
        print "{"
        for i in range (len (self.dfa)):
            if i == 0:
                print "  if (state == %d) {" % i
            else:
                print "  } else if (state == %d) {" % i
            state_len = len (self.dfa[i])
            trans = self.sort_transitions (self.dfa[i])
            for j in range (state_len):
                sym, ts = trans[j]
                if state_len == 1:
                    print "    state = %d;" % (ts,)
                elif j == state_len - 1:
                    # last range is always a catch-all
                    print "    } else {"
                    print "      state = %d;" % (ts,)
                    print "    }"
                else:
                    invert, ranges = sym.as_ranges()
                    test = []
                    for start, end in ranges:
                        if start == end:
                            test.append ("(ch == %d)" % (start,))
                        else:
                            test.append ("(ch >= %d && ch <= %d)" % (start, end))
                    test = ' || '.join (test)
                    if invert:
                        test = '!(%s)' % (test,)
                    if j == 0:
                        print "    if (%s) {" % (test,)
                    else:
                        print "    } else if (%s) {" % (test,)
                    print "      state = %d;" % (ts,)
        print "  }"
        print "  return state;"
        print "}"

    def gen_scheme_code (self):
        print ";; generated by lexer.py"
        print "(define (gen-finals)"
        print "  #("
        for i in range (len (self.dfa)):
            f = self.actions.get (i, None)
            if f:
                print "  %s" % (f,)
            else:
                print "  #f"
        print "  ))"
        sym_funs = {}
        print "(define (step ch st)"
        print "  (case st"
        for i in range (len (self.dfa)):
            print "((%d)" % (i,)
            print " (cond"
            state_len = len (self.dfa[i])
            trans = self.sort_transitions (self.dfa[i])
            for j in range (state_len):
                sym, ts = trans[j]
                if state_len == 1 or (j == state_len - 1):
                    print "   (else %d)" % (ts,)
                else:
                    invert, ranges = sym.as_ranges()
                    if sym.num_set == 1:
                        print "   ((eq? ch %d) %d)" % (ranges[0][0], ts)
                    else:
                        if not sym_funs.has_key (sym):
                            test = "(eq? (string-ref \"%s\" ch) #\\1)" % (sym.as_string(),)
                            counter = len (sym_funs)
                            sym_funs[sym] = counter, test
                        else:
                            counter, test = sym_funs[sym]
                        print "   ((sym-test-%d ch) %d)" % (counter, ts)
            print "  )" # close cond
            print " )" # close state tests...
        print "))" # close case, then define

        for sym, (counter, test) in sym_funs.iteritems():
            print "(define (sym-test-%d ch)" % (counter,)
            print "  " + test + ")"

    def gen_scheme_table (self):
        # generate a table-based dfa, using strings
        assert (len (self.dfa) < 256)
        print ";; generated by lexer.py"
        tables = []
        for i in range (len (self.dfa)):
            table = [1] * 256
            for sym, ts in self.dfa[i]:
                for i in range (256):
                    if sym[i]:
                        table[i] = ts
            table = ''.join ([chr(x) for x in table])
            tables.append (table)
        unique = {}
        for t in tables:
            if not unique.has_key (t):
                unique[t] = len (unique)
        indirect = []
        for t in tables:
            indirect.append (unique[t])
        print "(define dfa "
        print "  (let* ("
        for table, index in unique.iteritems():
            print "         (t%d \"%s\")" % (index, ''.join (["\\x%02x" % (ord (x),) for x in table]))
        print "        )"
        print "   (vector", " ".join (["t%d" % (x,) for x in indirect]), ")))"

        # find the sink state
        sink = unique["\x01" * 256]

        print "(define finals"
        print "  #("
        for i in range (len (self.dfa)):
            f = self.actions.get (i, None)
            if f:
                if indirect[i] == sink:
                    # is this a single-char final state?
                    print "  (%s . #t)" % (f,)
                else:
                    print "  (%s . #f)" % (f,)
            else:
                print "  #f"
        print "  ))"
        print "(define (step ch state)"
        print "  (char->ascii (string-ref (vector-ref dfa state) (char->ascii ch))))"
        return tables

    def gen_irken (self):
        # generate a table-based dfa, using strings
        assert (len (self.dfa) < 256)
        print ";; generated by lexer.py"
        print ("(datatype final\n"
               "  (union\n"
               "    (no)\n"
               "    (yes symbol bool)\n"
               "  ))")
        tables = []
        for i in range (len (self.dfa)):
            table = [1] * 256
            for sym, ts in self.dfa[i]:
                for i in range (256):
                    if sym[i]:
                        table[i] = ts
            table = ''.join ([chr(x) for x in table])
            tables.append (table)
        unique = {}
        for t in tables:
            if not unique.has_key (t):
                unique[t] = len (unique)
        indirect = []
        for t in tables:
            indirect.append (unique[t])
        print "(define dfa "
        print "  (let* ("
        for table, index in unique.iteritems():
            print "         (t%d \"%s\")" % (index, ''.join (["\\x%02x" % (ord (x),) for x in table]))
        print "        )"
        print "   #(", " ".join (["t%d" % (x,) for x in indirect]), ")))"

        # find the sink state
        sink = unique["\x01" * 256]

        print "(define finals"
        print "  #("
        for i in range (len (self.dfa)):
            f = self.actions.get (i, None)
            if f:
                if indirect[i] == sink:
                    # is this a single-char final state?
                    print "  (final/yes '%s #t)" % (f,)
                else:
                    print "  (final/yes '%s #f)" % (f,)
            else:
                print "  (final/no)"
        print "  ))"
        print "(define (step ch state)"
        print "  (char->ascii (string-ref dfa[state] (char->ascii ch))))"
        return tables

    def read (self, file):
        s = file.read()
        for j in range (10):
            self.state = 0
            current = []
            good = None
            slen = len(s)
            i = 0
            while i < slen:
                ch = s[i]
                while 1:
                    self.step (ord(ch))
                    #W ('[%d|%d]' % (ord(ch),self.state,))
                    action = self.actions.get (self.state, None)
                    if not good and action:
                        # transition in
                        good = action
                        break
                    if good and not action:
                        # transition out
                        kind, fun = good
                        fun (kind, ''.join (current))
                        current = []
                        self.state = 0
                        good = None
                    else:
                        break
                current.append (ch)
                i += 1

keywords = '|'.join ('and is in not if then else elif yield while for try def class'.split())
augassign = ('(' + '|'.join (r'\+ - \* / % & \| \^ << >> \*\* //'.split()) + ')=')

python_lexicon = [
    (r'[0-9]+',                 'number'),
    (r'[A-Za-z_][A-Za-z0-9_]*', 'ident'),
    (r'"([^\\\n"]|(\\.))*"',    'string1'),
    (r"'([^\\\n']|(\\.))*'",    'string2'),
    (r'[ \t]*#[^\n]*\n',        'comment'),
    (r'[-+]',                   'addop'),
    (r'([*/%]|//)',             'mulop'),
    (r'<|>|<=|>=|==',           'compare'),
    (r'<<|>>',                  'shift'),
    (r'\*\*',                   'power'),
    (r'[ \t]+',                 'whitespace'),
    (r'[\n]',                   'newline'),
    (r'\.',                     'getattr'),
    (r'=',                      'assign'),
    (r'\(',                     'lparen'),
    (r'\)',                     'rparen'),
    (r',',                      'comma'),
    (r':',                      'colon'),
    (r';',                      'semicolon'),
    (r'{',                      'lbracket'),
    (r'}',                      'rbracket'),
    (r'\[',                     'lbrace'),
    (r'\]',                     'rbrace'),
    (r'&',                      'bitand'),
    (r'\|',                     'bitor'),
    (r'^',                      'bitxor'),
    (r'~',                      'bitnot'),
    (keywords,                  'keyword'),
    (augassign,                 'augassign'),
    ]

if __name__ == '__main__':
    m = lexer (python_lexicon)
    #m.gen_scheme_code()
    #m.gen_scheme_table()
    m.gen_irken()

<commit_msg>updated for new type system<commit_after># -*- Mode: Python -*-

import regular
import automata
from pprint import pprint as pp
import sys
W = sys.stdout.write

# two approaches for generating a lexer:
#  1) generate tables and dispatch through them at run-time.
#  2) generate range-test code from the dfa.

# #2 should be faster, especially if we try a little harder.
# idea: feed a bunch of sample data through the lexer, collecting
#   probabilities at each transition, use this to reorder them.
#   [a branch predictor will do the same thing, is it worth it?]

# I think tables are better once the lexer hits a certain size.

class lexer:

    def __init__ (self, lexicon):
        # combine all the regexps into one, with each sub-expression wrapped with TOKEN
        tokens = regular.n_or ([ (regular.TOKEN, regular.parse (x), y) for (x, y) in lexicon ])
        #pp (tokens)
        c0 = automata.regexp_to_nfa_converter()
        nfa, start, end = c0.go (tokens)
        #pp (nfa)
        c1 = automata.nfa_dfa_converter (nfa, start, end)
        dfa, finals, actions = c1.go (c0.tokens)
        flat = c1.coalesce()
        #print 'dfa='
        #pp (flat)
        #print 'finals=', finals
        self.dfa = flat
        # XXX finals + actions == redundant
        self.finals = finals
        self.actions = {}
        for state, action in actions:
            self.actions[state] = action

    def sort_transitions (self, trans):
        # sort transitions in a way that generates the most efficient set of
        #   range tests.
        tests = [ (sym.as_ranges()[1], sym, ts) for (sym, ts) in trans ]
        tests.sort (lambda a,b: cmp (len(a[0]), len(b[0])))
        return [ (sym, ts) for (ranges, sym, ts) in tests ]

    def generate (self):
        print     "    def step (self, ch):"
        for i in range (len (self.dfa)):
            if i == 0:
                print "        if self.state == %d:" % i
            else:
                print "        elif self.state == %d:" % i
            state_len = len (self.dfa[i])
            trans = self.sort_transitions (self.dfa[i])
            for j in range (state_len):
                sym, ts = trans[j]
                if state_len == 1:
                    print "            self.state = %d" % (ts,)
                elif j == state_len - 1:
                    # last range is always a catch-all
                    print "            else:"
                    print "                self.state = %d" % (ts,)
                else:
                    invert, ranges = sym.as_ranges()
                    test = []
                    for start, end in ranges:
                        if start == end:
                            test.append ("ch == %d" % (start,))
                        else:
                            test.append ("%d <= ch <= %d" % (start, end))
                    test = ' or '.join (test)
                    if invert:
                        test = 'not (%s)' % (test,)
                    if j == 0:
                        print "            if %s:" % (test,)
                    else:
                        print "            elif %s:" % (test,)
                    print "                self.state = %d" % (ts,)
            if self.actions.has_key (i):
                print "                # TOKEN %s" % (self.actions[i],)

    def gen_c (self):
        print "int step (int state, char ch)"
        print "{"
        for i in range (len (self.dfa)):
            if i == 0:
                print "  if (state == %d) {" % i
            else:
                print "  } else if (state == %d) {" % i
            state_len = len (self.dfa[i])
            trans = self.sort_transitions (self.dfa[i])
            for j in range (state_len):
                sym, ts = trans[j]
                if state_len == 1:
                    print "    state = %d;" % (ts,)
                elif j == state_len - 1:
                    # last range is always a catch-all
                    print "    } else {"
                    print "      state = %d;" % (ts,)
                    print "    }"
                else:
                    invert, ranges = sym.as_ranges()
                    test = []
                    for start, end in ranges:
                        if start == end:
                            test.append ("(ch == %d)" % (start,))
                        else:
                            test.append ("(ch >= %d && ch <= %d)" % (start, end))
                    test = ' || '.join (test)
                    if invert:
                        test = '!(%s)' % (test,)
                    if j == 0:
                        print "    if (%s) {" % (test,)
                    else:
                        print "    } else if (%s) {" % (test,)
                    print "      state = %d;" % (ts,)
        print "  }"
        print "  return state;"
        print "}"

    def gen_scheme_code (self):
        print ";; generated by lexer.py"
        print "(define (gen-finals)"
        print "  #("
        for i in range (len (self.dfa)):
            f = self.actions.get (i, None)
            if f:
                print "  %s" % (f,)
            else:
                print "  #f"
        print "  ))"
        sym_funs = {}
        print "(define (step ch st)"
        print "  (case st"
        for i in range (len (self.dfa)):
            print "((%d)" % (i,)
            print " (cond"
            state_len = len (self.dfa[i])
            trans = self.sort_transitions (self.dfa[i])
            for j in range (state_len):
                sym, ts = trans[j]
                if state_len == 1 or (j == state_len - 1):
                    print "   (else %d)" % (ts,)
                else:
                    invert, ranges = sym.as_ranges()
                    if sym.num_set == 1:
                        print "   ((eq? ch %d) %d)" % (ranges[0][0], ts)
                    else:
                        if not sym_funs.has_key (sym):
                            test = "(eq? (string-ref \"%s\" ch) #\\1)" % (sym.as_string(),)
                            counter = len (sym_funs)
                            sym_funs[sym] = counter, test
                        else:
                            counter, test = sym_funs[sym]
                        print "   ((sym-test-%d ch) %d)" % (counter, ts)
            print "  )" # close cond
            print " )" # close state tests...
        print "))" # close case, then define

        for sym, (counter, test) in sym_funs.iteritems():
            print "(define (sym-test-%d ch)" % (counter,)
            print "  " + test + ")"

    def gen_scheme_table (self):
        # generate a table-based dfa, using strings
        assert (len (self.dfa) < 256)
        print ";; generated by lexer.py"
        tables = []
        for i in range (len (self.dfa)):
            table = [1] * 256
            for sym, ts in self.dfa[i]:
                for i in range (256):
                    if sym[i]:
                        table[i] = ts
            table = ''.join ([chr(x) for x in table])
            tables.append (table)
        unique = {}
        for t in tables:
            if not unique.has_key (t):
                unique[t] = len (unique)
        indirect = []
        for t in tables:
            indirect.append (unique[t])
        print "(define dfa "
        print "  (let* ("
        for table, index in unique.iteritems():
            print "         (t%d \"%s\")" % (index, ''.join (["\\x%02x" % (ord (x),) for x in table]))
        print "        )"
        print "   (vector", " ".join (["t%d" % (x,) for x in indirect]), ")))"

        # find the sink state
        sink = unique["\x01" * 256]

        print "(define finals"
        print "  #("
        for i in range (len (self.dfa)):
            f = self.actions.get (i, None)
            if f:
                if indirect[i] == sink:
                    # is this a single-char final state?
                    print "  (%s . #t)" % (f,)
                else:
                    print "  (%s . #f)" % (f,)
            else:
                print "  #f"
        print "  ))"
        print "(define (step ch state)"
        print "  (char->ascii (string-ref (vector-ref dfa state) (char->ascii ch))))"
        return tables

    def gen_irken (self):
        # generate a table-based dfa, using strings
        assert (len (self.dfa) < 256)
        print ";; generated by lexer.py"
        tables = []
        for i in range (len (self.dfa)):
            table = [1] * 256
            for sym, ts in self.dfa[i]:
                for i in range (256):
                    if sym[i]:
                        table[i] = ts
            table = ''.join ([chr(x) for x in table])
            tables.append (table)
        unique = {}
        for t in tables:
            if not unique.has_key (t):
                unique[t] = len (unique)
        indirect = []
        for t in tables:
            indirect.append (unique[t])
        print "(define dfa "
        print "  (let* ("
        for table, index in unique.iteritems():
            print "         (t%d \"%s\")" % (index, ''.join (["\\x%02x" % (ord (x),) for x in table]))
        print "        )"
        print "   #(", " ".join (["t%d" % (x,) for x in indirect]), ")))"

        # find the sink state
        sink = unique["\x01" * 256]

        print "(define finals"
        print "  #("
        for i in range (len (self.dfa)):
            f = self.actions.get (i, None)
            if f:
                if indirect[i] == sink:
                    # is this a single-char final state?
                    print "  (:yes '%s #t)" % (f,)
                else:
                    print "  (:yes '%s #f)" % (f,)
            else:
                print "  (:no)"
        print "  ))"
        print "(define (step ch state)"
        print "  (char->ascii (string-ref dfa[state] (char->ascii ch))))"
        return tables

    def read (self, file):
        s = file.read()
        for j in range (10):
            self.state = 0
            current = []
            good = None
            slen = len(s)
            i = 0
            while i < slen:
                ch = s[i]
                while 1:
                    self.step (ord(ch))
                    #W ('[%d|%d]' % (ord(ch),self.state,))
                    action = self.actions.get (self.state, None)
                    if not good and action:
                        # transition in
                        good = action
                        break
                    if good and not action:
                        # transition out
                        kind, fun = good
                        fun (kind, ''.join (current))
                        current = []
                        self.state = 0
                        good = None
                    else:
                        break
                current.append (ch)
                i += 1

keywords = '|'.join ('and is in not if then else elif yield while for try def class'.split())
augassign = ('(' + '|'.join (r'\+ - \* / % & \| \^ << >> \*\* //'.split()) + ')=')

python_lexicon = [
    (r'[0-9]+',                 'number'),
    (r'[A-Za-z_][A-Za-z0-9_]*', 'ident'),
    (r'"([^\\\n"]|(\\.))*"',    'string1'),
    (r"'([^\\\n']|(\\.))*'",    'string2'),
    (r'[ \t]*#[^\n]*\n',        'comment'),
    (r'[-+]',                   'addop'),
    (r'([*/%]|//)',             'mulop'),
    (r'<|>|<=|>=|==',           'compare'),
    (r'<<|>>',                  'shift'),
    (r'\*\*',                   'power'),
    (r'[ \t]+',                 'whitespace'),
    (r'[\n]',                   'newline'),
    (r'\.',                     'getattr'),
    (r'=',                      'assign'),
    (r'\(',                     'lparen'),
    (r'\)',                     'rparen'),
    (r',',                      'comma'),
    (r':',                      'colon'),
    (r';',                      'semicolon'),
    (r'{',                      'lbracket'),
    (r'}',                      'rbracket'),
    (r'\[',                     'lbrace'),
    (r'\]',                     'rbrace'),
    (r'&',                      'bitand'),
    (r'\|',                     'bitor'),
    (r'^',                      'bitxor'),
    (r'~',                      'bitnot'),
    (keywords,                  'keyword'),
    (augassign,                 'augassign'),
    ]

if __name__ == '__main__':
    m = lexer (python_lexicon)
    #m.gen_scheme_code()
    #m.gen_scheme_table()
    m.gen_irken()

<|endoftext|>


<commit_before>from copy import deepcopy

import dictdiffer

import virtool.db.otus
import virtool.errors
import virtool.otus
import virtool.utils
import virtool.history
from virtool.api.utils import paginate

MOST_RECENT_PROJECTION = [
    "_id",
    "description",
    "method_name",
    "user",
    "otu",
    "created_at"
]

LIST_PROJECTION = [
    "_id",
    "description",
    "method_name",
    "created_at",
    "index",
    "otu",
    "reference",
    "user"
]

PROJECTION = LIST_PROJECTION + [
    "diff"
]

async def add(db, method_name, old, new, description, user_id):
    """
    Add a change document to the history collection.

    :param db: the application database client
    :type db: :class:`~motor.motor_asyncio.AsyncIOMotorClient`

    :param method_name: the name of the handler method that executed the change
    :type method_name: str

    :param old: the otu document prior to the change
    :type new: Union[dict, None]

    :param new: the otu document after the change
    :type new: Union[dict, None]

    :param description: a human readable description of the change
    :type description: str

    :param user_id: the id of the requesting user
    :type user_id: str

    :return: the change document
    :rtype: Coroutine[dict]

    """
    try:
        otu_id = old["_id"]
    except TypeError:
        otu_id = new["_id"]

    try:
        otu_name = old["name"]
    except TypeError:
        otu_name = new["name"]

    try:
        otu_version = int(new["version"])
    except (TypeError, KeyError):
        otu_version = "removed"

    try:
        ref_id = old["reference"]["id"]
    except (TypeError, KeyError):
        ref_id = new["reference"]["id"]

    document = {
        "_id": ".".join([str(otu_id), str(otu_version)]),
        "method_name": method_name,
        "description": description,
        "created_at": virtool.utils.timestamp(),
        "otu": {
            "id": otu_id,
            "name": otu_name,
            "version": otu_version
        },
        "reference": {
            "id": ref_id
        },
        "index": {
            "id": "unbuilt",
            "version": "unbuilt"
        },
        "user": {
            "id": user_id
        }
    }

    if method_name == "create":
        document["diff"] = new

    elif method_name == "remove":
        document["diff"] = old

    else:
        document["diff"] = virtool.history.calculate_diff(old, new)

    await db.history.insert_one(document)

    return document

async def find(db, req_query, base_query=None, unbuilt=False):
    data = await paginate(
        db.history,
        {},
        req_query,
        base_query=base_query,
        sort="created_at",
        projection=LIST_PROJECTION,
        reverse=True
    )

    return data

async def get_contributors(db, query):
    """
    Return an list of contributors and their contribution count for a specific set of history.

    :param db: the application database client
    :type db: :class:`~motor.motor_asyncio.AsyncIOMotorClient`

    :param query: a query to filter scanned history by
    :type query: dict

    :return: a list of contributors to the scanned history changes
    :rtype: List[dict]

    """
    contributors = await db.history.aggregate([
        {"$match": query},
        {"$group": {
            "_id": "$user.id",
            "count": {"$sum": 1}
        }}
    ]).to_list(None)

    return [{"id": c["_id"], "count": c["count"]} for c in contributors]

async def get_most_recent_change(db, otu_id):
    """
    Get the most recent change for the otu identified by the passed ``otu_id``.

    :param db: the application database client
    :type db: :class:`~motor.motor_asyncio.AsyncIOMotorClient`

    :param otu_id: the target otu_id
    :type otu_id: str

    :return: the most recent change document
    :rtype: Coroutine[dict]

    """
    return await db.history.find_one({
        "otu.id": otu_id,
        "index.id": "unbuilt"
    }, MOST_RECENT_PROJECTION, sort=[("created_at", -1)])

async def patch_to_version(db, otu_id, version):
    """
    Take a joined otu back in time to the passed ``version``. Uses the diffs in the change documents associated with
    the otu.

    :param db: the application database client
    :type db: :class:`~motor.motor_asyncio.AsyncIOMotorClient`

    :param otu_id: the id of the otu to patch
    :type otu_id: str

    :param version: the version to patch to
    :type version: str or int

    :return: the current joined otu, patched otu, and the ids of changes reverted in the process
    :rtype: Coroutine[tuple]

    """
    # A list of history_ids reverted to produce the patched entry.
    reverted_history_ids = list()

    current = await virtool.db.otus.join(db, otu_id) or dict()

    if "version" in current and current["version"] == version:
        return current, deepcopy(current), reverted_history_ids

    patched = deepcopy(current)

    # Sort the changes by descending timestamp.
    async for change in db.history.find({"otu.id": otu_id}, sort=[("created_at", -1)]):
        if change["otu"]["version"] == "removed" or change["otu"]["version"] > version:
            reverted_history_ids.append(change["_id"])

            if change["method_name"] == "remove":
                patched = change["diff"]

            elif change["method_name"] == "create":
                patched = None

            else:
                diff = dictdiffer.swap(change["diff"])
                patched = dictdiffer.patch(diff, patched)
        else:
            break

    if current == {}:
        current = None

    return current, patched, reverted_history_ids

async def revert(db, change_id):
    """
    Revert a history change given by the passed ``change_id``.

    :param db: the application database client
    :type db: :class:`~motor.motor_asyncio.AsyncIOMotorClient`

    :param change_id: a unique id for the cah
    :return:
    """
    change = await db.history.find_one({"_id": change_id}, ["index"])

    if not change:
        raise virtool.errors.DatabaseError("Change does not exist")

    if change["index"]["id"] != "unbuilt" or change["index"]["version"] != "unbuilt":
        raise virtool.errors.DatabaseError("Change is included in a build an not revertible")

    otu_id, otu_version = change_id.split(".")

    if otu_version != "removed":
        otu_version = int(otu_version)

    _, patched, history_to_delete = await patch_to_version(
        db,
        otu_id,
        otu_version - 1
    )

    # Remove the old sequences from the collection.
    await db.sequences.delete_many({"otu_id": otu_id})

    if patched is not None:
        patched_otu, sequences = virtool.otus.split(patched)

        # Add the reverted sequences to the collection.
        for sequence in sequences:
            await db.sequences.insert_one(sequence)

        # Replace the existing otu with the patched one. If it doesn't exist, insert it.
        await db.otus.replace_one({"_id": otu_id}, patched_otu, upsert=True)

    else:
        await db.otus.delete_one({"_id": otu_id})

    await db.history.delete_many({"_id": {"$in": history_to_delete}})

    return patched
<commit_msg>Add patch_to_verified() to patch OTUs to last verified state<commit_after>from copy import deepcopy

import dictdiffer

import virtool.db.otus
import virtool.errors
import virtool.otus
import virtool.utils
import virtool.history
from virtool.api.utils import paginate

MOST_RECENT_PROJECTION = [
    "_id",
    "description",
    "method_name",
    "user",
    "otu",
    "created_at"
]

LIST_PROJECTION = [
    "_id",
    "description",
    "method_name",
    "created_at",
    "index",
    "otu",
    "reference",
    "user"
]

PROJECTION = LIST_PROJECTION + [
    "diff"
]

async def add(db, method_name, old, new, description, user_id):
    """
    Add a change document to the history collection.

    :param db: the application database client
    :type db: :class:`~motor.motor_asyncio.AsyncIOMotorClient`

    :param method_name: the name of the handler method that executed the change
    :type method_name: str

    :param old: the otu document prior to the change
    :type new: Union[dict, None]

    :param new: the otu document after the change
    :type new: Union[dict, None]

    :param description: a human readable description of the change
    :type description: str

    :param user_id: the id of the requesting user
    :type user_id: str

    :return: the change document
    :rtype: Coroutine[dict]

    """
    try:
        otu_id = old["_id"]
    except TypeError:
        otu_id = new["_id"]

    try:
        otu_name = old["name"]
    except TypeError:
        otu_name = new["name"]

    try:
        otu_version = int(new["version"])
    except (TypeError, KeyError):
        otu_version = "removed"

    try:
        ref_id = old["reference"]["id"]
    except (TypeError, KeyError):
        ref_id = new["reference"]["id"]

    document = {
        "_id": ".".join([str(otu_id), str(otu_version)]),
        "method_name": method_name,
        "description": description,
        "created_at": virtool.utils.timestamp(),
        "otu": {
            "id": otu_id,
            "name": otu_name,
            "version": otu_version
        },
        "reference": {
            "id": ref_id
        },
        "index": {
            "id": "unbuilt",
            "version": "unbuilt"
        },
        "user": {
            "id": user_id
        }
    }

    if method_name == "create":
        document["diff"] = new

    elif method_name == "remove":
        document["diff"] = old

    else:
        document["diff"] = virtool.history.calculate_diff(old, new)

    await db.history.insert_one(document)

    return document

async def find(db, req_query, base_query=None, unbuilt=False):
    data = await paginate(
        db.history,
        {},
        req_query,
        base_query=base_query,
        sort="created_at",
        projection=LIST_PROJECTION,
        reverse=True
    )

    return data

async def get_contributors(db, query):
    """
    Return an list of contributors and their contribution count for a specific set of history.

    :param db: the application database client
    :type db: :class:`~motor.motor_asyncio.AsyncIOMotorClient`

    :param query: a query to filter scanned history by
    :type query: dict

    :return: a list of contributors to the scanned history changes
    :rtype: List[dict]

    """
    contributors = await db.history.aggregate([
        {"$match": query},
        {"$group": {
            "_id": "$user.id",
            "count": {"$sum": 1}
        }}
    ]).to_list(None)

    return [{"id": c["_id"], "count": c["count"]} for c in contributors]

async def get_most_recent_change(db, otu_id):
    """
    Get the most recent change for the otu identified by the passed ``otu_id``.

    :param db: the application database client
    :type db: :class:`~motor.motor_asyncio.AsyncIOMotorClient`

    :param otu_id: the target otu_id
    :type otu_id: str

    :return: the most recent change document
    :rtype: Coroutine[dict]

    """
    return await db.history.find_one({
        "otu.id": otu_id,
        "index.id": "unbuilt"
    }, MOST_RECENT_PROJECTION, sort=[("created_at", -1)])

async def patch_to_verified(db, otu_id):
    current = await virtool.db.otus.join(db, otu_id) or dict()

    if current and current["verified"]:
        return current

    patched = deepcopy(current)

    async for change in db.history.find({"otu.id": otu_id}, sort=[("created_at", -1)]):
        if change["method_name"] == "remove":
            patched = change["diff"]

        elif change["method_name"] == "create":
            return None

        else:
            diff = dictdiffer.swap(change["diff"])
            patched = dictdiffer.patch(diff, patched)

        if patched["verified"]:
            return patched

async def patch_to_version(db, otu_id, version):
    """
    Take a joined otu back in time to the passed ``version``. Uses the diffs in the change documents associated with
    the otu.

    :param db: the application database client
    :type db: :class:`~motor.motor_asyncio.AsyncIOMotorClient`

    :param otu_id: the id of the otu to patch
    :type otu_id: str

    :param version: the version to patch to
    :type version: str or int

    :return: the current joined otu, patched otu, and the ids of changes reverted in the process
    :rtype: Coroutine[tuple]

    """
    # A list of history_ids reverted to produce the patched entry.
    reverted_history_ids = list()

    current = await virtool.db.otus.join(db, otu_id) or dict()

    if "version" in current and current["version"] == version:
        return current, deepcopy(current), reverted_history_ids

    patched = deepcopy(current)

    # Sort the changes by descending timestamp.
    async for change in db.history.find({"otu.id": otu_id}, sort=[("created_at", -1)]):
        if change["otu"]["version"] == "removed" or change["otu"]["version"] > version:
            reverted_history_ids.append(change["_id"])

            if change["method_name"] == "remove":
                patched = change["diff"]

            elif change["method_name"] == "create":
                patched = None

            else:
                diff = dictdiffer.swap(change["diff"])
                patched = dictdiffer.patch(diff, patched)
        else:
            break

    if current == {}:
        current = None

    return current, patched, reverted_history_ids

async def revert(db, change_id):
    """
    Revert a history change given by the passed ``change_id``.

    :param db: the application database client
    :type db: :class:`~motor.motor_asyncio.AsyncIOMotorClient`

    :param change_id: a unique id for the cah
    :return:
    """
    change = await db.history.find_one({"_id": change_id}, ["index"])

    if not change:
        raise virtool.errors.DatabaseError("Change does not exist")

    if change["index"]["id"] != "unbuilt" or change["index"]["version"] != "unbuilt":
        raise virtool.errors.DatabaseError("Change is included in a build an not revertible")

    otu_id, otu_version = change_id.split(".")

    if otu_version != "removed":
        otu_version = int(otu_version)

    _, patched, history_to_delete = await patch_to_version(
        db,
        otu_id,
        otu_version - 1
    )

    # Remove the old sequences from the collection.
    await db.sequences.delete_many({"otu_id": otu_id})

    if patched is not None:
        patched_otu, sequences = virtool.otus.split(patched)

        # Add the reverted sequences to the collection.
        for sequence in sequences:
            await db.sequences.insert_one(sequence)

        # Replace the existing otu with the patched one. If it doesn't exist, insert it.
        await db.otus.replace_one({"_id": otu_id}, patched_otu, upsert=True)

    else:
        await db.otus.delete_one({"_id": otu_id})

    await db.history.delete_many({"_id": {"$in": history_to_delete}})

    return patched
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    flask.wrappers
    ~~~~~~~~~~~~~~

    Implements the WSGI wrappers (request and response).

    :copyright: (c) 2010 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

from werkzeug import Request as RequestBase, Response as ResponseBase, \
    cached_property

from .helpers import json, _assert_have_json

class Request(RequestBase):
    """The request object used by default in flask.  Remembers the
    matched endpoint and view arguments.

    It is what ends up as :class:`~flask.request`.  If you want to replace
    the request object used you can subclass this and set
    :attr:`~flask.Flask.request_class` to your subclass.
    """

    #: the endpoint that matched the request.  This in combination with
    #: :attr:`view_args` can be used to reconstruct the same or a
    #: modified URL.  If an exception happened when matching, this will
    #: be `None`.
    endpoint = None

    #: a dict of view arguments that matched the request.  If an exception
    #: happened when matching, this will be `None`.
    view_args = None

    #: if matching the URL failed, this is the exception that will be
    #: raised / was raised as part of the request handling.  This is
    #: usually a :exc:`~werkzeug.exceptions.NotFound` exception or
    #: something similar.
    routing_exception = None

    @property
    def module(self):
        """The name of the current module"""
        if self.endpoint and '.' in self.endpoint:
            return self.endpoint.rsplit('.', 1)[0]

    @cached_property
    def json(self):
        """If the mimetype is `application/json` this will contain the
        parsed JSON data.
        """
        if __debug__:
            _assert_have_json()
        if self.mimetype == 'application/json':
            return json.loads(self.data)

class Response(ResponseBase):
    """The response object that is used by default in flask.  Works like the
    response object from Werkzeug but is set to have a HTML mimetype by
    default.  Quite often you don't have to create this object yourself because
    :meth:`~flask.Flask.make_response` will take care of that for you.

    If you want to replace the response object used you can subclass this and
    set :attr:`~flask.Flask.response_class` to your subclass.
    """
    default_mimetype = 'text/html'
<commit_msg>update<commit_after># -*- coding: utf-8 -*-
"""
    flask.wrappers
    ~~~~~~~~~~~~~~

    Implements the WSGI wrappers (request and response).

    :copyright: (c) 2010 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

# 说明:
#   - 最新版本的 werkzeug 模块组织结构发生改变, 改为上面的导包路径
#   - 故注释掉下一行的旧版本的导包路径
#
#from werkzeug import Request as RequestBase, Response as ResponseBase, cached_property
#
#
from werkzeug.wrappers import (
    Request as RequestBase,
    Response as ResponseBase,
)
from werkzeug.utils import cached_property

from .helpers import json, _assert_have_json

class Request(RequestBase):
    """The request object used by default in flask.  Remembers the
    matched endpoint and view arguments.

    It is what ends up as :class:`~flask.request`.  If you want to replace
    the request object used you can subclass this and set
    :attr:`~flask.Flask.request_class` to your subclass.
    """

    #: the endpoint that matched the request.  This in combination with
    #: :attr:`view_args` can be used to reconstruct the same or a
    #: modified URL.  If an exception happened when matching, this will
    #: be `None`.
    endpoint = None

    #: a dict of view arguments that matched the request.  If an exception
    #: happened when matching, this will be `None`.
    view_args = None

    #: if matching the URL failed, this is the exception that will be
    #: raised / was raised as part of the request handling.  This is
    #: usually a :exc:`~werkzeug.exceptions.NotFound` exception or
    #: something similar.
    routing_exception = None

    @property
    def module(self):
        """The name of the current module"""
        if self.endpoint and '.' in self.endpoint:
            return self.endpoint.rsplit('.', 1)[0]

    @cached_property
    def json(self):
        """If the mimetype is `application/json` this will contain the
        parsed JSON data.
        """
        if __debug__:
            _assert_have_json()
        if self.mimetype == 'application/json':
            return json.loads(self.data)

class Response(ResponseBase):
    """The response object that is used by default in flask.  Works like the
    response object from Werkzeug but is set to have a HTML mimetype by
    default.  Quite often you don't have to create this object yourself because
    :meth:`~flask.Flask.make_response` will take care of that for you.

    If you want to replace the response object used you can subclass this and
    set :attr:`~flask.Flask.response_class` to your subclass.
    """
    default_mimetype = 'text/html'
<|endoftext|>