<commit_before>"""Affine transformation matrices

The Affine package is derived from Casey Duncan's Planar package. See the
copyright statement below.
"""

#############################################################################
# Copyright (c) 2010 by Casey Duncan
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
#   this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
# * Neither the name(s) of the copyright holders nor the names of its
#   contributors may be used to endorse or promote products derived from this
#   software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AS IS AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
# EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#############################################################################

from __future__ import division

from collections import namedtuple
import math

__all__ = ['Affine']
__author__ = "Sean Gillies"
__version__ = "1.0"

EPSILON=1e-5
EPSILON2=EPSILON**2

def set_epsilon(epsilon):
    """Set the global absolute error value and rounding limit for approximate
    floating point comparison operations. This value is accessible via the
    :attr:`planar.EPSILON` global variable.

    The default value of ``0.00001`` is suitable for values
    that are in the "countable range". You may need a larger
    epsilon when using large absolute values, and a smaller value
    for very small values close to zero. Otherwise approximate
    comparison operations will not behave as expected.
    """
    EPSILON = float(epsilon)
    EPSILON2 = EPSILON**2

class TransformNotInvertibleError(Exception):
    """The transform could not be inverted"""

# Define assert_unorderable() depending on the language
# implicit ordering rules. This keeps things consistent
# across major Python versions
try:
    3 > ""
except TypeError: # pragma: no cover
    # No implicit ordering (newer Python)
    def assert_unorderable(a, b):
        """Assert that a and b are unorderable"""
        return NotImplemented
else: # pragma: no cover
    # Implicit ordering by default (older Python)
    # We must raise an exception ourselves
    # To prevent nonsensical ordering
    def assert_unorderable(a, b):
        """Assert that a and b are unorderable"""
        raise TypeError("unorderable types: %s and %s"
            % (type(a).__name__, type(b).__name__))

def cached_property(func):
    """Special property decorator that caches the computed
    property value in the object's instance dict the first
    time it is accessed.
    """

    def getter(self, name=func.func_name):
        try:
            return self.__dict__[name]
        except KeyError:
            self.__dict__[name] = value = func(self)
            return value

    getter.func_name = func.func_name
    return property(getter, doc=func.func_doc)

def cos_sin_deg(deg):
    """Return the cosine and sin for the given angle
    in degrees, with special-case handling of multiples
    of 90 for perfect right angles
    """
    deg = deg % 360.0
    if deg == 90.0:
        return 0.0, 1.0
    elif deg == 180.0:
        return -1.0, 0
    elif deg == 270.0:
        return 0, -1.0
    rad = math.radians(deg)
    return math.cos(rad), math.sin(rad)

class Affine(
        namedtuple('Affine', ('a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i'))):
    """Two dimensional affine transform for linear mapping from 2D coordinates
    to other 2D coordinates. Parallel lines are preserved by these
    transforms. Affine transforms can perform any combination of translations,
    scales/flips, shears, and rotations.  Class methods are provided to
    conveniently compose transforms from these operations.

    Internally the transform is stored as a 3x3 transformation matrix.  The
    transform may be constructed directly by specifying the first two rows of
    matrix values as 6 floats. Since the matrix is an affine transform, the
    last row is always ``(0, 0, 1)``.

    :param members: 6 floats for the first two matrix rows.
    :type members: float
    """

    def __new__(self, *members):
        if len(members) == 6:
            mat3x3 = [x * 1.0 for x in members] + [0.0, 0.0, 1.0]
            return tuple.__new__(Affine, mat3x3)
        else:
            raise TypeError(
                "Expected 6 number args, got %s" % len(members))

    @classmethod
    def from_gdal(cls, c, a, b, f, d, e):
        members = [a, b, c, d, e, f]
        mat3x3 = [x * 1.0 for x in members] + [0.0, 0.0, 1.0]
        return tuple.__new__(Affine, mat3x3)

    @classmethod
    def identity(cls):
        """Return the identity transform.

        :rtype: Affine
        """
        return identity

    @classmethod
    def translation(cls, xoff, yoff):
        """Create a translation transform from an offset vector.

        :param xoff: Translation x offset.
        :type xoff: float
        :param yoff: Translation y offset.
        :type yoff: float
        :rtype: Affine
        """
        return tuple.__new__(cls,
            (1.0, 0.0, xoff,
             0.0, 1.0, yoff,
             0.0, 0.0, 1.0))

    @classmethod
    def scale(cls, *scaling):
        """Create a scaling transform from a scalar or vector.

        :param scaling: The scaling factor. A scalar value will
            scale in both dimensions equally. A vector scaling
            value scales the dimensions independently.
        :type scaling: float or sequence
        :rtype: Affine
        """
        if len(scaling) == 1:
            sx = sy = float(scaling[0])
        else:
            sx, sy = scaling
        return tuple.__new__(cls,
            (sx, 0.0, 0.0,
             0.0, sy, 0.0,
             0.0, 0.0, 1.0))

    @classmethod
    def shear(cls, x_angle=0, y_angle=0):
        """Create a shear transform along one or both axes.

        :param x_angle: Angle in degrees to shear along the x-axis.
        :type x_angle: float
        :param y_angle: Angle in degrees to shear along the y-axis.
        :type y_angle: float
        :rtype: Affine
        """
        sx = math.tan(math.radians(x_angle))
        sy = math.tan(math.radians(y_angle))
        return tuple.__new__(cls,
            (1.0, sy, 0.0,
             sx, 1.0, 0.0,
             0.0, 0.0, 1.0))

    @classmethod
    def rotation(cls, angle, pivot=None):
        """Create a rotation transform at the specified angle,
        optionally about the specified pivot point.

        :param angle: Rotation angle in degrees
        :type angle: float
        :param pivot: Point to rotate about, if omitted the
            rotation is about the origin.
        :type pivot: sequence
        :rtype: Affine
        """
        ca, sa = cos_sin_deg(angle)
        if pivot is None:
            return tuple.__new__(cls,
                (ca, sa, 0.0,
                -sa, ca, 0.0,
                 0.0, 0.0, 1.0))
        else:
            px, py = pivot
            return tuple.__new__(cls,
                (ca, sa, px - px*ca + py*sa,
                -sa, ca, py - px*sa - py*ca,
                 0.0, 0.0, 1.0))

    def __str__(self):
        """Concise string representation."""
        return ("|% .2f,% .2f,% .2f|\n"
                "|% .2f,% .2f,% .2f|\n"
                "|% .2f,% .2f,% .2f|") % self

    def __repr__(self):
        """Precise string representation."""
        return ("Affine(%r, %r, %r,\n"
                "       %r, %r, %r)") % self[:6]

    def to_gdal(self):
        return (self.c, self.a, self.b, self.f, self.d, self.e)

    @property
    def xoff(self):
        return self.c

    @property
    def yoff(self):
        return self.f

    @cached_property
    def determinant(self):
        """The determinant of the transform matrix. This value
        is equal to the area scaling factor when the transform
        is applied to a shape.
        """
        a, b, c, d, e, f, g, h, i = self
        return a*e - b*d

    @cached_property
    def is_identity(self):
        """True if this transform equals the identity matrix,
        within rounding limits.
        """
        return self is identity or self.almost_equals(identity)

    @cached_property
    def is_rectilinear(self):
        """True if the transform is rectilinear, i.e., whether a shape would
        remain axis-aligned, within rounding limits, after applying the
        transform.
        """
        a, b, c, d, e, f, g, h, i = self
        return ((abs(a) < EPSILON and abs(e) < EPSILON)
            or (abs(d) < EPSILON and abs(b) < EPSILON))

    @cached_property
    def is_conformal(self):
        """True if the transform is conformal, i.e., if angles between points
        are preserved after applying the transform, within rounding limits.
        This implies that the transform has no effective shear.
        """
        a, b, c, d, e, f, g, h, i = self
        return abs(a*b + d*e) < EPSILON

    @cached_property
    def is_orthonormal(self):
        """True if the transform is orthonormal, which means that the
        transform represents a rigid motion, which has no effective scaling or
        shear. Mathematically, this means that the axis vectors of the
        transform matrix are perpendicular and unit-length.  Applying an
        orthonormal transform to a shape always results in a congruent shape.
        """
        a, b, c, d, e, f, g, h, i = self
        return (self.is_conformal
            and abs(1.0 - (a*a + d*d)) < EPSILON
            and abs(1.0 - (b*b + e*e)) < EPSILON)

    @cached_property
    def is_degenerate(self):
        """True if this transform is degenerate, which means that it will
        collapse a shape to an effective area of zero. Degenerate transforms
        cannot be inverted.
        """
        return abs(self.determinant) < EPSILON

    @property
    def column_vectors(self):
        """The values of the transform as three 2D column vectors"""
        a, b, c, d, e, f, _, _, _ = self
        return (a, d), (b, e), (c, f)

    def almost_equals(self, other):
        """Compare transforms for approximate equality.

        :param other: Transform being compared.
        :type other: Affine
        :return: True if absolute difference between each element
            of each respective tranform matrix < ``EPSILON``.
        """
        for i in (0, 1, 2, 3, 4, 5):
            if abs(self[i] - other[i]) >= EPSILON:
                return False
        return True

    def __gt__(self, other):
        return assert_unorderable(self, other)

    __ge__ = __lt__ = __le__ = __gt__

    # Override from base class. We do not support entrywise
    # addition, subtraction or scalar multiplication because
    # the result is not an affine transform

    def __add__(self, other):
        raise TypeError("Operation not supported")

    __iadd__ = __add__

    def __mul__(self, other):
        """Apply the transform using matrix multiplication, creating a
        resulting object of the same type.  A transform may be applied to
        another transform, a vector, vector array, or shape.

        :param other: The object to transform.
        :type other: Affine, :class:`~planar.Vec2`,
            :class:`~planar.Vec2Array`, :class:`~planar.Shape`
        :rtype: Same as ``other``
        """
        sa, sb, sc, sd, se, sf, _, _, _ = self
        if isinstance(other, Affine):
            oa, ob, oc, od, oe, of, _, _, _ = other
            return tuple.__new__(Affine,
                (sa*oa + sb*od, sa*ob + sb*oe, sa*oc + sb*of + sc,
                 sd*oa + se*od, sd*ob + se*oe, sd*oc + se*of + sf,
                 0.0, 0.0, 1.0))
        else:
            try:
                vx, vy = other
            except Exception:
                return NotImplemented
            return (vx*sa + vy*sd + sc, vx*sb + vy*se + sf)

    def __rmul__(self, other):
        # We should not be called if other is an affine instance
        # This is just a guarantee, since we would potentially
        # return the wrong answer in that case
        assert not isinstance(other, Affine)
        return self.__mul__(other)

    def __imul__(self, other):
        if isinstance(other, Affine) or isinstance(other, tuple):
            return self.__mul__(other)
        else:
            return NotImplemented

    def itransform(self, seq):
        """Transform a sequence of points or vectors in place.

        :param seq: Mutable sequence of :class:`~planar.Vec2` to be
            transformed.
        :returns: None, the input sequence is mutated in place.
        """
        if self is not identity and self != identity:
            sa, sb, sc, sd, se, sf, _, _, _ = self
            for i, (x, y) in enumerate(seq):
                seq[i] = (x*sa + y*sd + sc, x*sb + y*se + sf)

    def __invert__(self):
        """Return the inverse transform.

        :raises: :except:`TransformNotInvertible` if the transform
            is degenerate.
        """
        if self.is_degenerate:
            raise TransformNotInvertibleError(
                "Cannot invert degenerate transform")
        idet = 1.0 / self.determinant
        sa, sb, sc, sd, se, sf, _, _, _ = self
        ra = se * idet
        rb = -sb * idet
        rd = -sd * idet
        re = sa * idet
        return tuple.__new__(Affine,
            (ra, rb, -sc*ra - sf*rb,
             rd, re, -sc*rd - sf*re,
             0.0, 0.0, 1.0))

    __hash__ = tuple.__hash__ # hash is not inherited in Py 3

identity = Affine(1, 0, 0, 0, 1, 0)
"""The identity transform"""

# vim: ai ts=4 sts=4 et sw=4 tw=78

<commit_msg>Changes to caching decorator for Python 3.<commit_after>"""Affine transformation matrices

The Affine package is derived from Casey Duncan's Planar package. See the
copyright statement below.
"""

#############################################################################
# Copyright (c) 2010 by Casey Duncan
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
#   this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
# * Neither the name(s) of the copyright holders nor the names of its
#   contributors may be used to endorse or promote products derived from this
#   software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AS IS AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
# EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#############################################################################

from __future__ import division

from collections import namedtuple
import math

__all__ = ['Affine']
__author__ = "Sean Gillies"
__version__ = "1.0"

EPSILON=1e-5
EPSILON2=EPSILON**2

def set_epsilon(epsilon):
    """Set the global absolute error value and rounding limit for approximate
    floating point comparison operations. This value is accessible via the
    :attr:`planar.EPSILON` global variable.

    The default value of ``0.00001`` is suitable for values
    that are in the "countable range". You may need a larger
    epsilon when using large absolute values, and a smaller value
    for very small values close to zero. Otherwise approximate
    comparison operations will not behave as expected.
    """
    EPSILON = float(epsilon)
    EPSILON2 = EPSILON**2

class TransformNotInvertibleError(Exception):
    """The transform could not be inverted"""

# Define assert_unorderable() depending on the language
# implicit ordering rules. This keeps things consistent
# across major Python versions
try:
    3 > ""
except TypeError: # pragma: no cover
    # No implicit ordering (newer Python)
    def assert_unorderable(a, b):
        """Assert that a and b are unorderable"""
        return NotImplemented
else: # pragma: no cover
    # Implicit ordering by default (older Python)
    # We must raise an exception ourselves
    # To prevent nonsensical ordering
    def assert_unorderable(a, b):
        """Assert that a and b are unorderable"""
        raise TypeError("unorderable types: %s and %s"
            % (type(a).__name__, type(b).__name__))

def cached_property(func):
    """Special property decorator that caches the computed
    property value in the object's instance dict the first
    time it is accessed.
    """
    name = getattr(func, 'func_name', func.__name__)
    def getter(self, name=name):
        try:
            return self.__dict__[name]
        except KeyError:
            self.__dict__[name] = value = func(self)
            return value

    getter.func_name = func.func_name
    return property(getter, doc=func.func_doc)

def cos_sin_deg(deg):
    """Return the cosine and sin for the given angle
    in degrees, with special-case handling of multiples
    of 90 for perfect right angles
    """
    deg = deg % 360.0
    if deg == 90.0:
        return 0.0, 1.0
    elif deg == 180.0:
        return -1.0, 0
    elif deg == 270.0:
        return 0, -1.0
    rad = math.radians(deg)
    return math.cos(rad), math.sin(rad)

class Affine(
        namedtuple('Affine', ('a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i'))):
    """Two dimensional affine transform for linear mapping from 2D coordinates
    to other 2D coordinates. Parallel lines are preserved by these
    transforms. Affine transforms can perform any combination of translations,
    scales/flips, shears, and rotations.  Class methods are provided to
    conveniently compose transforms from these operations.

    Internally the transform is stored as a 3x3 transformation matrix.  The
    transform may be constructed directly by specifying the first two rows of
    matrix values as 6 floats. Since the matrix is an affine transform, the
    last row is always ``(0, 0, 1)``.

    :param members: 6 floats for the first two matrix rows.
    :type members: float
    """

    def __new__(self, *members):
        if len(members) == 6:
            mat3x3 = [x * 1.0 for x in members] + [0.0, 0.0, 1.0]
            return tuple.__new__(Affine, mat3x3)
        else:
            raise TypeError(
                "Expected 6 number args, got %s" % len(members))

    @classmethod
    def from_gdal(cls, c, a, b, f, d, e):
        members = [a, b, c, d, e, f]
        mat3x3 = [x * 1.0 for x in members] + [0.0, 0.0, 1.0]
        return tuple.__new__(Affine, mat3x3)

    @classmethod
    def identity(cls):
        """Return the identity transform.

        :rtype: Affine
        """
        return identity

    @classmethod
    def translation(cls, xoff, yoff):
        """Create a translation transform from an offset vector.

        :param xoff: Translation x offset.
        :type xoff: float
        :param yoff: Translation y offset.
        :type yoff: float
        :rtype: Affine
        """
        return tuple.__new__(cls,
            (1.0, 0.0, xoff,
             0.0, 1.0, yoff,
             0.0, 0.0, 1.0))

    @classmethod
    def scale(cls, *scaling):
        """Create a scaling transform from a scalar or vector.

        :param scaling: The scaling factor. A scalar value will
            scale in both dimensions equally. A vector scaling
            value scales the dimensions independently.
        :type scaling: float or sequence
        :rtype: Affine
        """
        if len(scaling) == 1:
            sx = sy = float(scaling[0])
        else:
            sx, sy = scaling
        return tuple.__new__(cls,
            (sx, 0.0, 0.0,
             0.0, sy, 0.0,
             0.0, 0.0, 1.0))

    @classmethod
    def shear(cls, x_angle=0, y_angle=0):
        """Create a shear transform along one or both axes.

        :param x_angle: Angle in degrees to shear along the x-axis.
        :type x_angle: float
        :param y_angle: Angle in degrees to shear along the y-axis.
        :type y_angle: float
        :rtype: Affine
        """
        sx = math.tan(math.radians(x_angle))
        sy = math.tan(math.radians(y_angle))
        return tuple.__new__(cls,
            (1.0, sy, 0.0,
             sx, 1.0, 0.0,
             0.0, 0.0, 1.0))

    @classmethod
    def rotation(cls, angle, pivot=None):
        """Create a rotation transform at the specified angle,
        optionally about the specified pivot point.

        :param angle: Rotation angle in degrees
        :type angle: float
        :param pivot: Point to rotate about, if omitted the
            rotation is about the origin.
        :type pivot: sequence
        :rtype: Affine
        """
        ca, sa = cos_sin_deg(angle)
        if pivot is None:
            return tuple.__new__(cls,
                (ca, sa, 0.0,
                -sa, ca, 0.0,
                 0.0, 0.0, 1.0))
        else:
            px, py = pivot
            return tuple.__new__(cls,
                (ca, sa, px - px*ca + py*sa,
                -sa, ca, py - px*sa - py*ca,
                 0.0, 0.0, 1.0))

    def __str__(self):
        """Concise string representation."""
        return ("|% .2f,% .2f,% .2f|\n"
                "|% .2f,% .2f,% .2f|\n"
                "|% .2f,% .2f,% .2f|") % self

    def __repr__(self):
        """Precise string representation."""
        return ("Affine(%r, %r, %r,\n"
                "       %r, %r, %r)") % self[:6]

    def to_gdal(self):
        return (self.c, self.a, self.b, self.f, self.d, self.e)

    @property
    def xoff(self):
        return self.c

    @property
    def yoff(self):
        return self.f

    @cached_property
    def determinant(self):
        """The determinant of the transform matrix. This value
        is equal to the area scaling factor when the transform
        is applied to a shape.
        """
        a, b, c, d, e, f, g, h, i = self
        return a*e - b*d

    @cached_property
    def is_identity(self):
        """True if this transform equals the identity matrix,
        within rounding limits.
        """
        return self is identity or self.almost_equals(identity)

    @cached_property
    def is_rectilinear(self):
        """True if the transform is rectilinear, i.e., whether a shape would
        remain axis-aligned, within rounding limits, after applying the
        transform.
        """
        a, b, c, d, e, f, g, h, i = self
        return ((abs(a) < EPSILON and abs(e) < EPSILON)
            or (abs(d) < EPSILON and abs(b) < EPSILON))

    @cached_property
    def is_conformal(self):
        """True if the transform is conformal, i.e., if angles between points
        are preserved after applying the transform, within rounding limits.
        This implies that the transform has no effective shear.
        """
        a, b, c, d, e, f, g, h, i = self
        return abs(a*b + d*e) < EPSILON

    @cached_property
    def is_orthonormal(self):
        """True if the transform is orthonormal, which means that the
        transform represents a rigid motion, which has no effective scaling or
        shear. Mathematically, this means that the axis vectors of the
        transform matrix are perpendicular and unit-length.  Applying an
        orthonormal transform to a shape always results in a congruent shape.
        """
        a, b, c, d, e, f, g, h, i = self
        return (self.is_conformal
            and abs(1.0 - (a*a + d*d)) < EPSILON
            and abs(1.0 - (b*b + e*e)) < EPSILON)

    @cached_property
    def is_degenerate(self):
        """True if this transform is degenerate, which means that it will
        collapse a shape to an effective area of zero. Degenerate transforms
        cannot be inverted.
        """
        return abs(self.determinant) < EPSILON

    @property
    def column_vectors(self):
        """The values of the transform as three 2D column vectors"""
        a, b, c, d, e, f, _, _, _ = self
        return (a, d), (b, e), (c, f)

    def almost_equals(self, other):
        """Compare transforms for approximate equality.

        :param other: Transform being compared.
        :type other: Affine
        :return: True if absolute difference between each element
            of each respective tranform matrix < ``EPSILON``.
        """
        for i in (0, 1, 2, 3, 4, 5):
            if abs(self[i] - other[i]) >= EPSILON:
                return False
        return True

    def __gt__(self, other):
        return assert_unorderable(self, other)

    __ge__ = __lt__ = __le__ = __gt__

    # Override from base class. We do not support entrywise
    # addition, subtraction or scalar multiplication because
    # the result is not an affine transform

    def __add__(self, other):
        raise TypeError("Operation not supported")

    __iadd__ = __add__

    def __mul__(self, other):
        """Apply the transform using matrix multiplication, creating a
        resulting object of the same type.  A transform may be applied to
        another transform, a vector, vector array, or shape.

        :param other: The object to transform.
        :type other: Affine, :class:`~planar.Vec2`,
            :class:`~planar.Vec2Array`, :class:`~planar.Shape`
        :rtype: Same as ``other``
        """
        sa, sb, sc, sd, se, sf, _, _, _ = self
        if isinstance(other, Affine):
            oa, ob, oc, od, oe, of, _, _, _ = other
            return tuple.__new__(Affine,
                (sa*oa + sb*od, sa*ob + sb*oe, sa*oc + sb*of + sc,
                 sd*oa + se*od, sd*ob + se*oe, sd*oc + se*of + sf,
                 0.0, 0.0, 1.0))
        else:
            try:
                vx, vy = other
            except Exception:
                return NotImplemented
            return (vx*sa + vy*sd + sc, vx*sb + vy*se + sf)

    def __rmul__(self, other):
        # We should not be called if other is an affine instance
        # This is just a guarantee, since we would potentially
        # return the wrong answer in that case
        assert not isinstance(other, Affine)
        return self.__mul__(other)

    def __imul__(self, other):
        if isinstance(other, Affine) or isinstance(other, tuple):
            return self.__mul__(other)
        else:
            return NotImplemented

    def itransform(self, seq):
        """Transform a sequence of points or vectors in place.

        :param seq: Mutable sequence of :class:`~planar.Vec2` to be
            transformed.
        :returns: None, the input sequence is mutated in place.
        """
        if self is not identity and self != identity:
            sa, sb, sc, sd, se, sf, _, _, _ = self
            for i, (x, y) in enumerate(seq):
                seq[i] = (x*sa + y*sd + sc, x*sb + y*se + sf)

    def __invert__(self):
        """Return the inverse transform.

        :raises: :except:`TransformNotInvertible` if the transform
            is degenerate.
        """
        if self.is_degenerate:
            raise TransformNotInvertibleError(
                "Cannot invert degenerate transform")
        idet = 1.0 / self.determinant
        sa, sb, sc, sd, se, sf, _, _, _ = self
        ra = se * idet
        rb = -sb * idet
        rd = -sd * idet
        re = sa * idet
        return tuple.__new__(Affine,
            (ra, rb, -sc*ra - sf*rb,
             rd, re, -sc*rd - sf*re,
             0.0, 0.0, 1.0))

    __hash__ = tuple.__hash__ # hash is not inherited in Py 3

identity = Affine(1, 0, 0, 0, 1, 0)
"""The identity transform"""

# vim: ai ts=4 sts=4 et sw=4 tw=78

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
# Copyright 2015, 2016, 2017 Guenter Bartsch
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
#
# HAL-PROLOG parser with AI specific directives and macros added
#

import os
import sys
import logging
import codecs
import re
import rdflib
from rdflib.plugins.sparql.parserutils import CompValue

from copy import copy

import model
from nltools.tokenizer import tokenize

from zamiaprolog.parser import PrologParser
from zamiaprolog.errors import PrologError
from zamiaprolog.logic  import *
from nlp_macro_engine   import NLPMacroEngine
from runtime            import AIPrologRuntime

class AIPrologParser(PrologParser):

    def __init__(self, trace = False, print_utterances=False, split_utterances=False, warn_level=0):

        super (AIPrologParser, self).__init__()

        self.trace            = trace
        self.print_utterances = print_utterances
        self.split_utterances = split_utterances
        self.warn_level       = warn_level
        self.test_cnt         = 0

        # register directives

        self.register_directive('nlp_macro',           self.nlp_macro,           None)
        self.register_directive('nlp_gen',             self.nlp_gen,             None)
        self.register_directive('nlp_test',            self.nlp_test,            None)

    def set_trace (self, trace):
        self.trace = trace

    def _get_variable(self, term):
        if not isinstance(term, Variable):
            raise PrologRuntimeError('Variable expected, %s found instead.' % term.__class__)
        return term.name

    def nlp_macro(self, module_name, clause, user_data):

        args = clause.head.args

        name = args[0].s

        macro_vars = map (lambda v: self._get_variable (v), args[1:])

        ai_rt = AIPrologRuntime(self.db, self.kb)

        solutions = ai_rt.search(clause)

        mappings = []

        for solution in solutions:

            mapping = {}
            for v in macro_vars:
                if not v in solution:
                    raise PrologError('Variable %s missing in macro %s solution' % (v, name), clause.location)
                mapping[v] = ai_rt.prolog_get_string (solution[v], {})

            mappings.append(mapping)

        self.macro_engine.define_named_macro(name, mappings, module_name, clause.location)

    # nlp_gen(de,
    #         '@HI:w @ADDRESSEE:w @VERB:w @PLEASE:w @MAL:w @WHAT:w @VERB:v',
    #         @WHAT:p, @VERB:p).

    def nlp_gen(self, module_name, clause, user_data):

        logging.debug (u'nlp_gen: %s' % clause)

        args = clause.head.args

        if len(args) < 3:
            raise PrologError (u'nlp_gen: at least 3 arguments expected: lang, input, response(s) (got: %s)' % clause, clause.location)

        lang = args[0].name

        # extract arguments

        nlp_input = args[1].s
        response = u''

        argc = 2

        while argc < len(args):

            a = args[argc]

            if isinstance (a, Predicate):
                if len(response)>0:
                    response += u';'
                response += unicode(a)

            elif isinstance (a, StringLiteral):

                if self.split_utterances:

                    # split strings into individual words
                    # generate one say(lang, word) predicate per word
                    # plus one eoa at the end to mark the end of the utterance

                    # make sure we keep punctuation when tokenizing
                    # so tts has a better chance at getting prosody right

                    for token in tokenize(a.s, lang=lang, keep_punctuation=True):

                        t = token.strip()

                        if len(t) == 0:
                            continue

                        if len(response)>0:
                            response += u';'
                        response += u'say(%s, "%s")' % (lang, t)

                    if len(response)>0:
                        response += u';'
                    response += u'eoa'

                else:
                    if len(response)>0:
                        response += u';'
                    response += u'say_eoa(%s, "%s")' % (lang, a.s)

            elif isinstance (a, MacroCall):

                if len(response)>0:
                    response += u';'
                response += u'@%s:%s' % (a.name, a.pred)

            else:

                raise PrologError (u'nlp_gen: unexpected argument: %s' % unicode(a))

            logging.debug (u'arg[%d]: %s response: %s' % (argc, repr(args[argc]), response))
            argc += 1

        # generate all macro-expansions

        logging.debug ("nlp_gen: %s: generating macro expansions..." % clause.location)

        cnt = 0

        ds = self.macro_engine.macro_expand(lang, nlp_input, response, clause.location)

        for inp, resp in ds:

            if len(inp.strip()) == 0:
                raise PrologError ('nlp_gen: empty input generated.', clause.location)

            if self.print_utterances:
                logging.info(u'inp: %s' % inp)

            if self.warn_level > 0 :
                # in an ideal world, inputs should be unique
                dr = self.db.session.query(model.DiscourseRound).filter(model.DiscourseRound.inp==inp,
                                                                        model.DiscourseRound.lang==lang).first()
                if dr:
                    msg = '%s: input not unique: found same input "%s" in module %s (r: %s vs %s)' % (clause.location, inp, dr.module, resp, dr.resp)
                    diff = resp != dr.resp

                    if self.warn_level == 1:
                        if diff:
                            logging.warning (msg)
                    elif self.warn_level == 2:
                        if diff:
                            raise PrologError(msg, clause.location)
                        else:
                            logging.warning(msg)
                    else:
                        raise PrologError(msg, clause.location)

            dr = model.DiscourseRound( lang      = lang,
                                       module    = module_name,
                                       inp       = inp,
                                       resp      = resp)
            self.db.session.add(dr)
            cnt += 1

        logging.debug ("nlp_gen: %s: %d generating macro expansions generated." % (clause.location, cnt))

    def nlp_test(self, module_name, clause, user_data):

        # store test in DB

        name = 'test_%06d' % self.test_cnt
        self.test_cnt += 1

        nlptest = model.NLPTest(module   = module_name,
                                name     = name,
                                test_src = unicode(clause),
                                location = str(clause.location))

        self.db.session.add(nlptest)

    def clear_module (self, module_name, db):

        logging.debug ('clearing discourses...')
        db.session.query(model.DiscourseRound).filter(model.DiscourseRound.module==module_name).delete()
        logging.debug ('clearing tests...')
        db.session.query(model.NLPTest).filter(model.NLPTest.module==module_name).delete()
        logging.debug ('clearing macros...')
        db.session.query(model.NLPMacro).filter(model.NLPMacro.module==module_name).delete()

        self.test_cnt = 0

        super(AIPrologParser, self).clear_module(module_name, db)

    def compile_file (self, filename, module_name, db, kb, clear_module=False):

        # setup compiler / test environment

        self.macro_engine = NLPMacroEngine(db.session)
        self.kb           = kb
        self.db           = db

        if clear_module:
            self.clear_module(module_name, db)

        super(AIPrologParser, self).compile_file(filename, module_name, db)

<commit_msg>bulk write for performance improvement<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
# Copyright 2015, 2016, 2017 Guenter Bartsch
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
#
# HAL-PROLOG parser with AI specific directives and macros added
#

import os
import sys
import logging
import codecs
import re
import rdflib
from rdflib.plugins.sparql.parserutils import CompValue

from copy import copy

import model
from nltools.tokenizer import tokenize

from zamiaprolog.parser import PrologParser
from zamiaprolog.errors import PrologError
from zamiaprolog.logic  import *
from nlp_macro_engine   import NLPMacroEngine
from runtime            import AIPrologRuntime

class AIPrologParser(PrologParser):

    def __init__(self, trace = False, print_utterances=False, split_utterances=False, warn_level=0):

        super (AIPrologParser, self).__init__()

        self.trace            = trace
        self.print_utterances = print_utterances
        self.split_utterances = split_utterances
        self.warn_level       = warn_level
        self.test_cnt         = 0

        # register directives

        self.register_directive('nlp_macro',           self.nlp_macro,           None)
        self.register_directive('nlp_gen',             self.nlp_gen,             None)
        self.register_directive('nlp_test',            self.nlp_test,            None)

    def set_trace (self, trace):
        self.trace = trace

    def _get_variable(self, term):
        if not isinstance(term, Variable):
            raise PrologRuntimeError('Variable expected, %s found instead.' % term.__class__)
        return term.name

    def nlp_macro(self, module_name, clause, user_data):

        args = clause.head.args

        name = args[0].s

        macro_vars = map (lambda v: self._get_variable (v), args[1:])

        ai_rt = AIPrologRuntime(self.db, self.kb)

        solutions = ai_rt.search(clause)

        mappings = []

        for solution in solutions:

            mapping = {}
            for v in macro_vars:
                if not v in solution:
                    raise PrologError('Variable %s missing in macro %s solution' % (v, name), clause.location)
                mapping[v] = ai_rt.prolog_get_string (solution[v], {})

            mappings.append(mapping)

        self.macro_engine.define_named_macro(name, mappings, module_name, clause.location)

    # nlp_gen(de,
    #         '@HI:w @ADDRESSEE:w @VERB:w @PLEASE:w @MAL:w @WHAT:w @VERB:v',
    #         @WHAT:p, @VERB:p).

    def nlp_gen(self, module_name, clause, user_data):

        logging.debug (u'nlp_gen: %s' % clause)

        args = clause.head.args

        if len(args) < 3:
            raise PrologError (u'nlp_gen: at least 3 arguments expected: lang, input, response(s) (got: %s)' % clause, clause.location)

        lang = args[0].name

        # extract arguments

        nlp_input = args[1].s
        response = u''

        argc = 2

        while argc < len(args):

            a = args[argc]

            if isinstance (a, Predicate):
                if len(response)>0:
                    response += u';'
                response += unicode(a)

            elif isinstance (a, StringLiteral):

                if self.split_utterances:

                    # split strings into individual words
                    # generate one say(lang, word) predicate per word
                    # plus one eoa at the end to mark the end of the utterance

                    # make sure we keep punctuation when tokenizing
                    # so tts has a better chance at getting prosody right

                    for token in tokenize(a.s, lang=lang, keep_punctuation=True):

                        t = token.strip()

                        if len(t) == 0:
                            continue

                        if len(response)>0:
                            response += u';'
                        response += u'say(%s, "%s")' % (lang, t)

                    if len(response)>0:
                        response += u';'
                    response += u'eoa'

                else:
                    if len(response)>0:
                        response += u';'
                    response += u'say_eoa(%s, "%s")' % (lang, a.s)

            elif isinstance (a, MacroCall):

                if len(response)>0:
                    response += u';'
                response += u'@%s:%s' % (a.name, a.pred)

            else:

                raise PrologError (u'nlp_gen: unexpected argument: %s' % unicode(a))

            logging.debug (u'arg[%d]: %s response: %s' % (argc, repr(args[argc]), response))
            argc += 1

        # generate all macro-expansions

        logging.debug ("nlp_gen: %s: generating macro expansions..." % clause.location)

        cnt = 0

        ds = self.macro_engine.macro_expand(lang, nlp_input, response, clause.location)

        rounds = []

        for inp, resp in ds:

            if len(inp.strip()) == 0:
                raise PrologError ('nlp_gen: empty input generated.', clause.location)

            if self.print_utterances:
                logging.info(u'inp: %s' % inp)

            if self.warn_level > 0 :
                # in an ideal world, inputs should be unique
                dr = self.db.session.query(model.DiscourseRound).filter(model.DiscourseRound.inp==inp,
                                                                        model.DiscourseRound.lang==lang).first()
                if dr:
                    msg = '%s: input not unique: found same input "%s" in module %s (r: %s vs %s)' % (clause.location, inp, dr.module, resp, dr.resp)
                    diff = resp != dr.resp

                    if self.warn_level == 1:
                        if diff:
                            logging.warning (msg)
                    elif self.warn_level == 2:
                        if diff:
                            raise PrologError(msg, clause.location)
                        else:
                            logging.warning(msg)
                    else:
                        raise PrologError(msg, clause.location)

            rounds.append(model.DiscourseRound( lang      = lang,
                                                module    = module_name,
                                                inp       = inp,
                                                resp      = resp))
            cnt += 1

        logging.debug (u'nlp_gen: %s bulk save...' % clause)
        self.db.session.bulk_save_objects(rounds)

        logging.debug ("nlp_gen: %s: %d generating macro expansions generated." % (clause.location, cnt))

    def nlp_test(self, module_name, clause, user_data):

        # store test in DB

        name = 'test_%06d' % self.test_cnt
        self.test_cnt += 1

        nlptest = model.NLPTest(module   = module_name,
                                name     = name,
                                test_src = unicode(clause),
                                location = str(clause.location))

        self.db.session.add(nlptest)

    def clear_module (self, module_name, db):

        logging.debug ('clearing discourses...')
        db.session.query(model.DiscourseRound).filter(model.DiscourseRound.module==module_name).delete()
        logging.debug ('clearing tests...')
        db.session.query(model.NLPTest).filter(model.NLPTest.module==module_name).delete()
        logging.debug ('clearing macros...')
        db.session.query(model.NLPMacro).filter(model.NLPMacro.module==module_name).delete()

        self.test_cnt = 0

        super(AIPrologParser, self).clear_module(module_name, db)

    def compile_file (self, filename, module_name, db, kb, clear_module=False):

        # setup compiler / test environment

        self.macro_engine = NLPMacroEngine(db.session)
        self.kb           = kb
        self.db           = db

        if clear_module:
            self.clear_module(module_name, db)

        super(AIPrologParser, self).compile_file(filename, module_name, db)

<|endoftext|>


<commit_before>import logging

def tweak(words):
    # Strip silence from ends of words
    for wd in words:
        if len(wd['phones']) > 0 and wd['phones'][0]['phone'] == 'sil':
            p = wd['phones'].pop(0)
            wd['start'] += p['duration']
            wd['duration'] -= p['duration']

        if len(wd['phones']) > 0 and wd['phones'][-1]['phone'] == 'sil':
            p = wd['phones'].pop()
            wd['duration'] -= p['duration']

    # Move mis-aligned phonemes
    last_wd = None
    next_phonemes = []
    for wd in words:
        if len(next_phonemes) > 0:
            # Add the previous phonemes
            duration = sum([X['duration'] for X in next_phonemes])

            # Merge if the edge is identical
            if len(wd['phones']) > 0 and next_phonemes[-1]['phone'] == wd['phones'][0]['phone']:
                wd['phones'][0]['duration'] += next_phonemes[-1]['duration']
                next_phonemes = next_phonemes[:-1]

            wd['phones'] = next_phonemes + wd['phones']
            wd['start'] -= duration
            wd['duration'] += duration

            next_phonemes = []

        # Did we get the end of the last word
        elif len(wd['phones']) > 1 \
             and last_wd is not None \
             and wd['phones'][0]['phone'].endswith('_E') \
             and ((not last_wd['phones'][-1]['phone'].endswith('_E'))
                  or last_wd['phones'][-1]['phone'] == wd['phones'][0]['phone']):

            logging.info('moving the end back')
            first_phone = wd['phones'][0]
            wd['phones'] = wd['phones'][1:]
            wd['duration'] -= first_phone['duration']
            wd['start'] += first_phone['duration']

            if last_wd['phones'][-1]['phone'] == first_phone['phone']:
                last_wd['phones'][-1]['duration'] += first_phone['duration']
            else:
                last_wd['phones'].append(first_phone)
            last_wd['duration'] += first_phone['duration']

        if len(wd['phones']) > 1:
            if not (wd['phones'][0]['phone'].endswith('_B') or wd['phones'][0]['phone'].endswith('_S')):
                logging.info('Word does not start correctly, %s', str(wd))
                logging.info('previously: %s', str(last_wd))
                # XXX: In this (rare!) case, I think getting rid of
                # the starting phoneme may be the right thing to do
                # (if, say, the second looks reasonable).
                #
                # I don't think this case should happen under normal
                # circumstance--it's only happening in debugging at
                # the beginning of the 20-second intervals.

            else:
                # Is the beginning of the next word here?
                for idx,ph in enumerate(wd['phones']):
                    if idx > 0 and ph['phone'].endswith('_B'):
                        next_phonemes = wd['phones'][idx:]
                        if len(next_phonemes) > 3:
                            logging.info("Trying to move many phonemes (%d): %s", len(next_phonemes), str(wd))
                        wd['phones'] = wd['phones'][:idx]
                        wd['duration'] -= sum([X['duration'] for X in next_phonemes])
                        logging.info('Word contains the next beginning (%d)', idx)
                        break

        last_wd = wd

    # Strip silence from ends of words
    for wd in words:
        if len(wd['phones']) > 0 and wd['phones'][0]['phone'] == 'sil':
            p = wd['phones'].pop(0)
            wd['start'] += p['duration']
            wd['duration'] -= p['duration']

        if len(wd['phones']) > 0 and wd['phones'][-1]['phone'] == 'sil':
            p = wd['phones'].pop()
            wd['duration'] -= p['duration']

    return words

if __name__=='__main__':
    import json
    import sys
    logging.getLogger().setLevel(logging.DEBUG)

    IN_JSON = sys.argv[1]
    OUT_JSON = sys.argv[2]

    inp = json.load(open(IN_JSON))

    words = [X for X in inp['words'] if X['case'] != 'not-found-in-audio']

    # ugh. normalize to start/duration
    for wd in words:
        wd['duration'] = wd['end'] - wd['start']

    words = tweak(words)

    # ...and back to start/end
    for wd in words:
        wd['end'] = wd['duration'] + wd['start']

    inp['words'] = words

    json.dump(inp, open(OUT_JSON, 'w'), indent=2)
<commit_msg>better handling of singleton phonemes<commit_after>import logging

def tweak(words):
    # Strip silence from ends of words
    for wd in words:
        if len(wd['phones']) > 0 and wd['phones'][0]['phone'] == 'sil':
            p = wd['phones'].pop(0)
            wd['start'] += p['duration']
            wd['duration'] -= p['duration']

        if len(wd['phones']) > 0 and wd['phones'][-1]['phone'] == 'sil':
            p = wd['phones'].pop()
            wd['duration'] -= p['duration']

    # Move mis-aligned phonemes
    last_wd = None
    next_phonemes = []
    for wd in words:
        if len(next_phonemes) > 0:
            # Add the previous phonemes
            duration = sum([X['duration'] for X in next_phonemes])

            # Merge if the edge is identical
            if len(wd['phones']) > 0 and next_phonemes[-1]['phone'] == wd['phones'][0]['phone']:
                wd['phones'][0]['duration'] += next_phonemes[-1]['duration']
                next_phonemes = next_phonemes[:-1]

            wd['phones'] = next_phonemes + wd['phones']
            wd['start'] -= duration
            wd['duration'] += duration

            next_phonemes = []

        # Did we get the end of the last word
        elif len(wd['phones']) >= 1 \
             and last_wd is not None \
             and wd['phones'][0]['phone'][-1] in 'ES' \
             and (len(last_wd['phones']) == 0 or \
                  (not last_wd['phones'][-1]['phone'].endswith('_E')) \
                   or last_wd['phones'][-1]['phone'] == wd['phones'][0]['phone']):

            logging.info('moving the end back')
            logging.info('last_wd: %s', str(last_wd))
            logging.info('wd: %s', str(wd))

            first_phone = wd['phones'][0]
            wd['phones'] = wd['phones'][1:]
            wd['duration'] -= first_phone['duration']
            wd['start'] += first_phone['duration']

            if len(last_wd['phones']) > 0 and last_wd['phones'][-1]['phone'] == first_phone['phone']:
                last_wd['phones'][-1]['duration'] += first_phone['duration']
            else:
                last_wd['phones'].append(first_phone)
            last_wd['duration'] += first_phone['duration']

        if len(wd['phones']) > 1:
            if not (wd['phones'][0]['phone'].endswith('_B') or wd['phones'][0]['phone'].endswith('_S')):
                logging.info('Word does not start correctly, %s', str(wd))
                logging.info('previously: %s', str(last_wd))
                # XXX: In this (rare!) case, I think getting rid of
                # the starting phoneme may be the right thing to do
                # (if, say, the second looks reasonable).
                #
                # I don't think this case should happen under normal
                # circumstance--it's only happening in debugging at
                # the beginning of the 20-second intervals.

            else:
                # Is the beginning of the next word here?
                for idx,ph in enumerate(wd['phones']):
                    if idx > 0 and ph['phone'].endswith('_B'):
                        next_phonemes = wd['phones'][idx:]
                        if len(next_phonemes) > 3:
                            logging.warn("Trying to move many phonemes (%d): %s", len(next_phonemes), str(wd))
                            continue
                        wd['phones'] = wd['phones'][:idx]
                        wd['duration'] -= sum([X['duration'] for X in next_phonemes])
                        logging.info('Word contains the next beginning (%d)', idx)
                        break

        last_wd = wd

    # Strip silence from ends of words
    for wd in words:
        if len(wd['phones']) > 0 and wd['phones'][0]['phone'] == 'sil':
            p = wd['phones'].pop(0)
            wd['start'] += p['duration']
            wd['duration'] -= p['duration']

        if len(wd['phones']) > 0 and wd['phones'][-1]['phone'] == 'sil':
            p = wd['phones'].pop()
            wd['duration'] -= p['duration']

    return words

if __name__=='__main__':
    import json
    import sys
    logging.getLogger().setLevel(logging.DEBUG)

    IN_JSON = sys.argv[1]
    OUT_JSON = sys.argv[2]

    inp = json.load(open(IN_JSON))

    words = [X for X in inp['words'] if X['case'] != 'not-found-in-audio']

    # ugh. normalize to start/duration
    for wd in words:
        wd['duration'] = wd['end'] - wd['start']

    words = tweak(words)

    # ...and back to start/end
    for wd in words:
        wd['end'] = wd['duration'] + wd['start']

    inp['words'] = words

    json.dump(inp, open(OUT_JSON, 'w'), indent=2)
<|endoftext|>


<commit_before>import subprocess
import sys

from git import Repository

try:
    from termcolor import colored
except ImportError:
    print 'You should run "pip install termcolor" to fully utilize these utilities.'

    def colored(text, *args, **kwargs):
        print text

CONTENT_LENGTHS = {}
def content_length(key, text=None):
    CONTENT_LENGTHS[key] = max(CONTENT_LENGTHS.get(key, 0), len(text or ''))
    return CONTENT_LENGTHS[key]

def get_status():
    repo = Repository()
    repo.fetch()
    branches = repo.branches('origin')
    tags = repo.tags()

    branches = filter(lambda x: x != 'master', branches)
    branch_states = []

    for branch in branches:
        parent = branch in ('staging', 'master',) and 'master' or 'staging'

        color, status, review_commits, ahead_commits, behind_commits = 'red', '?', [], [], []

        ahead_commits = repo.git('log', '--pretty=format:"- %s [%h]"', 'origin/%s..origin/%s' % (parent, branch), split=True)
        behind_commits = repo.git('log', '--pretty=format:"- %s [%h]"', 'origin/%s..origin/%s' % (branch, parent), split=True)

        if "%s-codereview" % branch not in tags:
            color, status = 'red', 'new'
            review_commits = ahead_commits
        else:
            review_commits = repo.git('log', '--pretty=format:"- %s [%h]"', '%s-codereview..origin/%s' % (branch, branch), split=True)
            if review_commits:
                color, status = 'red', 'review'
            else:
                if ahead_commits:
                    color, status = 'yellow', 'merge'
                else:
                    color, status = 'green', 'done'

        branch_states.append({
            'status': status,
            'color': color,
            'branch': branch,
            'ahead_commits': ahead_commits,
            'behind_commits': behind_commits,
            'review_commits': review_commits,
            })

        # Track length of longest string for each column
        content_length('status', status)
        content_length('branch', branch)

    # Print out pretty upper container
    print '-%s-%s-%s-%s-%s-' % (
        '-'.join([ '' for x in range(content_length('status') + len('  ') + 1) ]),
        '-'.join([ '' for x in range(content_length('branch') + len('  ') + 1) ]),
        '-'.join([ '' for x in range(16 + 1) ]),
        '-'.join([ '' for x in range(11 + 1) ]),
        '-'.join([ '' for x in range(12 + 1) ]),
        )

    # Print out status for each branch
    for s in branch_states:
        review_commits = len(s['review_commits'])
        ahead_commits = len(s['ahead_commits'])
        behind_commits = len(s['behind_commits'])
        color = s['color']
        branch = s['branch']
        status = s['status']

        print '|%s|%s|%s|%s|%s|' % (
            colored(' %s ' % status.upper().rjust(content_length('status')), color),
            ' %s ' % branch.ljust(content_length('branch')),
            colored((' %s unreviewed ' % review_commits).rjust(16), review_commits and color or None, attrs=review_commits and ['reverse'] or []),
            colored((' %s ahead ' % ahead_commits).rjust(11), ahead_commits and color or None, attrs=ahead_commits and ['reverse'] or []),
            colored((' %s behind ' % behind_commits).rjust(12), behind_commits and color or None, attrs=behind_commits and ['reverse'] or []),
            )

    # Print out pretty lower container
    print '-%s-%s-%s-%s-%s-' % (
        '-'.join([ '' for x in range(content_length('status') + len('  ') + 1) ]),
        '-'.join([ '' for x in range(content_length('branch') + len('  ') + 1) ]),
        '-'.join([ '' for x in range(16 + 1) ]),
        '-'.join([ '' for x in range(11 + 1) ]),
        '-'.join([ '' for x in range(12 + 1) ]),
        )

def complete_review():
    repo = Repository()
    repo.fetch()
    branch = repo.branch()
    repo.git('tag', '-a', '%s-codereview' % branch, '-f', '-m', 'creating code review for branch %s' % branch)
    repo.git('push', '--tags')
    print 'Created tag %s-codereview' % branch
    repo.git('checkout', 'staging')
    print colored('Switched back to staging branch.', 'yellow')
<commit_msg>Updated codereview.<commit_after>import subprocess
import sys

from git import Repository

try:
    from termcolor import colored
except ImportError:
    print 'You should run "pip install termcolor" to fully utilize these utilities.'

    def colored(text, *args, **kwargs):
        print text

CONTENT_LENGTHS = {}
def content_length(key, text=None):
    CONTENT_LENGTHS[key] = max(CONTENT_LENGTHS.get(key, 0), len(text or ''))
    return CONTENT_LENGTHS[key]

def get_status():
    repo = Repository()
    repo.fetch()
    branches = repo.branches('origin')
    local_branches = repo.branches()
    tags = repo.tags()

    branches = filter(lambda x: x != 'master', branches)
    branch_states = []

    for branch in branches:
        parent = branch in ('staging', 'master',) and 'master' or 'staging'

        color, status, review_commits, ahead_commits, behind_commits, local_ahead_commits, local_behind_commits= 'red', '?', [], [], [], None, None

        ahead_commits = repo.git('log', '--pretty=format:"- %s [%h]"', 'origin/%s..origin/%s' % (parent, branch), split=True)
        behind_commits = repo.git('log', '--pretty=format:"- %s [%h]"', 'origin/%s..origin/%s' % (branch, parent), split=True)
        if branch in local_branches:
            local_ahead_commits = repo.git('log', '--pretty=format:"- %s [%h]"', 'origin/%s..%s' % (branch, branch), split=True)
            local_behind_commits = repo.git('log', '--pretty=format:"- %s [%h]"', '%s..origin/%s' % (branch, branch), split=True)

        if "%s-codereview" % branch not in tags:
            color, status = 'red', 'new'
            review_commits = ahead_commits
        else:
            review_commits = repo.git('log', '--pretty=format:"- %s [%h]"', '%s-codereview..origin/%s' % (branch, branch), split=True)
            if review_commits:
                color, status = 'red', 'review'
            else:
                if ahead_commits:
                    color, status = 'yellow', 'merge'
                else:
                    color, status = 'green', 'done'

        branch_states.append({
            'status': status,
            'color': color,
            'branch': branch,
            'review_commits': review_commits,
            'ahead_commits': ahead_commits,
            'behind_commits': behind_commits,
            'local_ahead_commits': local_ahead_commits,
            'local_behind_commits': local_behind_commits,
            })

        # Track length of longest string for each column
        content_length('status', status)
        content_length('branch', branch)

    print u'|%s|%s|%s|%s|%s|%s|%s|' % (
        u' Status '.ljust(content_length('status', 'status') + len('  ')),
        u' Branch '.ljust(content_length('branch', 'branch') + len('  ')),
        u' Review '.ljust(16),
        u' Ahead '.ljust(11),
        u' Behind '.ljust(12),
        u' Pull ',
        u' Push ',
        )

    # Print out pretty upper container
    print '-%s-%s-%s-%s-%s-%s-%s-' % (
        u'-'.join([ u'' for x in range(content_length('status') + len('  ') + 1) ]),
        u'-'.join([ u'' for x in range(content_length('branch') + len('  ') + 1) ]),
        u'-'.join([ u'' for x in range(16 + 1) ]),
        u'-'.join([ u'' for x in range(11 + 1) ]),
        u'-'.join([ u'' for x in range(12 + 1) ]),
        u'-'.join([ u'' for x in range(6 + 1) ]),
        u'-'.join([ u'' for x in range(6 + 1) ]),
        )

    # Print out status for each branch
    for s in branch_states:
        review_commits = len(s['review_commits'])
        ahead_commits = len(s['ahead_commits'])
        behind_commits = len(s['behind_commits'])
        color = s['color']
        branch = s['branch']
        status = s['status']
        pull = bool(s['local_behind_commits'])
        push = bool(s['local_ahead_commits'])
        not_local = s['local_behind_commits'] is None

        print u'|%s|%s|%s|%s|%s|%s|%s|' % (
            colored(u' %s ' % status.upper().rjust(content_length('status')), color),
            u' %s ' % branch.ljust(content_length('branch')),
            colored((u' %s unreviewed ' % review_commits).rjust(16), review_commits and color or None, attrs=review_commits and ['reverse'] or []),
            colored((u' %s ahead ' % ahead_commits).rjust(11), ahead_commits and color or None, attrs=ahead_commits and ['reverse'] or []),
            colored((u' %s behind ' % behind_commits).rjust(12), behind_commits and color or None, attrs=behind_commits and ['reverse'] or []),
            not_local and colored(u'  \u2049   ', 'yellow') or (pull and colored(u'  \u2718   ', 'red') or colored(u'  \u2714   ', 'green')),
            not_local and colored(u'  \u2049   ', 'yellow') or (push and colored(u'  \u2718   ', 'red') or colored(u'  \u2714   ', 'green')),
            )

    # Print out pretty lower container
    print u'-%s-%s-%s-%s-%s-%s-%s-' % (
        u'-'.join([ u'' for x in range(content_length('status') + len('  ') + 1) ]),
        u'-'.join([ u'' for x in range(content_length('branch') + len('  ') + 1) ]),
        u'-'.join([ u'' for x in range(16 + 1) ]),
        u'-'.join([ u'' for x in range(11 + 1) ]),
        u'-'.join([ u'' for x in range(12 + 1) ]),
        u'-'.join([ u'' for x in range(6 + 1) ]),
        u'-'.join([ u'' for x in range(6 + 1) ]),
        )

def complete_review():
    repo = Repository()
    repo.fetch()
    branch = repo.branch()
    repo.git('tag', '-a', '%s-codereview' % branch, '-f', '-m', 'creating code review for branch %s' % branch)
    repo.git('push', '--tags')
    print 'Created tag %s-codereview' % branch
    repo.git('checkout', 'staging')
    print colored('Switched back to staging branch.', 'yellow')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

version = '1.10.5'
<commit_msg>[AIRFLOW-XXX] Update version to 1.10.6<commit_after># -*- coding: utf-8 -*-
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

version = '1.10.6'
<|endoftext|>


<commit_before>#!/usr/bin/env python
__author__ = 'alex'
import pymongo
import pickle

# load pickle file
print "Loading season mapping pickle file...\n"
subject_season_map = pickle.load( open( "subject_season_map.p", "rb" ) )

print "Scanning classifications db..."
# connect to the mongo server
client = pymongo.MongoClient()
db = client['serengeti3']
classification_collection = db["serengeti_classifications"]

# for storing the users seen for each season
known_users = {}
anon_users_counts = {}

# scan through *ALL* classifications (add a .skip or .limit to look at subset)
pageSize = 100000
first_classification = classification_collection.find_one()
completed_page_rows=1
last_id = first_classification["_id"]

next_results = classification_collection.find({"_id":{"$gt":last_id}},{"user_name":1,"subjects":1},no_cursor_timeout=True).limit(pageSize)
while next_results.count()>0:
  for ii, classification in enumerate(next_results):
    completed_page_rows+=1
    if completed_page_rows % pageSize == 0:
      print "%s (id = %s)" % (completed_page_rows,classification["_id"])
    last_id = classification["_id"]
    if "tutorial" in classification:
      continue
    else:
      subject_id = classification["subjects"][0]["zooniverse_id"]
      season = subject_season_map[subject_id]
      if "user_name" in classification:
        user_name = classification["user_name"]
        if season not in known_users.keys():
          known_users[season]=[]
        if user_name not in known_users[season]:
          known_users[season].append(user_name)
      else:
        if season not in anon_users_counts.keys():
          anon_users_counts[season]=0
        anon_users_counts[season] += 1
  next_results = classification_collection.find({"_id":{"$gt":last_id}},{"user_name":1,"subjects":1},no_cursor_timeout=True).limit(pageSize)

print "Saving known users and anonymous users..."

pickle.dump( known_users, open( "known_users.p", "wb" ) )
pickle.dump( anon_users_counts, open( "anon_users_counts.p", "wb" ) )<commit_msg>ensure proper skipping of tutorial images<commit_after>#!/usr/bin/env python
__author__ = 'alex'
import pymongo
import pickle

# load pickle file
print "Loading season mapping pickle file...\n"
subject_season_map = pickle.load( open( "subject_season_map.p", "rb" ) )

print "Scanning classifications db..."
# connect to the mongo server
client = pymongo.MongoClient()
db = client['serengeti3']
classification_collection = db["serengeti_classifications"]

# for storing the users seen for each season
known_users = {}
anon_users_counts = {}

# scan through *ALL* classifications (add a .skip or .limit to look at subset)
pageSize = 100000
first_classification = classification_collection.find_one()
completed_page_rows=1
last_id = first_classification["_id"]

next_results = classification_collection.find({"_id":{"$gt":last_id}},{"user_name":1,"subjects":1},no_cursor_timeout=True).limit(pageSize)
while next_results.count()>0:
  for ii, classification in enumerate(next_results):
    completed_page_rows+=1
    if completed_page_rows % pageSize == 0:
      print "%s (id = %s)" % (completed_page_rows,classification["_id"])
    last_id = classification["_id"]
    next_results = classification_collection.find({"_id":{"$gt":last_id}},{"user_name":1,"subjects":1},no_cursor_timeout=True).limit(pageSize)
    if "tutorial" in classification:
      continue
    else:
      subject_id = classification["subjects"][0]["zooniverse_id"]
      season = subject_season_map[subject_id]
      if "user_name" in classification:
        user_name = classification["user_name"]
        if season not in known_users.keys():
          known_users[season]=[]
        if user_name not in known_users[season]:
          known_users[season].append(user_name)
      else:
        if season not in anon_users_counts.keys():
          anon_users_counts[season]=0
        anon_users_counts[season] += 1

print "Saving known users and anonymous users..."

pickle.dump( known_users, open( "known_users.p", "wb" ) )
pickle.dump( anon_users_counts, open( "anon_users_counts.p", "wb" ) )<|endoftext|>


<commit_before>#!/usr/bin/env python

import os

cur_dir = os.getcwd()
get_summary = sum(os.path.getsize(f) for f in os.listdir('.') if os.path.isfile(f))

def get_files(dirname, reverse=True):
    # Getting our list of files
    filepaths = []
    for basename in os.listdir(dirname):
        filename = os.path.join(dirname, basename)
        if os.path.isfile(filename):
            filepaths.append(filename)

    for i in xrange(len(filepaths)):
        filepaths[i] = (filepaths[i], os.path.getsize(filepaths[i]))

    # Sorting our list by file size, largest to smallest
    filepaths.sort(key=lambda filename: filename[1], reverse=reverse)

    return filepaths

print "Total Size: ", get_summary, "bytes"
print get_files(cur_dir)

#########################################################

"""
# A slightly quicker, but messier, method
import os

cur_dir = os.getcwd()
d = os.listdir(cur_dir)

print "Total Size: ", sum(os.path.getsize(f) for f in os.listdir('.') if os.path.isfile(f)), "bytes"

for x in d:
	print x, os.path.getsize(x), "bytes"
"""
<commit_msg>	modified:   get_filesize.py<commit_after>#!/usr/bin/env python

import os

cur_dir = os.getcwd()
get_summary = sum(os.path.getsize(f) for f in os.listdir('.') if os.path.isfile(f))

def get_files(dirname, reverse=True):
    # Getting our list of files
    filepaths = []
    for basename in os.listdir(dirname):
        filename = os.path.join(dirname, basename)
        if os.path.isfile(filename):
            filepaths.append(filename)

    for i in xrange(len(filepaths)):
        filepaths[i] = (filepaths[i], os.path.getsize(filepaths[i]))

    # Sorting our list by file size, largest to smallest
    filepaths.sort(key=lambda filename: filename[1], reverse=reverse)

    return filepaths

print "Total Size: ", get_summary, "bytes"'\n'
print get_files(cur_dir)

#########################################################

"""
# A slightly quicker, but messier, method
import os

cur_dir = os.getcwd()
d = os.listdir(cur_dir)

print "Total Size: ", sum(os.path.getsize(f) for f in os.listdir('.') if os.path.isfile(f)), "bytes"

for x in d:
	print x, os.path.getsize(x), "bytes"
"""
<|endoftext|>


<commit_before>from django.conf.urls import patterns, url, include
from getpaid.views import NewPaymentView, FallbackView
from getpaid.utils import import_backend_modules

includes_list = []
for backend_name, urls in list(import_backend_modules('urls').items()):
    includes_list.append(url(r'^%s/' % backend_name, include(urls)))

urlpatterns = patterns('',
    url(r'^new/payment/(?P<currency>[A-Z]{3})/$', NewPaymentView.as_view(), name='getpaid-new-payment'),
    url(r'^payment/success/(?P<pk>\d+)/$', FallbackView.as_view(success=True), name='getpaid-success-fallback'),
    url(r'^payment/failure/(?P<pk>\d+)$', FallbackView.as_view(success=False), name='getpaid-failure-fallback'),
    *includes_list

)
<commit_msg>1.9 deprecations removal<commit_after>from django.conf.urls import patterns, url, include
from getpaid.views import NewPaymentView, FallbackView
from getpaid.utils import import_backend_modules

urlpatterns = [
    url(r'^new/payment/(?P<currency>[A-Z]{3})/$', NewPaymentView.as_view(), name='getpaid-new-payment'),
    url(r'^payment/success/(?P<pk>\d+)/$', FallbackView.as_view(success=True), name='getpaid-success-fallback'),
    url(r'^payment/failure/(?P<pk>\d+)$', FallbackView.as_view(success=False), name='getpaid-failure-fallback'),
]

for backend_name, urls in list(import_backend_modules('urls').items()):
    urlpatterns.append(url(r'^%s/' % backend_name, include(urls)))
<|endoftext|>


<commit_before># coding=utf-8

"""
Collect stats from postfix-stats. postfix-stats is a simple threaded stats
aggregator for Postfix. When running as a syslog destination, it can be used to
get realtime cumulative stats.

#### Dependencies

 * json
 * socket
 * StringIO
 * [postfix-stats](https://github.com/disqus/postfix-stats)

"""

import diamond.collector
import socket
from StringIO import StringIO

try:
    import json
except ImportError:
    import simplejson as json

class PostfixCollector(diamond.collector.Collector):

    def get_default_config_help(self):
        config_help = super(PostfixCollector, self).get_default_config_help()
        config_help.update({
            'host': 'Hostname to coonect to',
            'port': 'Port to connect to',
        })
        return config_help

    def get_default_config(self):
        """
        Returns the default collector settings
        """
        config = super(PostfixCollector, self).get_default_config()
        config.update({
            'path':     'postfix',
            'host':     'localhost',
            'port':     7777,
        })
        return config

    def getJson(self):
        try:
            sock = socket.socket( socket.AF_INET, socket.SOCK_STREAM )
            sock.connect( (self.config['host'], int(self.config['port'])) )
            sock.send("stats\n")
            jsondata = sock.recv(4096)
            sock.close()
            return jsondata
        except socket.error:
            return None

    def getData(self):
        jsondata = self.getJson()
        io = StringIO(jsondata)
        data = json.load(io)
        return data

    def collect(self):
        data = self.getData()

        if not data:
            return

        for nodetype in data.keys():
            for nodesubtype in data[nodetype].keys():
                for metric in data[nodetype][nodesubtype].keys():

                    # End metrics are sometimes codes like 2.0.0
                    metricname = metric.replace('.', '_')

                    metric_name = "%s.%s.%s" % (nodetype, nodesubtype, metricname)
                    metric_value = data[nodetype][nodesubtype][metric]

                    metric_value = self.derivative(metric_name, metric_value)

                    self.publish(metric_name, metric_value)
<commit_msg>Catch an error when run before a email has been sent take 2<commit_after># coding=utf-8

"""
Collect stats from postfix-stats. postfix-stats is a simple threaded stats
aggregator for Postfix. When running as a syslog destination, it can be used to
get realtime cumulative stats.

#### Dependencies

 * json
 * socket
 * StringIO
 * [postfix-stats](https://github.com/disqus/postfix-stats)

"""

import diamond.collector
import socket
from StringIO import StringIO

try:
    import json
except ImportError:
    import simplejson as json

class PostfixCollector(diamond.collector.Collector):

    def get_default_config_help(self):
        config_help = super(PostfixCollector, self).get_default_config_help()
        config_help.update({
            'host': 'Hostname to coonect to',
            'port': 'Port to connect to',
        })
        return config_help

    def get_default_config(self):
        """
        Returns the default collector settings
        """
        config = super(PostfixCollector, self).get_default_config()
        config.update({
            'path':     'postfix',
            'host':     'localhost',
            'port':     7777,
        })
        return config

    def getJson(self):
        try:
            sock = socket.socket( socket.AF_INET, socket.SOCK_STREAM )
            sock.connect( (self.config['host'], int(self.config['port'])) )
            sock.send("stats\n")
            jsondata = sock.recv(4096)
            sock.close()
            return jsondata
        except socket.error:
            return None

    def getData(self):
        try:
            jsondata = self.getJson()
            io = StringIO(jsondata)
            data = json.load(io)
            return data
        except ValueError:
            return None

    def collect(self):
        data = self.getData()

        if not data:
            return

        for nodetype in data.keys():
            for nodesubtype in data[nodetype].keys():
                for metric in data[nodetype][nodesubtype].keys():

                    # End metrics are sometimes codes like 2.0.0
                    metricname = metric.replace('.', '_')

                    metric_name = "%s.%s.%s" % (nodetype, nodesubtype, metricname)
                    metric_value = data[nodetype][nodesubtype][metric]

                    metric_value = self.derivative(metric_name, metric_value)

                    self.publish(metric_name, metric_value)
<|endoftext|>


<commit_before>from Queue import Queue

class Iterator(object):
    def __init__(self, source, operation=lambda message: [message]):
        if isinstance(source, list):
            self.source = iter(source).__iter__()
        else:
            self.source = source
        self.operation = operation
        self.queue = Queue()

    def __iter__(self):
        return self

    def next(self):
        if not self.queue.empty():
            return self.queue.get()
        for message in self.operation(self.source.next()):
            self.queue.put(message)
        return self.next()

class TestCase(object):
    def test_iter(self):
        iterator = Iterator([1, 2, 3])

        assert [1, 2, 3] == [n for n in iterator]

    def test_double_iter(self):
        iterator = Iterator(Iterator([1, 2, 3]))

        assert [1, 2, 3] == [n for n in iterator]

    def test_apply(self):
        iterator = Iterator([1, 2, 3], operation=lambda x: [2 * x])

        assert [2, 4, 6] == [n for n in iterator]
<commit_msg>basic and effect iterators<commit_after>from Queue import Queue

class Iterator(object):
    def __init__(self, source, operation=lambda message: [message]):
        if isinstance(source, list):
            self.source = iter(source).__iter__()
        else:
            self.source = source
        self.operation = operation
        self.queue = Queue()

    def __iter__(self):
        return self

    def next(self):
        if not self.queue.empty():
            return self.queue.get()
        self.queue.put(self.source.next())
        return self.next()

class OperationIterator(Iterator):
    def next(self):
        if not self.queue.empty():
            return self.queue.get()
        for message in self.operation(self.source.next()):
            self.queue.put(message)
        return self.next()

class TestCase(object):
    def test_iter(self):
        iterator = Iterator([1, 2, 3])

        assert [1, 2, 3] == [n for n in iterator]

    def test_double_iter(self):
        iterator = Iterator(Iterator([1, 2, 3]))

        assert [1, 2, 3] == [n for n in iterator]

    def test_apply(self):
        iterator = OperationIterator([1, 2, 3], operation=lambda x: [2 * x])

        assert [2, 4, 6] == [n for n in iterator]

    def test_filter(self):
        pass
<|endoftext|>


<commit_before>from typing import Optional, List

from rx import from_future
from rx.core import Observable, typing
from rx.disposable import CompositeDisposable, SingleAssignmentDisposable
from rx.internal.utils import is_future

# pylint: disable=redefined-builtin

def _zip(*args: Observable) -> Observable:
    """Merges the specified observable sequences into one observable
    sequence by creating a tuple whenever all of the
    observable sequences have produced an element at a corresponding
    index.

    Example:
        >>> res = zip(obs1, obs2)

    Args:
        args: Observable sources to zip.

    Returns:
        An observable sequence containing the result of combining
        elements of the sources as tuple.
    """

    sources = list(args)

    def subscribe(observer: typing.Observer, scheduler: Optional[typing.Scheduler] = None):
        n = len(sources)
        queues : List[List] = [[] for _ in range(n)]
        is_done = [False] * n

        def next(i):
            if all([len(q) for q in queues]):
                try:
                    queued_values = [x.pop(0) for x in queues]
                    res = tuple(queued_values)
                except Exception as ex:  # pylint: disable=broad-except
                    observer.on_error(ex)
                    return

                observer.on_next(res)
            elif all([x for j, x in enumerate(is_done) if j != i]):
                observer.on_completed()

        def done(i):
            is_done[i] = True
            if all(is_done):
                observer.on_completed()

        subscriptions = [None]*n

        def func(i):
            source = sources[i]
            sad = SingleAssignmentDisposable()
            source = from_future(source) if is_future(source) else source

            def on_next(x):
                queues[i].append(x)
                next(i)

            sad.disposable = source.subscribe_(on_next, observer.on_error, lambda: done(i), scheduler)
            subscriptions[i] = sad
        for idx in range(n):
            func(idx)
        return CompositeDisposable(subscriptions)
    return Observable(subscribe)
<commit_msg>Fix race condition in zip operator (#506)<commit_after>from typing import Optional, List

from rx import from_future
from rx.core import Observable, typing
from rx.disposable import CompositeDisposable, SingleAssignmentDisposable
from rx.internal.utils import is_future
from rx.internal.concurrency import synchronized
from threading import RLock

# pylint: disable=redefined-builtin

def _zip(*args: Observable) -> Observable:
    """Merges the specified observable sequences into one observable
    sequence by creating a tuple whenever all of the
    observable sequences have produced an element at a corresponding
    index.

    Example:
        >>> res = zip(obs1, obs2)

    Args:
        args: Observable sources to zip.

    Returns:
        An observable sequence containing the result of combining
        elements of the sources as tuple.
    """

    sources = list(args)

    def subscribe(observer: typing.Observer, scheduler: Optional[typing.Scheduler] = None):
        n = len(sources)
        queues: List[List] = [[] for _ in range(n)]
        lock = RLock()
        is_done = [False] * n

        @synchronized(lock)
        def next(i):
            if all([len(q) for q in queues]):
                try:
                    queued_values = [x.pop(0) for x in queues]
                    res = tuple(queued_values)
                except Exception as ex:  # pylint: disable=broad-except
                    observer.on_error(ex)
                    return

                observer.on_next(res)
            elif all([x for j, x in enumerate(is_done) if j != i]):
                observer.on_completed()

        def done(i):
            is_done[i] = True
            if all(is_done):
                observer.on_completed()

        subscriptions = [None] * n

        def func(i):
            source = sources[i]
            sad = SingleAssignmentDisposable()
            source = from_future(source) if is_future(source) else source

            def on_next(x):
                queues[i].append(x)
                next(i)

            sad.disposable = source.subscribe_(on_next, observer.on_error, lambda: done(i), scheduler)
            subscriptions[i] = sad

        for idx in range(n):
            func(idx)
        return CompositeDisposable(subscriptions)

    return Observable(subscribe)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import fnmatch
import gzip
import json
import logging
import os
import shutil
import tempfile

import magic

from s3backup.clients import SyncClient, SyncObject

logger = logging.getLogger(__name__)

def traverse(path, ignore_files=None):
    if not os.path.exists(path):
        return

    if ignore_files is None:
        ignore_files = []

    for item in os.scandir(path):
        if any(fnmatch.fnmatch(item.name, pattern) for pattern in ignore_files):
            logger.debug('Ignoring %s', item)
            continue

        if item.is_dir():
            for result in traverse(item.path, ignore_files):
                yield os.path.join(item.name, result)
        else:
            yield item.name

class LocalSyncClient(SyncClient):
    def __init__(self, path):
        self.path = path
        self.reload_index()
        self.reload_ignore_files()

    def get_client_name(self):
        return 'local'

    def __repr__(self):
        return 'LocalSyncClient<{}>'.format(self.path)

    def get_uri(self):
        return self.path

    def index_path(self):
        return os.path.join(self.path, '.index')

    def put(self, key, sync_object, callback=None):
        path = os.path.join(self.path, key)

        parent = os.path.dirname(path)
        if not os.path.exists(parent):
            os.makedirs(parent)

        BUFFER_SIZE = 4096
        _, temp_path = tempfile.mkstemp()

        try:
            with open(temp_path, 'wb') as fp1:
                while True:
                    data = sync_object.fp.read(BUFFER_SIZE)
                    fp1.write(data)
                    if callback is not None:
                        callback(len(data))
                    if len(data) < BUFFER_SIZE:
                        break
            shutil.move(temp_path, path)
        except Exception:
            os.remove(temp_path)
            raise

        self.set_remote_timestamp(key, sync_object.timestamp)

    def get(self, key):
        path = os.path.join(self.path, key)
        if os.path.exists(path):
            fp = open(path, 'rb')
            stat = os.stat(path)
            return SyncObject(fp, stat.st_size, stat.st_mtime)
        else:
            return None

    def delete(self, key):
        path = os.path.join(self.path, key)
        if os.path.exists(path):
            os.remove(path)
            return True
        else:
            return False

    def reload_index(self):
        self.index = self._load_index()

    def _load_index(self):
        index_path = self.index_path()
        if not os.path.exists(index_path):
            return {}

        content_type = magic.from_file(index_path, mime=True)
        if content_type == 'text/plain':
            logger.debug('Detected plaintext encoding for reading index')
            method = open
        elif content_type in ('application/gzip', 'application/x-gzip'):
            logger.debug('Detected gzip encoding for reading index')
            method = gzip.open
        else:
            raise ValueError('Index is of unknown type', content_type)

        with method(index_path, 'rt') as fp:
            data = json.load(fp)
        return data

    def flush_index(self, compressed=True):
        if compressed:
            logger.debug('Using gzip encoding for writing index')
            method = gzip.open
        else:
            logger.debug('Using plaintext encoding for writing index')
            method = open

        with method(self.index_path(), 'wt') as fp:
            json.dump(self.index, fp)

    def get_local_keys(self):
        return list(traverse(self.path, ignore_files=self.ignore_files))

    def get_real_local_timestamp(self, key):
        full_path = os.path.join(self.path, key)
        if os.path.exists(full_path):
            return os.path.getmtime(full_path)
        else:
            return None

    def get_index_keys(self):
        return self.index.keys()

    def get_index_local_timestamp(self, key):
        return self.index.get(key, {}).get('local_timestamp')

    def get_all_real_local_timestamps(self):
        result = {}
        for key in self.get_local_keys():
            result[key] = self.get_real_local_timestamp(key)
        return result

    def get_all_remote_timestamps(self):
        return {key: value.get('remote_timestamp') for key, value in self.index.items()}

    def get_all_index_local_timestamps(self):
        return {key: value.get('local_timestamp') for key, value in self.index.items()}

    def set_index_local_timestamp(self, key, timestamp):
        if key not in self.index:
            self.index[key] = {}
        self.index[key]['local_timestamp'] = timestamp

    def get_remote_timestamp(self, key):
        return self.index.get(key, {}).get('remote_timestamp')

    def set_remote_timestamp(self, key, timestamp):
        if key not in self.index:
            self.index[key] = {}
        self.index[key]['remote_timestamp'] = timestamp

    def reload_ignore_files(self):
        self.ignore_files = ['.index']
        ignore_path = os.path.join(self.path, '.syncignore')
        if os.path.exists(ignore_path):
            with open(ignore_path, 'r') as fp:
                ignore_list = fp.read().split('\n')
            self.ignore_files.extend(ignore_list)
<commit_msg>better fault tolerance when flushing index on LocalSyncClient<commit_after># -*- coding: utf-8 -*-

import fnmatch
import gzip
import json
import logging
import os
import shutil
import tempfile

import magic

from s3backup.clients import SyncClient, SyncObject

logger = logging.getLogger(__name__)

def traverse(path, ignore_files=None):
    if not os.path.exists(path):
        return

    if ignore_files is None:
        ignore_files = []

    for item in os.scandir(path):
        if any(fnmatch.fnmatch(item.name, pattern) for pattern in ignore_files):
            logger.debug('Ignoring %s', item)
            continue

        if item.is_dir():
            for result in traverse(item.path, ignore_files):
                yield os.path.join(item.name, result)
        else:
            yield item.name

class LocalSyncClient(SyncClient):
    def __init__(self, path):
        self.path = path
        self.reload_index()
        self.reload_ignore_files()

    def get_client_name(self):
        return 'local'

    def __repr__(self):
        return 'LocalSyncClient<{}>'.format(self.path)

    def get_uri(self):
        return self.path

    def index_path(self):
        return os.path.join(self.path, '.index')

    def put(self, key, sync_object, callback=None):
        path = os.path.join(self.path, key)

        parent = os.path.dirname(path)
        if not os.path.exists(parent):
            os.makedirs(parent)

        BUFFER_SIZE = 4096
        _, temp_path = tempfile.mkstemp()

        try:
            with open(temp_path, 'wb') as fp1:
                while True:
                    data = sync_object.fp.read(BUFFER_SIZE)
                    fp1.write(data)
                    if callback is not None:
                        callback(len(data))
                    if len(data) < BUFFER_SIZE:
                        break
            shutil.move(temp_path, path)
        except Exception:
            os.remove(temp_path)
            raise

        self.set_remote_timestamp(key, sync_object.timestamp)

    def get(self, key):
        path = os.path.join(self.path, key)
        if os.path.exists(path):
            fp = open(path, 'rb')
            stat = os.stat(path)
            return SyncObject(fp, stat.st_size, stat.st_mtime)
        else:
            return None

    def delete(self, key):
        path = os.path.join(self.path, key)
        if os.path.exists(path):
            os.remove(path)
            return True
        else:
            return False

    def reload_index(self):
        self.index = self._load_index()

    def _load_index(self):
        index_path = self.index_path()
        if not os.path.exists(index_path):
            return {}

        content_type = magic.from_file(index_path, mime=True)
        if content_type == 'text/plain':
            logger.debug('Detected plaintext encoding for reading index')
            method = open
        elif content_type in ('application/gzip', 'application/x-gzip'):
            logger.debug('Detected gzip encoding for reading index')
            method = gzip.open
        else:
            raise ValueError('Index is of unknown type', content_type)

        with method(index_path, 'rt') as fp:
            data = json.load(fp)
        return data

    def flush_index(self, compressed=True):
        if compressed:
            logger.debug('Using gzip encoding for writing index')
            method = gzip.open
        else:
            logger.debug('Using plaintext encoding for writing index')
            method = open

        _, temp_path = tempfile.mkstemp()
        with method(temp_path, 'wt') as fp:
            json.dump(self.index, fp)

        shutil.move(temp_path, self.index_path())

    def get_local_keys(self):
        return list(traverse(self.path, ignore_files=self.ignore_files))

    def get_real_local_timestamp(self, key):
        full_path = os.path.join(self.path, key)
        if os.path.exists(full_path):
            return os.path.getmtime(full_path)
        else:
            return None

    def get_index_keys(self):
        return self.index.keys()

    def get_index_local_timestamp(self, key):
        return self.index.get(key, {}).get('local_timestamp')

    def get_all_real_local_timestamps(self):
        result = {}
        for key in self.get_local_keys():
            result[key] = self.get_real_local_timestamp(key)
        return result

    def get_all_remote_timestamps(self):
        return {key: value.get('remote_timestamp') for key, value in self.index.items()}

    def get_all_index_local_timestamps(self):
        return {key: value.get('local_timestamp') for key, value in self.index.items()}

    def set_index_local_timestamp(self, key, timestamp):
        if key not in self.index:
            self.index[key] = {}
        self.index[key]['local_timestamp'] = timestamp

    def get_remote_timestamp(self, key):
        return self.index.get(key, {}).get('remote_timestamp')

    def set_remote_timestamp(self, key, timestamp):
        if key not in self.index:
            self.index[key] = {}
        self.index[key]['remote_timestamp'] = timestamp

    def reload_ignore_files(self):
        self.ignore_files = ['.index']
        ignore_path = os.path.join(self.path, '.syncignore')
        if os.path.exists(ignore_path):
            with open(ignore_path, 'r') as fp:
                ignore_list = fp.read().split('\n')
            self.ignore_files.extend(ignore_list)
<|endoftext|>