<commit_before>from sympy import Expr, Symbol, Mul, Add, Pow, expand, sympify, Tuple, Integer
from sympy.core.basic import Basic
from sympy.core.singleton import S
from sympy.core.decorators import _sympifyit, call_highest_priority
from sympy.matrices import ShapeError

class MatrixExpr(Expr):
    """ Matrix Expression Class
    Matrix Expressions subclass SymPy Expr's so that
    MatAdd inherits from Add
    MatMul inherits from Mul
    MatPow inherits from Pow

    They use _op_priority to gain control with binary operations (+, *, -, **)
    are used

    They implement operations specific to Matrix Algebra.
    """

    _op_priority = 11.0

    is_Matrix = True
    is_MatrixExpr = True
    is_Identity = None
    is_Inverse = False
    is_Transpose = False
    is_ZeroMatrix = False
    is_BlockMatrix = False

    is_commutative = False

    # The following is adapted from the core Expr object

    def __neg__(self):
        return MatMul(S.NegativeOne, self)
    def __abs__(self):
        raise NotImplementedError

    @_sympifyit('other', NotImplemented)
    @call_highest_priority('__radd__')
    def __add__(self, other):
        return MatAdd(self, other)
    @_sympifyit('other', NotImplemented)
    @call_highest_priority('__add__')
    def __radd__(self, other):
        return MatAdd(other, self)

    @_sympifyit('other', NotImplemented)
    @call_highest_priority('__rsub__')
    def __sub__(self, other):
        return MatAdd(self, -other)
    @_sympifyit('other', NotImplemented)
    @call_highest_priority('__sub__')
    def __rsub__(self, other):
        return MatAdd(other, -self)

    @_sympifyit('other', NotImplemented)
    @call_highest_priority('__rmul__')
    def __mul__(self, other):
        return MatMul(self, other)
    @_sympifyit('other', NotImplemented)
    @call_highest_priority('__mul__')
    def __rmul__(self, other):
        return MatMul(other, self)

    @_sympifyit('other', NotImplemented)
    @call_highest_priority('__rpow__')
    def __pow__(self, other):
        if other == -S.One:
            return Inverse(self)
        return MatPow(self, other)
    @_sympifyit('other', NotImplemented)
    @call_highest_priority('__pow__')
    def __rpow__(self, other):
        raise NotImplementedError("Matrix Power not defined")
    @_sympifyit('other', NotImplemented)
    @call_highest_priority('__rdiv__')
    def __div__(self, other):
        return MatMul(self, other**S.NegativeOne)
    @_sympifyit('other', NotImplemented)
    @call_highest_priority('__div__')
    def __rdiv__(self, other):
        raise NotImplementedError()
        #return MatMul(other, Pow(self, S.NegativeOne))

    __truediv__ = __div__
    __rtruediv__ = __rdiv__

    @property
    def rows(self):
        return self.shape[0]

    @property
    def cols(self):
        return self.shape[1]

    @property
    def is_square(self):
        return self.rows == self.cols

    def _eval_transpose(self):
        raise NotImplementedError()

    def _eval_inverse(self):
        raise NotImplementedError()

    def adjoint(self):
        return MatMul(*[arg.adjoint() for arg in self.args[::-1]])

    def conjugate(self):
        return MatMul(*[arg.conjugate() for arg in self.args])

    def transpose(self):
        try:
            return self._eval_transpose()
        except (AttributeError, NotImplementedError):
            return Basic.__new__(Transpose, self)

    C = property(conjugate, None, None, 'By-element conjugation.')

    T = property(transpose, None, None, 'Matrix transposition.')

    @property
    def I(self):
        return Inverse(self)

    def _entry(self, i, j):
        raise NotImplementedError(
            "Indexing not implemented for %s" % self.__class__.__name__)

    def valid_index(self, i, j):
        def is_valid(idx):
            return isinstance(idx, (int, Integer, Symbol))
        return (is_valid(i) and is_valid(j) and
                0 <= i < self.rows and 0 <= j < self.cols)

    def __getitem__(self, key):
        if isinstance(key, tuple) and len(key)==2:
            i, j = key
            if isinstance(i, slice) or isinstance(j, slice):
                raise NotImplementedError(
                "Slicing is not implemented for %s" % self.__class__.__name__)
            i, j = sympify(i), sympify(j)
            if self.valid_index(i, j) is not False:
                return self._entry(i, j)
            else:
                raise IndexError("Invalid indices (%s, %s)" % (i, j))
        raise IndexError("Invalid index, wanted %s[i,j]" % self)

    def as_explicit(self):
        """
        Returns a dense Matrix with elements represented explicitly

        Returns an object of type ImmutableMatrix.

        Examples
        ========
        >>> from sympy import Identity
        >>> I = Identity(3)
        >>> I
        I
        >>> I.as_explicit()
        [1, 0, 0]
        [0, 1, 0]
        [0, 0, 1]

        See Also
        ========
        as_mutable: returns mutable Matrix type
        """
        from sympy.matrices.immutable import ImmutableMatrix
        return ImmutableMatrix([[    self[i,j]
                            for j in range(self.cols)]
                            for i in range(self.rows)])

    def as_mutable(self):
        """
        Returns a dense, mutable matrix with elements represented explicitly

        Examples
        ========

        >>> from sympy import Identity
        >>> I = Identity(3)
        >>> I
        I
        >>> I.shape
        (3, 3)
        >>> I.as_mutable()
        [1, 0, 0]
        [0, 1, 0]
        [0, 0, 1]

        See Also
        ========
        as_explicit: returns ImmutableMatrix
        """
        return self.as_explicit().as_mutable()

    def __array__(self):
        from numpy import empty
        a = empty(self.shape, dtype=object)
        for i in range(self.rows):
            for j in range(self.cols):
                a[i, j] = self[i, j]
        return a

    def equals(self, other):
        """
        Test elementwise equality between matrices, potentially of different
        types

        >>> from sympy import Identity, eye
        >>> Identity(3).equals(eye(3))
        True
        """
        return self.as_explicit().equals(other)

class MatrixSymbol(MatrixExpr, Symbol):
    """Symbolic representation of a Matrix object

    Creates a SymPy Symbol to represent a Matrix. This matrix has a shape and
    can be included in Matrix Expressions

    >>> from sympy import MatrixSymbol, Identity
    >>> A = MatrixSymbol('A', 3, 4) # A 3 by 4 Matrix
    >>> B = MatrixSymbol('B', 4, 3) # A 4 by 3 Matrix
    >>> A.shape
    (3, 4)
    >>> 2*A*B + Identity(3)
    I + 2*A*B
    """
    is_commutative = False

    def __new__(cls, name, n, m):
        n, m = sympify(n), sympify(m)
        obj = Basic.__new__(cls, name, n, m)
        return obj

    def _hashable_content(self):
        return(self.name, self.shape)

    @property
    def shape(self):
        return self.args[1:3]

    @property
    def name(self):
        return self.args[0]

    def _eval_subs(self, old, new):
        # only do substitutions in shape
        shape = Tuple(*self.shape)._subs(old, new)
        return MatrixSymbol(self.name, *shape)

    def __call__(self, *args):
        raise TypeError( "%s object is not callable" % self.__class__ )

    def _entry(self, i, j):
        # MatMul _entry will pass us a Dummy and ask that we remember it
        # so that it can be summed over later. We'll use the function syntax
        if i.is_Dummy or j.is_Dummy:
            return Symbol(self.name)(i,j)
        # If that isn't the case we'd really rather just make a symbol
        # They are simpler and look much nicer
        else:
            return Symbol('%s_%s%s'%(self.name, str(i), str(j)))

class Identity(MatrixSymbol):
    """The Matrix Identity I - multiplicative identity
    >>> from sympy.matrices import Identity, MatrixSymbol
    >>> A = MatrixSymbol('A', 3, 5)
    >>> I = Identity(3)
    >>> I*A
    A
    """

    is_Identity = True
    def __new__(cls, n):
        return MatrixSymbol.__new__(cls, "I", n, n)

    def _eval_transpose(self):
        return self

    def _eval_trace(self):
        return self.rows

    def _eval_inverse(self):
        return self

    def conjugate(self):
        return self

    def _entry(self, i, j):
        if i==j:
            return S.One
        else:
            return S.Zero

class ZeroMatrix(MatrixSymbol):
    """The Matrix Zero 0 - additive identity
    >>> from sympy import MatrixSymbol, ZeroMatrix
    >>> A = MatrixSymbol('A', 3, 5)
    >>> Z = ZeroMatrix(3, 5)
    >>> A+Z
    A
    >>> Z*A.T
    0
    """
    is_ZeroMatrix = True
    def __new__(cls, n, m):
        return MatrixSymbol.__new__(cls, "0", n, m)

    def _eval_transpose(self):
        return ZeroMatrix(self.cols, self.rows)

    def _eval_trace(self):
        return S.Zero

    def conjugate(self):
        return self

    def _entry(self, i, j):
        return S.Zero

def matrix_symbols(expr):
    return [sym for sym in expr.free_symbols if sym.is_Matrix]

def matrixify(expr):
    """
    Recursively walks down an expression tree changing Expr's to MatExpr's
    i.e. Add -> MatAdd
         Mul -> MatMul

    Only changes those Exprs which contain MatrixSymbols

    This function is useful when traditional SymPy functions which use Mul and
    Add are called on MatrixExpressions. Examples flatten, expand, simplify...

    Calling matrixify after calling these functions will reset classes back to
    their matrix equivalents
    """
    class_dict = {Mul:MatMul, Add:MatAdd, MatMul:MatMul, MatAdd:MatAdd,
            Pow:MatPow, MatPow:MatPow}

    if expr.__class__ not in class_dict:
        return expr

    args = map(matrixify, expr.args) # Recursively call down the tree

    if not any(arg.is_Matrix for arg in args):
        return expr
    else:
        return Basic.__new__(class_dict[expr.__class__], *args)

def linear_factors(expr, *syms):
    """Reduce a Matrix Expression to a sum of linear factors

    Given symbols and a matrix expression linear in those symbols return a
    dict mapping symbol to the linear factor

    >>> from sympy import MatrixSymbol, linear_factors, symbols
    >>> n, m, l = symbols('n m l')
    >>> A = MatrixSymbol('A', n, m)
    >>> B = MatrixSymbol('B', m, l)
    >>> C = MatrixSymbol('C', n, l)
    >>> linear_factors(2*A*B + C, B, C)
    {B: 2*A, C: I}
    """

    expr = matrixify(expand(expr))
    d = {}
    if expr.is_Matrix and expr.is_Symbol:
        if expr in syms:
            d[expr] = Identity(expr.rows)

    if expr.is_Add:
        for sym in syms:
            total_factor = 0
            for arg in expr.args:
                factor = arg.coeff(sym)
                if not factor:
                    # .coeff fails when powers are in the expression
                    if sym in arg.free_symbols:
                        raise ValueError("Expression not linear in symbols")
                    else:
                        factor = 0
                factor = sympify(factor)
                if not factor.is_Matrix:
                    if factor.is_zero:
                        factor = ZeroMatrix(expr.rows, sym.rows)
                        if not sym.cols == expr.cols:
                            raise ShapeError(
                            "%s not compatible as factor of %s"%(sym, expr))
                    else:
                        factor = Identity(sym.rows)*factor
                total_factor += factor
            d[sym] = total_factor
    elif expr.is_Mul:
        for sym in syms:
            factor = expr.coeff(sym)
            if not factor:
                # .coeff fails when powers are in the expression
                if sym in expr.free_symbols:
                    raise ValueError("Expression not linear in symbols")
                else:
                    factor = 0
            factor = sympify(factor)
            if not factor.is_Matrix:
                if factor.is_zero:
                    factor = ZeroMatrix(expr.rows, sym.rows)
                    if not sym.cols == expr.cols:
                        raise ShapeError("%s not compatible as factor of %s"%
                                (sym, expr))
                else:
                    factor = Identity(sym.rows)*factor
            d[sym] = factor

    if any(sym in matrix_symbols(Tuple(*d.values())) for sym in syms):
        raise ValueError("Expression not linear in symbols")

    return d

from matmul import MatMul
from matadd import MatAdd
from matpow import MatPow
from transpose import Transpose
from inverse import Inverse
<commit_msg>raise notimplemented on MatrixExpr.conjugate<commit_after>from sympy import Expr, Symbol, Mul, Add, Pow, expand, sympify, Tuple, Integer
from sympy.core.basic import Basic
from sympy.core.singleton import S
from sympy.core.decorators import _sympifyit, call_highest_priority
from sympy.matrices import ShapeError

class MatrixExpr(Expr):
    """ Matrix Expression Class
    Matrix Expressions subclass SymPy Expr's so that
    MatAdd inherits from Add
    MatMul inherits from Mul
    MatPow inherits from Pow

    They use _op_priority to gain control with binary operations (+, *, -, **)
    are used

    They implement operations specific to Matrix Algebra.
    """

    _op_priority = 11.0

    is_Matrix = True
    is_MatrixExpr = True
    is_Identity = None
    is_Inverse = False
    is_Transpose = False
    is_ZeroMatrix = False
    is_BlockMatrix = False

    is_commutative = False

    # The following is adapted from the core Expr object

    def __neg__(self):
        return MatMul(S.NegativeOne, self)
    def __abs__(self):
        raise NotImplementedError

    @_sympifyit('other', NotImplemented)
    @call_highest_priority('__radd__')
    def __add__(self, other):
        return MatAdd(self, other)
    @_sympifyit('other', NotImplemented)
    @call_highest_priority('__add__')
    def __radd__(self, other):
        return MatAdd(other, self)

    @_sympifyit('other', NotImplemented)
    @call_highest_priority('__rsub__')
    def __sub__(self, other):
        return MatAdd(self, -other)
    @_sympifyit('other', NotImplemented)
    @call_highest_priority('__sub__')
    def __rsub__(self, other):
        return MatAdd(other, -self)

    @_sympifyit('other', NotImplemented)
    @call_highest_priority('__rmul__')
    def __mul__(self, other):
        return MatMul(self, other)
    @_sympifyit('other', NotImplemented)
    @call_highest_priority('__mul__')
    def __rmul__(self, other):
        return MatMul(other, self)

    @_sympifyit('other', NotImplemented)
    @call_highest_priority('__rpow__')
    def __pow__(self, other):
        if other == -S.One:
            return Inverse(self)
        return MatPow(self, other)
    @_sympifyit('other', NotImplemented)
    @call_highest_priority('__pow__')
    def __rpow__(self, other):
        raise NotImplementedError("Matrix Power not defined")
    @_sympifyit('other', NotImplemented)
    @call_highest_priority('__rdiv__')
    def __div__(self, other):
        return MatMul(self, other**S.NegativeOne)
    @_sympifyit('other', NotImplemented)
    @call_highest_priority('__div__')
    def __rdiv__(self, other):
        raise NotImplementedError()
        #return MatMul(other, Pow(self, S.NegativeOne))

    __truediv__ = __div__
    __rtruediv__ = __rdiv__

    @property
    def rows(self):
        return self.shape[0]

    @property
    def cols(self):
        return self.shape[1]

    @property
    def is_square(self):
        return self.rows == self.cols

    def _eval_transpose(self):
        raise NotImplementedError()

    def _eval_inverse(self):
        raise NotImplementedError()

    def adjoint(self):
        raise NotImplementedError()

    def conjugate(self):
        raise NotImplementedError()

    def transpose(self):
        try:
            return self._eval_transpose()
        except (AttributeError, NotImplementedError):
            return Basic.__new__(Transpose, self)

    C = property(conjugate, None, None, 'By-element conjugation.')

    T = property(transpose, None, None, 'Matrix transposition.')

    @property
    def I(self):
        return Inverse(self)

    def _entry(self, i, j):
        raise NotImplementedError(
            "Indexing not implemented for %s" % self.__class__.__name__)

    def valid_index(self, i, j):
        def is_valid(idx):
            return isinstance(idx, (int, Integer, Symbol))
        return (is_valid(i) and is_valid(j) and
                0 <= i < self.rows and 0 <= j < self.cols)

    def __getitem__(self, key):
        if isinstance(key, tuple) and len(key)==2:
            i, j = key
            if isinstance(i, slice) or isinstance(j, slice):
                raise NotImplementedError(
                "Slicing is not implemented for %s" % self.__class__.__name__)
            i, j = sympify(i), sympify(j)
            if self.valid_index(i, j) is not False:
                return self._entry(i, j)
            else:
                raise IndexError("Invalid indices (%s, %s)" % (i, j))
        raise IndexError("Invalid index, wanted %s[i,j]" % self)

    def as_explicit(self):
        """
        Returns a dense Matrix with elements represented explicitly

        Returns an object of type ImmutableMatrix.

        Examples
        ========
        >>> from sympy import Identity
        >>> I = Identity(3)
        >>> I
        I
        >>> I.as_explicit()
        [1, 0, 0]
        [0, 1, 0]
        [0, 0, 1]

        See Also
        ========
        as_mutable: returns mutable Matrix type
        """
        from sympy.matrices.immutable import ImmutableMatrix
        return ImmutableMatrix([[    self[i,j]
                            for j in range(self.cols)]
                            for i in range(self.rows)])

    def as_mutable(self):
        """
        Returns a dense, mutable matrix with elements represented explicitly

        Examples
        ========

        >>> from sympy import Identity
        >>> I = Identity(3)
        >>> I
        I
        >>> I.shape
        (3, 3)
        >>> I.as_mutable()
        [1, 0, 0]
        [0, 1, 0]
        [0, 0, 1]

        See Also
        ========
        as_explicit: returns ImmutableMatrix
        """
        return self.as_explicit().as_mutable()

    def __array__(self):
        from numpy import empty
        a = empty(self.shape, dtype=object)
        for i in range(self.rows):
            for j in range(self.cols):
                a[i, j] = self[i, j]
        return a

    def equals(self, other):
        """
        Test elementwise equality between matrices, potentially of different
        types

        >>> from sympy import Identity, eye
        >>> Identity(3).equals(eye(3))
        True
        """
        return self.as_explicit().equals(other)

class MatrixSymbol(MatrixExpr, Symbol):
    """Symbolic representation of a Matrix object

    Creates a SymPy Symbol to represent a Matrix. This matrix has a shape and
    can be included in Matrix Expressions

    >>> from sympy import MatrixSymbol, Identity
    >>> A = MatrixSymbol('A', 3, 4) # A 3 by 4 Matrix
    >>> B = MatrixSymbol('B', 4, 3) # A 4 by 3 Matrix
    >>> A.shape
    (3, 4)
    >>> 2*A*B + Identity(3)
    I + 2*A*B
    """
    is_commutative = False

    def __new__(cls, name, n, m):
        n, m = sympify(n), sympify(m)
        obj = Basic.__new__(cls, name, n, m)
        return obj

    def _hashable_content(self):
        return(self.name, self.shape)

    @property
    def shape(self):
        return self.args[1:3]

    @property
    def name(self):
        return self.args[0]

    def _eval_subs(self, old, new):
        # only do substitutions in shape
        shape = Tuple(*self.shape)._subs(old, new)
        return MatrixSymbol(self.name, *shape)

    def __call__(self, *args):
        raise TypeError( "%s object is not callable" % self.__class__ )

    def _entry(self, i, j):
        # MatMul _entry will pass us a Dummy and ask that we remember it
        # so that it can be summed over later. We'll use the function syntax
        if i.is_Dummy or j.is_Dummy:
            return Symbol(self.name)(i,j)
        # If that isn't the case we'd really rather just make a symbol
        # They are simpler and look much nicer
        else:
            return Symbol('%s_%s%s'%(self.name, str(i), str(j)))

class Identity(MatrixSymbol):
    """The Matrix Identity I - multiplicative identity
    >>> from sympy.matrices import Identity, MatrixSymbol
    >>> A = MatrixSymbol('A', 3, 5)
    >>> I = Identity(3)
    >>> I*A
    A
    """

    is_Identity = True
    def __new__(cls, n):
        return MatrixSymbol.__new__(cls, "I", n, n)

    def _eval_transpose(self):
        return self

    def _eval_trace(self):
        return self.rows

    def _eval_inverse(self):
        return self

    def conjugate(self):
        return self

    def _entry(self, i, j):
        if i==j:
            return S.One
        else:
            return S.Zero

class ZeroMatrix(MatrixSymbol):
    """The Matrix Zero 0 - additive identity
    >>> from sympy import MatrixSymbol, ZeroMatrix
    >>> A = MatrixSymbol('A', 3, 5)
    >>> Z = ZeroMatrix(3, 5)
    >>> A+Z
    A
    >>> Z*A.T
    0
    """
    is_ZeroMatrix = True
    def __new__(cls, n, m):
        return MatrixSymbol.__new__(cls, "0", n, m)

    def _eval_transpose(self):
        return ZeroMatrix(self.cols, self.rows)

    def _eval_trace(self):
        return S.Zero

    def conjugate(self):
        return self

    def _entry(self, i, j):
        return S.Zero

def matrix_symbols(expr):
    return [sym for sym in expr.free_symbols if sym.is_Matrix]

def matrixify(expr):
    """
    Recursively walks down an expression tree changing Expr's to MatExpr's
    i.e. Add -> MatAdd
         Mul -> MatMul

    Only changes those Exprs which contain MatrixSymbols

    This function is useful when traditional SymPy functions which use Mul and
    Add are called on MatrixExpressions. Examples flatten, expand, simplify...

    Calling matrixify after calling these functions will reset classes back to
    their matrix equivalents
    """
    class_dict = {Mul:MatMul, Add:MatAdd, MatMul:MatMul, MatAdd:MatAdd,
            Pow:MatPow, MatPow:MatPow}

    if expr.__class__ not in class_dict:
        return expr

    args = map(matrixify, expr.args) # Recursively call down the tree

    if not any(arg.is_Matrix for arg in args):
        return expr
    else:
        return Basic.__new__(class_dict[expr.__class__], *args)

def linear_factors(expr, *syms):
    """Reduce a Matrix Expression to a sum of linear factors

    Given symbols and a matrix expression linear in those symbols return a
    dict mapping symbol to the linear factor

    >>> from sympy import MatrixSymbol, linear_factors, symbols
    >>> n, m, l = symbols('n m l')
    >>> A = MatrixSymbol('A', n, m)
    >>> B = MatrixSymbol('B', m, l)
    >>> C = MatrixSymbol('C', n, l)
    >>> linear_factors(2*A*B + C, B, C)
    {B: 2*A, C: I}
    """

    expr = matrixify(expand(expr))
    d = {}
    if expr.is_Matrix and expr.is_Symbol:
        if expr in syms:
            d[expr] = Identity(expr.rows)

    if expr.is_Add:
        for sym in syms:
            total_factor = 0
            for arg in expr.args:
                factor = arg.coeff(sym)
                if not factor:
                    # .coeff fails when powers are in the expression
                    if sym in arg.free_symbols:
                        raise ValueError("Expression not linear in symbols")
                    else:
                        factor = 0
                factor = sympify(factor)
                if not factor.is_Matrix:
                    if factor.is_zero:
                        factor = ZeroMatrix(expr.rows, sym.rows)
                        if not sym.cols == expr.cols:
                            raise ShapeError(
                            "%s not compatible as factor of %s"%(sym, expr))
                    else:
                        factor = Identity(sym.rows)*factor
                total_factor += factor
            d[sym] = total_factor
    elif expr.is_Mul:
        for sym in syms:
            factor = expr.coeff(sym)
            if not factor:
                # .coeff fails when powers are in the expression
                if sym in expr.free_symbols:
                    raise ValueError("Expression not linear in symbols")
                else:
                    factor = 0
            factor = sympify(factor)
            if not factor.is_Matrix:
                if factor.is_zero:
                    factor = ZeroMatrix(expr.rows, sym.rows)
                    if not sym.cols == expr.cols:
                        raise ShapeError("%s not compatible as factor of %s"%
                                (sym, expr))
                else:
                    factor = Identity(sym.rows)*factor
            d[sym] = factor

    if any(sym in matrix_symbols(Tuple(*d.values())) for sym in syms):
        raise ValueError("Expression not linear in symbols")

    return d

from matmul import MatMul
from matadd import MatAdd
from matpow import MatPow
from transpose import Transpose
from inverse import Inverse
<|endoftext|>


<commit_before># coding=utf-8
from __future__ import division, unicode_literals

import re

from django.conf import settings
from django.core.exceptions import ValidationError
from django.utils.translation import gettext_lazy as _
try:
    from django.utils.encoding import smart_text
except ImportError:  # django < 1.4.2
    from django.utils.encoding import smart_unicode as smart_text

COMMON_SEQUENCES = [
    "0123456789",
    "`1234567890-=",
    "~!@#$%^&*()_+",
    "abcdefghijklmnopqrstuvwxyz",
    "qwertyuiop[]\\asdfghjkl;\'zxcvbnm,./",
    'qwertyuiop{}|asdfghjkl;"zxcvbnm<>?',
    "qwertyuiopasdfghjklzxcvbnm",
    "1qaz2wsx3edc4rfv5tgb6yhn7ujm8ik,9ol.0p;/-['=]\\",
    "qazwsxedcrfvtgbyhnujmikolp",
    "qwertzuiopü+asdfghjklöä#<yxcvbnm,.-",
    "qwertzuiopü*asdfghjklöä'>yxcvbnm;:_",
    "qaywsxedcrfvtgbzhnujmikolp",
]
DICT_CACHE = []
DICT_FILESIZE = -1
DICT_MAX_CACHE = 1000000

# Settings
PASSWORD_MIN_LENGTH = getattr(
    settings, "PASSWORD_MIN_LENGTH", 6)
PASSWORD_MAX_LENGTH = getattr(
    settings, "PASSWORD_MAX_LENGTH", None)
PASSWORD_DICTIONARY = getattr(
    settings, "PASSWORD_DICTIONARY", None)
PASSWORD_MATCH_THRESHOLD = getattr(
    settings, "PASSWORD_MATCH_THRESHOLD", 0.9)
PASSWORD_COMMON_SEQUENCES = getattr(
    settings, "PASSWORD_COMMON_SEQUENCES", COMMON_SEQUENCES)
PASSWORD_COMPLEXITY = getattr(
    settings, "PASSWORD_COMPLEXITY", None)

class LengthValidator(object):
    message = _("Invalid Length (%s)")
    code = "length"

    def __init__(self, min_length=None, max_length=None):
        self.min_length = min_length
        self.max_length = max_length

    def __call__(self, value):
        err = None
        if self.min_length is not None and len(value) < self.min_length:
            err = _("Must be %s characters or more") % self.min_length
        elif self.max_length is not None and len(value) > self.max_length:
            err = _("Must be %s characters or less") % self.max_length

        if err is not None:
            raise ValidationError(self.message % err, code=self.code)

class ComplexityValidator(object):
    message = _("Must be more complex (%s)")
    code = "complexity"

    def __init__(self, complexities):
        self.complexities = complexities

    def __call__(self, value):
        if self.complexities is None:
            return

        uppercase, lowercase, letters = set(), set(), set()
        digits, special = set(), set()

        for character in value:
            if character.isupper():
                uppercase.add(character)
                letters.add(character)
            elif character.islower():
                lowercase.add(character)
                letters.add(character)
            elif character.isdigit():
                digits.add(character)
            elif not character.isspace():
                special.add(character)

        words = set(re.findall(r'\b\w+', value, re.UNICODE))

        errors = []
        if len(uppercase) < self.complexities.get("UPPER", 0):
            errors.append(
                _("%(UPPER)s or more unique uppercase characters") %
                self.complexities)
        if len(lowercase) < self.complexities.get("LOWER", 0):
            errors.append(
                _("%(LOWER)s or more unique lowercase characters") %
                self.complexities)
        if len(letters) < self.complexities.get("LETTERS", 0):
            errors.append(
                _("%(LETTERS)s or more unique letters") %
                self.complexities)
        if len(digits) < self.complexities.get("DIGITS", 0):
            errors.append(
                _("%(DIGITS)s or more unique digits") %
                self.complexities)
        if len(special) < self.complexities.get("SPECIAL", 0):
            errors.append(
                _("%(SPECIAL)s or more non unique special characters") %
                self.complexities)
        if len(words) < self.complexities.get("WORDS", 0):
            errors.append(
                _("%(WORDS)s or more unique words") %
                self.complexities)

        if errors:
            raise ValidationError(self.message % (_(u'must contain ') + u', '.join(errors),),
                                  code=self.code)

class BaseSimilarityValidator(object):
    message = _("Too Similar to [%(haystacks)s]")
    code = "similarity"

    def __init__(self, haystacks=None, threshold=None):
        self.haystacks = haystacks if haystacks else []
        if threshold is None:
            self.threshold = PASSWORD_MATCH_THRESHOLD
        else:
            self.threshold = threshold

    def fuzzy_substring(self, needle, haystack):
        needle, haystack = needle.lower(), haystack.lower()
        m, n = len(needle), len(haystack)

        if m == 1:
            if needle not in haystack:
                return -1
        if n == 0:
            return m

        row1 = [0] * (n + 1)
        for i in range(0, m):
            row2 = [i + 1]
            for j in range(0, n):
                cost = 1 if needle[i] != haystack[j] else 0
                row2.append(min(
                    row1[j + 1] + 1,
                    row2[j] + 1,
                    row1[j] + cost))
            row1 = row2
        return min(row1)

    def __call__(self, value):
        for haystack in self.haystacks:
            distance = self.fuzzy_substring(value, haystack)
            longest = max(len(value), len(haystack))
            similarity = (longest - distance) / longest
            if similarity >= self.threshold:
                raise ValidationError(
                    self.message % {"haystacks": ", ".join(self.haystacks)},
                    code=self.code)

class DictionaryValidator(BaseSimilarityValidator):
    message = _("Based on a dictionary word")
    code = "dictionary_word"

    def __init__(self, words=None, dictionary=None, threshold=None):
        haystacks = []
        if dictionary:
            words = self.get_dictionary_words(dictionary)
        if words:
            haystacks.extend(words)
        super(DictionaryValidator, self).__init__(
            haystacks=haystacks,
            threshold=threshold)

    def get_dictionary_words(self, dictionary):
        if DICT_CACHE:
            return DICT_CACHE
        if DICT_FILESIZE is -1:
            f = open(dictionary)
            f.seek(0,2)
            DICT_FILESIZE = f.tell()
            f.close()
            if DICT_FILESIZE < 1000000:
                with open(dictionary) as dictionary:
                    DICT_CACHE = [smart_text(x.strip()) for x in dictionary.readlines()]
                    return DICT_CACHE
        with open(dictionary) as dictionary:
            return [smart_text(x.strip()) for x in dictionary.readlines()]

class CommonSequenceValidator(BaseSimilarityValidator):
    message = _("Based on a common sequence of characters")
    code = "common_sequence"

validate_length = LengthValidator(PASSWORD_MIN_LENGTH, PASSWORD_MAX_LENGTH)
complexity = ComplexityValidator(PASSWORD_COMPLEXITY)
dictionary_words = DictionaryValidator(dictionary=PASSWORD_DICTIONARY)
common_sequences = CommonSequenceValidator(PASSWORD_COMMON_SEQUENCES)
<commit_msg>more django 4.x fixes<commit_after># coding=utf-8
from __future__ import division, unicode_literals

import re

from django.conf import settings
from django.core.exceptions import ValidationError
from django.utils.translation import gettext_lazy as _
from django.utils.encoding import smart_str

COMMON_SEQUENCES = [
    "0123456789",
    "`1234567890-=",
    "~!@#$%^&*()_+",
    "abcdefghijklmnopqrstuvwxyz",
    "qwertyuiop[]\\asdfghjkl;\'zxcvbnm,./",
    'qwertyuiop{}|asdfghjkl;"zxcvbnm<>?',
    "qwertyuiopasdfghjklzxcvbnm",
    "1qaz2wsx3edc4rfv5tgb6yhn7ujm8ik,9ol.0p;/-['=]\\",
    "qazwsxedcrfvtgbyhnujmikolp",
    "qwertzuiopü+asdfghjklöä#<yxcvbnm,.-",
    "qwertzuiopü*asdfghjklöä'>yxcvbnm;:_",
    "qaywsxedcrfvtgbzhnujmikolp",
]
DICT_CACHE = []
DICT_FILESIZE = -1
DICT_MAX_CACHE = 1000000

# Settings
PASSWORD_MIN_LENGTH = getattr(
    settings, "PASSWORD_MIN_LENGTH", 6)
PASSWORD_MAX_LENGTH = getattr(
    settings, "PASSWORD_MAX_LENGTH", None)
PASSWORD_DICTIONARY = getattr(
    settings, "PASSWORD_DICTIONARY", None)
PASSWORD_MATCH_THRESHOLD = getattr(
    settings, "PASSWORD_MATCH_THRESHOLD", 0.9)
PASSWORD_COMMON_SEQUENCES = getattr(
    settings, "PASSWORD_COMMON_SEQUENCES", COMMON_SEQUENCES)
PASSWORD_COMPLEXITY = getattr(
    settings, "PASSWORD_COMPLEXITY", None)

class LengthValidator(object):
    message = _("Invalid Length (%s)")
    code = "length"

    def __init__(self, min_length=None, max_length=None):
        self.min_length = min_length
        self.max_length = max_length

    def __call__(self, value):
        err = None
        if self.min_length is not None and len(value) < self.min_length:
            err = _("Must be %s characters or more") % self.min_length
        elif self.max_length is not None and len(value) > self.max_length:
            err = _("Must be %s characters or less") % self.max_length

        if err is not None:
            raise ValidationError(self.message % err, code=self.code)

class ComplexityValidator(object):
    message = _("Must be more complex (%s)")
    code = "complexity"

    def __init__(self, complexities):
        self.complexities = complexities

    def __call__(self, value):
        if self.complexities is None:
            return

        uppercase, lowercase, letters = set(), set(), set()
        digits, special = set(), set()

        for character in value:
            if character.isupper():
                uppercase.add(character)
                letters.add(character)
            elif character.islower():
                lowercase.add(character)
                letters.add(character)
            elif character.isdigit():
                digits.add(character)
            elif not character.isspace():
                special.add(character)

        words = set(re.findall(r'\b\w+', value, re.UNICODE))

        errors = []
        if len(uppercase) < self.complexities.get("UPPER", 0):
            errors.append(
                _("%(UPPER)s or more unique uppercase characters") %
                self.complexities)
        if len(lowercase) < self.complexities.get("LOWER", 0):
            errors.append(
                _("%(LOWER)s or more unique lowercase characters") %
                self.complexities)
        if len(letters) < self.complexities.get("LETTERS", 0):
            errors.append(
                _("%(LETTERS)s or more unique letters") %
                self.complexities)
        if len(digits) < self.complexities.get("DIGITS", 0):
            errors.append(
                _("%(DIGITS)s or more unique digits") %
                self.complexities)
        if len(special) < self.complexities.get("SPECIAL", 0):
            errors.append(
                _("%(SPECIAL)s or more non unique special characters") %
                self.complexities)
        if len(words) < self.complexities.get("WORDS", 0):
            errors.append(
                _("%(WORDS)s or more unique words") %
                self.complexities)

        if errors:
            raise ValidationError(self.message % (_(u'must contain ') + u', '.join(errors),),
                                  code=self.code)

class BaseSimilarityValidator(object):
    message = _("Too Similar to [%(haystacks)s]")
    code = "similarity"

    def __init__(self, haystacks=None, threshold=None):
        self.haystacks = haystacks if haystacks else []
        if threshold is None:
            self.threshold = PASSWORD_MATCH_THRESHOLD
        else:
            self.threshold = threshold

    def fuzzy_substring(self, needle, haystack):
        needle, haystack = needle.lower(), haystack.lower()
        m, n = len(needle), len(haystack)

        if m == 1:
            if needle not in haystack:
                return -1
        if n == 0:
            return m

        row1 = [0] * (n + 1)
        for i in range(0, m):
            row2 = [i + 1]
            for j in range(0, n):
                cost = 1 if needle[i] != haystack[j] else 0
                row2.append(min(
                    row1[j + 1] + 1,
                    row2[j] + 1,
                    row1[j] + cost))
            row1 = row2
        return min(row1)

    def __call__(self, value):
        for haystack in self.haystacks:
            distance = self.fuzzy_substring(value, haystack)
            longest = max(len(value), len(haystack))
            similarity = (longest - distance) / longest
            if similarity >= self.threshold:
                raise ValidationError(
                    self.message % {"haystacks": ", ".join(self.haystacks)},
                    code=self.code)

class DictionaryValidator(BaseSimilarityValidator):
    message = _("Based on a dictionary word")
    code = "dictionary_word"

    def __init__(self, words=None, dictionary=None, threshold=None):
        haystacks = []
        if dictionary:
            words = self.get_dictionary_words(dictionary)
        if words:
            haystacks.extend(words)
        super(DictionaryValidator, self).__init__(
            haystacks=haystacks,
            threshold=threshold)

    def get_dictionary_words(self, dictionary):
        if DICT_CACHE:
            return DICT_CACHE
        if DICT_FILESIZE is -1:
            f = open(dictionary)
            f.seek(0,2)
            DICT_FILESIZE = f.tell()
            f.close()
            if DICT_FILESIZE < 1000000:
                with open(dictionary) as dictionary:
                    DICT_CACHE = [smart_str(x.strip()) for x in dictionary.readlines()]
                    return DICT_CACHE
        with open(dictionary) as dictionary:
            return [smart_str(x.strip()) for x in dictionary.readlines()]

class CommonSequenceValidator(BaseSimilarityValidator):
    message = _("Based on a common sequence of characters")
    code = "common_sequence"

validate_length = LengthValidator(PASSWORD_MIN_LENGTH, PASSWORD_MAX_LENGTH)
complexity = ComplexityValidator(PASSWORD_COMPLEXITY)
dictionary_words = DictionaryValidator(dictionary=PASSWORD_DICTIONARY)
common_sequences = CommonSequenceValidator(PASSWORD_COMMON_SEQUENCES)
<|endoftext|>


<commit_before>#
# Copyright 2014 Jon Wayne Parrott, [proppy], Michael R. Bernstein
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# Imported from:
# https://github.com/jonparrott/Darth-Vendor/

import site
import os.path
import sys

def add(folder, position=1):
  """
  Adds the given folder to the python path. Supports namespaced packages.
  By default, packages in the given folder take precedence over site-packages
  and any previous path manipulations.

  Args:
    folder: Path to the folder containing packages, relative to ``os.getcwd()``
    position: Where in ``sys.path`` to insert the vendor packages. By default
      this is set to 1. It is inadvisable to set it to 0 as it will override
      any modules in the current working directory.
  """

  # Check if the path contains a virtualenv.
  site_dir = os.path.join(folder, 'lib', 'python' + sys.version[:3], 'site-packages')
  if os.path.exists(site_dir):
    folder = site_dir
  # Otherwise it's just a normal path, make it absolute.
  else:
    folder = os.path.join(os.path.dirname(__file__), folder)

  # Use site.addsitedir() because it appropriately reads .pth
  # files for namespaced packages. Unfortunately, there's not an
  # option to choose where addsitedir() puts its paths in sys.path
  # so we have to do a little bit of magic to make it play along.

  # We're going to grab the current sys.path and split it up into
  # the first entry and then the rest. Essentially turning
  #   ['.', '/site-packages/x', 'site-packages/y']
  # into
  #   ['.'] and ['/site-packages/x', 'site-packages/y']
  # The reason for this is we want '.' to remain at the top of the
  # list but we want our vendor files to override everything else.
  sys.path, remainder = sys.path[:1], sys.path[1:]

  # Now we call addsitedir which will append our vendor directories
  # to sys.path (which was truncated by the last step.)
  site.addsitedir(folder)

  # Finally, we'll add the paths we removed back.
  # The final product is something like this:
  #   ['.', '/vendor-folder', /site-packages/x', 'site-packages/y']
  sys.path.extend(remainder)
<commit_msg>vendor: add more comment about darth modifications<commit_after>#
# Copyright 2014 Jon Wayne Parrott, [proppy], Michael R. Bernstein
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# Notes:
# - Imported from https://github.com/jonparrott/Darth-Vendor/
# - Renamed `darth.vendor` to `vendor.add` to match upcoming SDK interface.
# - Renamed `position` param to `index` to match upcoming SDK interface.

import site
import os.path
import sys

def add(folder, index=1):
  """
  Adds the given folder to the python path. Supports namespaced packages.
  By default, packages in the given folder take precedence over site-packages
  and any previous path manipulations.

  Args:
    folder: Path to the folder containing packages, relative to ``os.getcwd()``
    position: Where in ``sys.path`` to insert the vendor packages. By default
      this is set to 1. It is inadvisable to set it to 0 as it will override
      any modules in the current working directory.
  """

  # Check if the path contains a virtualenv.
  site_dir = os.path.join(folder, 'lib', 'python' + sys.version[:3], 'site-packages')
  if os.path.exists(site_dir):
    folder = site_dir
  # Otherwise it's just a normal path, make it absolute.
  else:
    folder = os.path.join(os.path.dirname(__file__), folder)

  # Use site.addsitedir() because it appropriately reads .pth
  # files for namespaced packages. Unfortunately, there's not an
  # option to choose where addsitedir() puts its paths in sys.path
  # so we have to do a little bit of magic to make it play along.

  # We're going to grab the current sys.path and split it up into
  # the first entry and then the rest. Essentially turning
  #   ['.', '/site-packages/x', 'site-packages/y']
  # into
  #   ['.'] and ['/site-packages/x', 'site-packages/y']
  # The reason for this is we want '.' to remain at the top of the
  # list but we want our vendor files to override everything else.
  sys.path, remainder = sys.path[:1], sys.path[1:]

  # Now we call addsitedir which will append our vendor directories
  # to sys.path (which was truncated by the last step.)
  site.addsitedir(folder)

  # Finally, we'll add the paths we removed back.
  # The final product is something like this:
  #   ['.', '/vendor-folder', /site-packages/x', 'site-packages/y']
  sys.path.extend(remainder)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

'''
Script to evaluate POS performance over an automatically tagged file.
'''

import argparse
from itertools import izip

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('system', help='File tagged by system')
    parser.add_argument('gold', help='Gold File')
    args = parser.parse_args()

    total_tokens = 0
    hits = 0

    with open(args.system, 'rb') as fs, open(args.gold, 'rb') as fg:
        for line_system, line_gold in izip(fs, fg):
            items_system = line_system.split()
            items_gold = line_gold.split()

            for item_system, item_gold in izip(items_system, items_gold):
                token_system, tag_system = item_system.rsplit('_', 1)
                token_gold, tag_gold = item_gold.rsplit('_', 1)

                assert token_gold == token_system, 'Tokens differ.'

                if tag_gold == tag_system:
                    hits += 1
                total_tokens += 1

    accuracy = 100 * float(hits) / total_tokens

    print '{} hits out of {}'.format(hits, total_tokens)
    print 'Accuracy: {:f}'.format(accuracy)
<commit_msg>counting common errors<commit_after># -*- coding: utf-8 -*-

'''
Script to evaluate POS performance over an automatically tagged file.
'''

import argparse
from collections import Counter
from itertools import izip

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('system', help='File tagged by system')
    parser.add_argument('gold', help='Gold File')
    parser.add_argument('-c', type=int, dest='count_words', default=None,
                        help='Count words tagged wrong most often and show the top ones')
    args = parser.parse_args()

    total_tokens = 0
    hits = 0
    if args.count_words:
        wrong_words = Counter()

    with open(args.system, 'rb') as fs, open(args.gold, 'rb') as fg:
        for line_system, line_gold in izip(fs, fg):
            items_system = line_system.split()
            items_gold = line_gold.split()

            for item_system, item_gold in izip(items_system, items_gold):
                token_system, tag_system = item_system.rsplit('_', 1)
                token_gold, tag_gold = item_gold.rsplit('_', 1)

                assert token_gold == token_system, 'Tokens differ.'

                if tag_gold == tag_system:
                    hits += 1
                elif args.count_words:
                    lowered = token_system.lower()
                    wrong_words[lowered] += 1

                total_tokens += 1

    accuracy = 100 * float(hits) / total_tokens

    print '{} hits out of {}'.format(hits, total_tokens)
    print 'Accuracy: {:f}'.format(accuracy)

    if args.count_words:
        print
        print 'Words tagged wrong most often:'
        top_words = wrong_words.most_common(args.count_words)
        for word, count in top_words:
            print word, count
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""Determine package version for git repositories from tags

Each time this file is imported it checks if the ".git" folder is
present and if so, obtains the version from the git history using
`git describe`. This information is then stored in the file
`_version_save.py` which is not versioned by git, but distributed
along e.g. on PyPI.
"""
from __future__ import print_function

# Put the entire script into a `True` statement and add the hint
# `pragma: no cover` to ignore code coverage here.
if True:  # pragma: no cover
    import imp
    import os
    from os.path import abspath, basename, dirname, join
    import subprocess
    import sys
    import time
    import traceback
    import warnings

    def git_describe():
        """
        Return a string describing the version returned by the
        command `git describe --tags HEAD`.
        If it is not possible to determine the correct version,
        then an empty string is returned.
        """
        # Make sure we are in a directory that belongs to the correct
        # repository.
        ourdir = dirname(abspath(__file__))

        def _minimal_ext_cmd(cmd):
            # construct minimal environment
            env = {}
            for k in ['SYSTEMROOT', 'PATH']:
                v = os.environ.get(k)
                if v is not None:
                    env[k] = v
            # LANGUAGE is used on win32
            env['LANGUAGE'] = 'C'
            env['LANG'] = 'C'
            env['LC_ALL'] = 'C'
            pop = subprocess.Popen(cmd,
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE,
                                   env=env)
            out = pop.communicate()[0]
            return out

        # change directory
        olddir = abspath(os.curdir)
        os.chdir(ourdir)

        try:
            out = _minimal_ext_cmd(['git', 'describe', '--tags', 'HEAD'])
            git_revision = out.strip().decode('ascii')
        except OSError:
            git_revision = ""

        # go back to original directory
        os.chdir(olddir)

        return git_revision

    def load_version(versionfile):
        """load version from version_save.py"""
        longversion = ""
        try:
            _version_save = imp.load_source("_version_save", versionfile)
            longversion = _version_save.longversion
        except BaseException:
            try:
                from ._version_save import longversion
            except BaseException:
                try:
                    from _version_save import longversion
                except BaseException:
                    pass

        return longversion

    def save_version(version, versionfile):
        """save version to version_save.py"""
        data = "#!/usr/bin/env python\n" \
            + "# This file was created automatically\n" \
            + "longversion = '{VERSION}'\n"
        try:
            with open(versionfile, "w") as fd:
                fd.write(data.format(VERSION=version))
        except BaseException:
            msg = "Could not write package version to {}.".format(versionfile)
            warnings.warn(msg)

    hdir = dirname(abspath(__file__))
    if basename(__file__) == "conf.py" and "name" in locals():
        # This script is executed in conf.py from the docs directory
        versionfile = join(join(join(hdir, ".."),
                                name),  # noqa: F821
                           "_version_save.py")
    else:
        # This script is imported as a module
        versionfile = join(hdir, "_version_save.py")

    # Determine the accurate version
    longversion = ""

    # 1. git describe
    try:
        # Get the version using `git describe`
        longversion = git_describe()
    except BaseException:
        pass

    # 2. previously created version file
    if longversion == "":
        # Either this is this is not a git repository or we are in the
        # wrong git repository.
        # Get the version from the previously generated `_version_save.py`
        longversion = load_version(versionfile)

    # 3. last resort: date
    if longversion == "":
        print("Could not determine version. Reason:")
        print(traceback.format_exc())
        ctime = os.stat(__file__)[8]
        longversion = time.strftime("%Y.%m.%d-%H-%M-%S", time.gmtime(ctime))
        print("Using creation time as version: {}".format(longversion))

    if not hasattr(sys, 'frozen'):
        # Save the version to `_version_save.py` to allow distribution using
        # `python setup.py sdist`.
        # This is only done if the program is not frozen (with e.g.
        # pyinstaller),
        if longversion != load_version(versionfile):
            save_version(longversion, versionfile)

    # PEP 440-conform development version:
    version = ".dev".join(longversion.split("-")[:2])
<commit_msg>update radontea/_version.py with git_cast_file2repos.py<commit_after>#!/usr/bin/env python
"""Determine package version for git repositories from tags

Each time this file is imported it checks if the ".git" folder is
present and if so, obtains the version from the git history using
`git describe`. This information is then stored in the file
`_version_save.py` which is not versioned by git, but distributed
along e.g. on PyPI.
"""
from __future__ import print_function

# Put the entire script into a `True` statement and add the hint
# `pragma: no cover` to ignore code coverage here.
if True:  # pragma: no cover
    import imp
    import os
    from os.path import abspath, basename, dirname, join
    import subprocess
    import sys
    import time
    import traceback
    import warnings

    def git_describe():
        """
        Return a string describing the version returned by the
        command `git describe --tags HEAD`.
        If it is not possible to determine the correct version,
        then an empty string is returned.
        """
        # Make sure we are in a directory that belongs to the correct
        # repository.
        ourdir = dirname(abspath(__file__))

        def _minimal_ext_cmd(cmd):
            # construct minimal environment
            env = {}
            for k in ['SYSTEMROOT', 'PATH']:
                v = os.environ.get(k)
                if v is not None:
                    env[k] = v
            # LANGUAGE is used on win32
            env['LANGUAGE'] = 'C'
            env['LANG'] = 'C'
            env['LC_ALL'] = 'C'
            pop = subprocess.Popen(cmd,
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE,
                                   env=env)
            out = pop.communicate()[0]
            return out

        # change directory
        olddir = abspath(os.curdir)
        os.chdir(ourdir)

        try:
            out = _minimal_ext_cmd(['git', 'describe', '--tags', 'HEAD'])
            git_revision = out.strip().decode('ascii')
        except OSError:
            git_revision = ""

        # go back to original directory
        os.chdir(olddir)

        return git_revision

    def load_version(versionfile):
        """load version from version_save.py"""
        longversion = ""
        try:
            _version_save = imp.load_source("_version_save", versionfile)
            longversion = _version_save.longversion
        except BaseException:
            try:
                from ._version_save import longversion
            except BaseException:
                try:
                    from _version_save import longversion
                except BaseException:
                    pass

        return longversion

    def save_version(version, versionfile):
        """save version to version_save.py"""
        data = "#!/usr/bin/env python\n" \
            + "# This file was created automatically\n" \
            + "longversion = '{VERSION}'\n"
        try:
            with open(versionfile, "w") as fd:
                fd.write(data.format(VERSION=version))
        except BaseException:
            msg = "Could not write package version to {}.".format(versionfile)
            warnings.warn(msg)

    hdir = dirname(abspath(__file__))
    if basename(__file__) == "conf.py" and "name" in locals():
        # This script is executed in conf.py from the docs directory
        versionfile = join(join(join(hdir, ".."),
                                name),  # noqa: F821
                           "_version_save.py")
    else:
        # This script is imported as a module
        versionfile = join(hdir, "_version_save.py")

    # Determine the accurate version
    longversion = ""

    # 1. git describe
    try:
        # Get the version using `git describe`
        longversion = git_describe()
    except BaseException:
        pass

    # 2. previously created version file
    if longversion == "":
        # Either this is this is not a git repository or we are in the
        # wrong git repository.
        # Get the version from the previously generated `_version_save.py`
        longversion = load_version(versionfile)

    # 3. last resort: date
    if longversion == "":
        print("Could not determine version. Reason:")
        print(traceback.format_exc())
        ctime = os.stat(__file__)[8]
        longversion = time.strftime("%Y.%m.%d-%H-%M-%S", time.gmtime(ctime))
        print("Using creation time as version: {}".format(longversion))

    if not hasattr(sys, 'frozen'):
        # Save the version to `_version_save.py` to allow distribution using
        # `python setup.py sdist`.
        # This is only done if the program is not frozen (with e.g.
        # pyinstaller),
        if longversion != load_version(versionfile):
            save_version(longversion, versionfile)

    # PEP 440-conform development version:
    version = ".post".join(longversion.split("-")[:2])
<|endoftext|>


<commit_before>"""Convert articles to tf-df vectors"""

from articurate.pymotherlode import api

import nltk
from nltk.stem.wordnet import WordNetLemmatizer
import math
import numpy
import json

from articurate.metrics import metrics
from articurate.pymotherlode.api import *

from articurate.utils.config import *

IDF = {} # to make script faster
unique_tokens_dict = {} # unique_token["token"] = id
unique_tokens = [] # list of tokens

def truncated_SVD_vector(vectors, reduced_dim): # transforms input vectors to lower dimensions using SVD of reduced dim

    U, S, V = numpy.linalg.svd(vectors, full_matrices=False)

    print U.shape, S.shape, V.shape

    newU = U[:, :reduced_dim]
    newS = S[:reduced_dim]
    newV = V[:reduced_dim, :]

    print newU.shape, newS.shape, newV.shape

    diagS = numpy.zeros((reduced_dim, reduced_dim))
    diagS[:, :] = numpy.diag(newS)

    truncated_vectors = numpy.dot(newU, numpy.dot(diagS, newV))

    print numpy.allclose(vectors, truncated_vectors)

    dummy = raw_input()

    return truncated_vectors

def tf(word, document, method = 'log'): # finds the term frequency of word in document

    if method == 'log':
        return math.log(document.count(word) + 1)
    if method == 'absolute':
        return document.count(word)

def tfidf(text, boost_ne, ne_dict = {}): # creates the tf-idf vector using the global 'unique_tokens_dict'

    text = list(text)
    unique_text = set(text) # for iteration to create vector

    # our tfidf vector, initialized to zero
    word_tfidf = [0]*len(unique_tokens_dict)

    # euclidean norm to divide by
    euclidean_norm = 0

    # default boost of one
    boost = 1
    num_ne = 0

    # populate vector
    for word in unique_text:

        if boost_ne:
            # check if work in ne_dict
            if word in ne_dict['ORGANIZATION'] or word in ne_dict['LOCATION'] or word in ne_dict['PERSON']:
                boost = 1.2  # ne boost
                num_ne = num_ne + 1 # increse number of named entities found

        index = unique_tokens_dict[word]
        word_tfidf[index] = tf(word, text, 'log') * IDF[word] * boost

    # find the euclidean norm of the vector
    euclidean_norm = math.sqrt(sum([item*item for item in word_tfidf]))

    # normalize the tfidf vector
    word_tfidf = [item/euclidean_norm for item in word_tfidf]

    return {'tfidf_vector': word_tfidf, 'num_ne': num_ne}

def extend_stopwords(stopwords, area): # adds more stopwords as required

    if area == 'technology':
        stopwords.extend(['techcrunch', 'verge', 'allthingsd', 'engadget', 'gigaom', 'huffpost', 'wsj', 'bloomberg',
                      'businessweek', 'vogue', 'technica', '9to5mac', 'reuters', '9to5',
                      'cnet', 'zdnet', 'venturebeat', 'forbes',
                      'anandtech', 'macrumors', 'dealbook', 'businessinsider', 'torrentfreak', 'ap',
                      'bgr', 'spiegel', 'citeworld', 'hillicon', '9to5google', 'techdirt', 'supersite',
                      'latimes', 'geekwire', 'fiercewireless', 'co.design', 'bbc', 'fireball', 'daring',
                      'pandodaily', 'usatoday', 'androidpolice', 'uncrunched', 'ifixit', 'appleinsider',
                      'slashgear', 'usatoday', 'techmeme', 'variety', 'ign', 'valleywag', 'readwrite',
                      'phonearena', 'newyorker', 'bizjournals', 'nakedsecurity', 'moneybeat', 'marco',
                      'techinasia', 'searchengineland', 'stratechery', 'tuaw', 'eurogamer', 'datacenterknowledge',
                      'moneybeat', 'agileblog', 'globeandmail', 'theregister', 'localytics', 'hollywoodreporter',
                      'theregister', 'pcworld', 'buzzfeed', 'firedoglake', 'laptopmag', 'hunterwalk', 'mashable',
                      'adage', 'adweek'])

    return stopwords

@metrics.track_by("tech")
def get_stopwords():
    # list of stopwords
    stopwords = nltk.corpus.stopwords.words('english')
    stopwords = extend_stopwords(stopwords, 'technology')
    return stopwords

def cleanify_text(text): # converts text to tokens

    # our stemmer
    lmtzr = WordNetLemmatizer()

    # list of stopwords
    #stopwords = nltk.corpus.stopwords.words('english')
    #stopwords = extend_stopwords(stopwords, 'technology')

    #Do this for storing it first time in DB
    if config['db.coldStart']:
        stopwords = get_stopwords()
    #Do this for retrieving from DB henceforth
    else:
        stopwords = getMetricByKey("articurate.clustering.vectorer.get_stopwords", "tech")

    # tokenize article (makes list of words), make everything smallcaps, and lemmatize it (keeps only stems, eg: 'winning' to 'win')
    tokens = [lmtzr.lemmatize(w.lower()) for w in nltk.word_tokenize(text.encode("utf-8"))]

    # removes tokens that are stopwords
    tokens = [token for token in tokens if token not in stopwords]

    # only keeps tokens of length greater than 2
    tokens = [token for token in tokens if len(token) > 2]

    # removes tokens that are numbers
    tokens = [token for token in tokens if not token.isdigit()]

    return tokens

def vectorize_articles(articles, only_titles = True, use_SVD = False, boost_ne = True): # given a list of articles, converts to the tf-idf vector space

    global unique_tokens
    global unique_tokens_dict
    global IDF

    num_articles = len(articles)

    # collection of all texts and titles
    texts = []
    titles = []

    print "Generating tokens"
    for count, article in enumerate(articles):

        if count % 100 == 0:
            print count

        # get tokens from article
        tokens = cleanify_text(article.title) if only_titles == True else cleanify_text(article.content)

        # create text object from tokens and add to collection of texts
        texts.append(nltk.Text(tokens))

        # make entry in IDF dictionary
        for token in set(tokens):
            if token not in IDF:
                IDF[token] = 1
            else:
                IDF[token] = IDF[token] + 1
    print "Finished generating tokens\n"

    # populate the Inverse Document Frequency
    print "Initializing IDF dictionary"
    for key in IDF:
        IDF[key] = math.log(num_articles/IDF[key])
    print "IDF dictionary ready for use\n"

    # convert tokens to collection object
    collection = nltk.TextCollection(texts)
    print "Created a collection of", len(collection), "terms."

    # get a list of unique tokens, and recreate reverse dictionary
    unique_tokens = list(set(collection))
    for count, term in enumerate(unique_tokens):
        unique_tokens_dict[term] = count
    print "Unique terms found: ", len(unique_tokens_dict), "\n"

    # And here we actually call the function and create our array of vectors.
    print "Creating vectors"
    vectors = []

    # get the NER list
    ne_dict = json.loads(api.getMetric("articurate.nertagger.celery_tasks.save_celery")) if boost_ne else {}

    for count, item in enumerate(texts):

        if count % 100 == 0:
            print count

        # get the tfidf vector
        tfidf_info = tfidf(item, boost_ne, ne_dict)
        tfidf_vector = tfidf_info['tfidf_vector']
        articles[count].tfidf_vector = tfidf_vector # insert in article object
        articles[count].num_ne = tfidf_info['num_ne'] # insert number of named entities in article object
        vectors.append(numpy.array(tfidf_vector))
    print "Vectors created\n"

    return IDF, unique_tokens_dict, unique_tokens, vectors

# SCRATCH

        # generate bigrams
        #bi_tokens = nltk.bigrams(tokens)
        #tri_tokens = nltk.trigrams(tokens)
        #bi_tokens = [' '.join(token).lower() for token in bi_tokens]
        #bi_tokens = [token for token in bi_tokens if token not in stopwords]
        # generate trigrams
        #tri_tokens = [' '.join(token).lower() for token in tri_tokens]
        #tri_tokens = [token for token in tri_tokens if token not in stopwords]

<commit_msg>Cold start bugfix<commit_after>"""Convert articles to tf-df vectors"""

from articurate.pymotherlode import api

import nltk
from nltk.stem.wordnet import WordNetLemmatizer
import math
import numpy
import json

from articurate.metrics import metrics
from articurate.pymotherlode.api import *

from articurate.utils.config import *

IDF = {} # to make script faster
unique_tokens_dict = {} # unique_token["token"] = id
unique_tokens = [] # list of tokens

def truncated_SVD_vector(vectors, reduced_dim): # transforms input vectors to lower dimensions using SVD of reduced dim

    U, S, V = numpy.linalg.svd(vectors, full_matrices=False)

    print U.shape, S.shape, V.shape

    newU = U[:, :reduced_dim]
    newS = S[:reduced_dim]
    newV = V[:reduced_dim, :]

    print newU.shape, newS.shape, newV.shape

    diagS = numpy.zeros((reduced_dim, reduced_dim))
    diagS[:, :] = numpy.diag(newS)

    truncated_vectors = numpy.dot(newU, numpy.dot(diagS, newV))

    print numpy.allclose(vectors, truncated_vectors)

    dummy = raw_input()

    return truncated_vectors

def tf(word, document, method = 'log'): # finds the term frequency of word in document

    if method == 'log':
        return math.log(document.count(word) + 1)
    if method == 'absolute':
        return document.count(word)

def tfidf(text, boost_ne, ne_dict = {}): # creates the tf-idf vector using the global 'unique_tokens_dict'

    text = list(text)
    unique_text = set(text) # for iteration to create vector

    # our tfidf vector, initialized to zero
    word_tfidf = [0]*len(unique_tokens_dict)

    # euclidean norm to divide by
    euclidean_norm = 0

    # default boost of one
    boost = 1
    num_ne = 0

    # populate vector
    for word in unique_text:

        if boost_ne:
            # check if work in ne_dict
            if word in ne_dict['ORGANIZATION'] or word in ne_dict['LOCATION'] or word in ne_dict['PERSON']:
                boost = 1.2  # ne boost
                num_ne = num_ne + 1 # increse number of named entities found

        index = unique_tokens_dict[word]
        word_tfidf[index] = tf(word, text, 'log') * IDF[word] * boost

    # find the euclidean norm of the vector
    euclidean_norm = math.sqrt(sum([item*item for item in word_tfidf]))

    # normalize the tfidf vector
    word_tfidf = [item/euclidean_norm for item in word_tfidf]

    return {'tfidf_vector': word_tfidf, 'num_ne': num_ne}

def extend_stopwords(stopwords, area): # adds more stopwords as required

    if area == 'technology':
        stopwords.extend(['techcrunch', 'verge', 'allthingsd', 'engadget', 'gigaom', 'huffpost', 'wsj', 'bloomberg',
                      'businessweek', 'vogue', 'technica', '9to5mac', 'reuters', '9to5',
                      'cnet', 'zdnet', 'venturebeat', 'forbes',
                      'anandtech', 'macrumors', 'dealbook', 'businessinsider', 'torrentfreak', 'ap',
                      'bgr', 'spiegel', 'citeworld', 'hillicon', '9to5google', 'techdirt', 'supersite',
                      'latimes', 'geekwire', 'fiercewireless', 'co.design', 'bbc', 'fireball', 'daring',
                      'pandodaily', 'usatoday', 'androidpolice', 'uncrunched', 'ifixit', 'appleinsider',
                      'slashgear', 'usatoday', 'techmeme', 'variety', 'ign', 'valleywag', 'readwrite',
                      'phonearena', 'newyorker', 'bizjournals', 'nakedsecurity', 'moneybeat', 'marco',
                      'techinasia', 'searchengineland', 'stratechery', 'tuaw', 'eurogamer', 'datacenterknowledge',
                      'moneybeat', 'agileblog', 'globeandmail', 'theregister', 'localytics', 'hollywoodreporter',
                      'theregister', 'pcworld', 'buzzfeed', 'firedoglake', 'laptopmag', 'hunterwalk', 'mashable',
                      'adage', 'adweek'])

    return stopwords

@metrics.track_by("tech")
def get_stopwords():
    # list of stopwords
    stopwords = nltk.corpus.stopwords.words('english')
    stopwords = extend_stopwords(stopwords, 'technology')
    return stopwords

def cleanify_text(text): # converts text to tokens

    # our stemmer
    lmtzr = WordNetLemmatizer()

    # list of stopwords
    #stopwords = nltk.corpus.stopwords.words('english')
    #stopwords = extend_stopwords(stopwords, 'technology')

    #Do this for storing it first time in DB
    if config['db.coldStart']:
        stopwords = get_stopwords()
    #Do this for retrieving from DB henceforth
    else:
        stopwords = getMetricByKey("articurate.clustering.vectorer.get_stopwords", "tech")

    # tokenize article (makes list of words), make everything smallcaps, and lemmatize it (keeps only stems, eg: 'winning' to 'win')
    tokens = [lmtzr.lemmatize(w.lower()) for w in nltk.word_tokenize(text.encode("utf-8"))]

    # removes tokens that are stopwords
    tokens = [token for token in tokens if token not in stopwords]

    # only keeps tokens of length greater than 2
    tokens = [token for token in tokens if len(token) > 2]

    # removes tokens that are numbers
    tokens = [token for token in tokens if not token.isdigit()]

    return tokens

def vectorize_articles(articles, only_titles = True, use_SVD = False, boost_ne = True): # given a list of articles, converts to the tf-idf vector space

    global unique_tokens
    global unique_tokens_dict
    global IDF

    num_articles = len(articles)

    # collection of all texts and titles
    texts = []
    titles = []

    print "Generating tokens"
    for count, article in enumerate(articles):

        if count % 100 == 0:
            print count

        # get tokens from article
        tokens = cleanify_text(article.title) if only_titles == True else cleanify_text(article.content)

        # create text object from tokens and add to collection of texts
        texts.append(nltk.Text(tokens))

        # make entry in IDF dictionary
        for token in set(tokens):
            if token not in IDF:
                IDF[token] = 1
            else:
                IDF[token] = IDF[token] + 1
    print "Finished generating tokens\n"

    # populate the Inverse Document Frequency
    print "Initializing IDF dictionary"
    for key in IDF:
        IDF[key] = math.log(num_articles/IDF[key])
    print "IDF dictionary ready for use\n"

    # convert tokens to collection object
    collection = nltk.TextCollection(texts)
    print "Created a collection of", len(collection), "terms."

    # get a list of unique tokens, and recreate reverse dictionary
    unique_tokens = list(set(collection))
    for count, term in enumerate(unique_tokens):
        unique_tokens_dict[term] = count
    print "Unique terms found: ", len(unique_tokens_dict), "\n"

    # And here we actually call the function and create our array of vectors.
    print "Creating vectors"
    vectors = []

    # get the NER list
    try:
        ne_dict = json.loads(api.getMetric("articurate.nertagger.celery_tasks.save_celery")) if boost_ne else {}
    except:
        ne_dict = {}

    for count, item in enumerate(texts):

        if count % 100 == 0:
            print count

        # get the tfidf vector
        tfidf_info = tfidf(item, boost_ne, ne_dict)
        tfidf_vector = tfidf_info['tfidf_vector']
        articles[count].tfidf_vector = tfidf_vector # insert in article object
        articles[count].num_ne = tfidf_info['num_ne'] # insert number of named entities in article object
        vectors.append(numpy.array(tfidf_vector))
    print "Vectors created\n"

    return IDF, unique_tokens_dict, unique_tokens, vectors

# SCRATCH

        # generate bigrams
        #bi_tokens = nltk.bigrams(tokens)
        #tri_tokens = nltk.trigrams(tokens)
        #bi_tokens = [' '.join(token).lower() for token in bi_tokens]
        #bi_tokens = [token for token in bi_tokens if token not in stopwords]
        # generate trigrams
        #tri_tokens = [' '.join(token).lower() for token in tri_tokens]
        #tri_tokens = [token for token in tri_tokens if token not in stopwords]

<|endoftext|>


<commit_before>import tempfile
import os
import bcbio.pipeline.datadict as dd
from bcbio.distributed.transaction import file_transaction
from bcbio.provenance import do
from bcbio.utils import (file_exists, safe_makedir, is_gzipped, rbind, partition)
from bcbio.pipeline import config_utils
from bcbio.rnaseq import gtf
import pandas as pd
import numpy as np

def run_sailfish(data):
    samplename = dd.get_sample_name(data)
    files = dd.get_input_sequence_files(data)
    work_dir = dd.get_work_dir(data)
    if len(files) == 2:
        fq1, fq2 = files
    else:
        fq1, fq2 = files[0], None
    sailfish_dir = os.path.join(work_dir, "sailfish", samplename)
    gtf_file = dd.get_gtf_file(data)
    assert file_exists(gtf_file), "%s was not found, exiting." % gtf_file
    fasta_file = dd.get_ref_file(data)
    assert file_exists(fasta_file), "%s was not found, exiting." % fasta_file
    stranded = dd.get_strandedness(data).lower()
    out_file = sailfish(fq1, fq2, sailfish_dir, gtf_file, fasta_file, stranded, data)
    data = dd.set_sailfish(data, out_file)
    data = dd.set_sailfish_dir(data, sailfish_dir)
    return [[data]]

def sailfish(fq1, fq2, sailfish_dir, gtf_file, ref_file, strandedness, data):
    safe_makedir(sailfish_dir)
    samplename = dd.get_sample_name(data)
    out_file = os.path.join(sailfish_dir, "quant.sf")
    if file_exists(out_file):
        return out_file
    sailfish_idx = sailfish_index(gtf_file, ref_file, data)
    num_cores = dd.get_num_cores(data)
    sailfish = config_utils.get_program("sailfish", data["config"])
    cmd = "{sailfish} quant -i {sailfish_idx} -p {num_cores} "
    cmd += _libtype_string(fq1, fq2, strandedness)
    fq1_cmd = "{fq1}" if not is_gzipped(fq1) else "<(gzip -cd {fq1})"
    fq1_cmd = fq1_cmd.format(fq1=fq1)
    if not fq2:
        cmd = " -r {fq1_cmd} "
    else:
        fq2_cmd = "{fq2}" if not is_gzipped(fq2) else "<(gzip -cd {fq2})"
        fq2_cmd = fq2_cmd.format(fq2=fq2)
        cmd += " -1 {fq1_cmd} -2 {fq2_cmd} "
        cmd += "--useVBOpt --numBootstraps 30 "
    cmd += "-o {tx_out_dir}"
    message = "Quantifying transcripts in {fq1} and {fq2}."
    with file_transaction(data, sailfish_dir) as tx_out_dir:
        do.run(cmd.format(**locals()), message.format(**locals()), None)
    return out_file

def sailfish_index(gtf_file, ref_file, data):
    sailfish = config_utils.get_program("sailfish", data["config"])
    gtf_fa_dirty = _gtf_to_fasta(gtf_file, ref_file, data)
    gtf_fa = _clean_gtf_fa(gtf_fa_dirty, data)
    out_dir = tempfile.mkdtemp(prefix="sailfish_index")
    cmd = "{sailfish} index -t {gtf_fa} -o {out_dir} -k 25"
    message = "Creating sailfish index for {gtf_fa}."
    do.run(cmd.format(**locals()), message.format(**locals()), None)
    return out_dir

def _libtype_string(fq1, fq2, strandedness):
    """
    supports just the Tophat unstranded/firstrand/secondstrand
    """
    libtype = "-l I" if fq2 else "-l "
    strand = _sailfish_strand_string(strandedness)
    return libtype + strand

def _sailfish_strand_string(strandedness):
    return {'unstranded': "U",
            'firststrand': "SR",
            'secondstrand': "SF"}.get(strandedness, "U")

def _gtf_to_fasta(gtf_file, ref_file, data):
    gtf_fa = tempfile.NamedTemporaryFile(delete=False, suffix=".fa").name
    with file_transaction(data, gtf_fa) as tx_gtf_fa:
        cmd = "gtf_to_fasta {gtf_file} {ref_file} {tx_gtf_fa}"
        message = "Extracting genomic sequences of {gtf_file}."
        do.run(cmd.format(**locals()), message.format(**locals()), None)
    return gtf_fa

def _clean_gtf_fa(gtf_fa, data):
    """
    convert the gtf_to_fasta sequence names to just the transcript ID
    >1 ENST00000389680 chrM+ 648-1601 -> >ENST00000389680
    """
    out_file = tempfile.NamedTemporaryFile(delete=False, suffix=".fa").name
    with file_transaction(data, out_file) as tx_out_file:
        with open(gtf_fa) as in_handle, open(tx_out_file, "w") as out_handle:
            for line in in_handle:
                if line.startswith(">"):
                    line = ">" + line.split()[1] + "\n"
                out_handle.write(line)
    return out_file

def combine_sailfish(samples):
    work_dir = dd.get_in_samples(samples, dd.get_work_dir)
    gtf_file = dd.get_in_samples(samples, dd.get_gtf_file)
    dont_combine, to_combine = partition(dd.get_sailfish,
                                         dd.sample_data_iterator(samples), True)
    if not to_combine:
        return samples

    tidy_file = os.path.join(work_dir, "sailfish", "combined.sf")
    transcript_tpm_file = os.path.join(work_dir, "sailfish",
                                       "combined.isoform.sf.tpm")
    gene_tpm_file = os.path.join(work_dir, "sailfish",
                                 "combined.gene.sf.tpm")
    if not all([file_exists(x) for x in [gene_tpm_file, tidy_file,
                                         transcript_tpm_file]]):
        df = pd.DataFrame()
        for data in to_combine:
            sailfish_file = dd.get_sailfish(data)
            samplename = dd.get_sample_name(data)
            new_df = _sailfish_expression_parser(sailfish_file, samplename)
            if df.empty:
                df = new_df
            else:
                df = rbind([df, new_df])

        with file_transaction(tidy_file) as tx_out_file:
            df.to_csv(tx_out_file, sep="\t", index_label="name")

        with file_transaction(transcript_tpm_file) as  tx_out_file:
            df.pivot(None, "sample", "tpm").to_csv(tx_out_file, sep="\t")

        with file_transaction(gene_tpm_file) as  tx_out_file:
            pivot = df.pivot(None, "sample", "tpm")
            tdf = pd.DataFrame.from_dict(gtf.transcript_to_gene(gtf_file),
                                         orient="index")
            tdf.columns = ["gene_id"]
            pivot = pivot.join(tdf)
            pivot = pivot.groupby("gene_id").agg(np.sum)
            pivot.to_csv(tx_out_file, sep="\t")

    updated_samples = []
    for data in dd.sample_data_iterator(samples):
        data = dd.set_sailfish_tidy(data, tidy_file)
        data = dd.set_sailfish_transcript_tpm(data, transcript_tpm_file)
        data = dd.set_sailfish_gene_tpm(data, gene_tpm_file)
        updated_samples.append([data])
    return updated_samples

def _sailfish_expression_parser(sailfish_file, samplename):
    col_names = ["name", "length", "tpm", "numreads"]
    df = pd.read_csv(sailfish_file, comment="#", header=None, index_col=0,
                     names=col_names, sep="\t")
    df["sample"] = samplename
    return df
<commit_msg>Cleanup whitespace.<commit_after>import tempfile
import os
import bcbio.pipeline.datadict as dd
from bcbio.distributed.transaction import file_transaction
from bcbio.provenance import do
from bcbio.utils import (file_exists, safe_makedir, is_gzipped, rbind, partition)
from bcbio.pipeline import config_utils
from bcbio.rnaseq import gtf
import pandas as pd
import numpy as np

def run_sailfish(data):
    samplename = dd.get_sample_name(data)
    files = dd.get_input_sequence_files(data)
    work_dir = dd.get_work_dir(data)
    if len(files) == 2:
        fq1, fq2 = files
    else:
        fq1, fq2 = files[0], None
    sailfish_dir = os.path.join(work_dir, "sailfish", samplename)
    gtf_file = dd.get_gtf_file(data)
    assert file_exists(gtf_file), "%s was not found, exiting." % gtf_file
    fasta_file = dd.get_ref_file(data)
    assert file_exists(fasta_file), "%s was not found, exiting." % fasta_file
    stranded = dd.get_strandedness(data).lower()
    out_file = sailfish(fq1, fq2, sailfish_dir, gtf_file, fasta_file, stranded, data)
    data = dd.set_sailfish(data, out_file)
    data = dd.set_sailfish_dir(data, sailfish_dir)
    return [[data]]

def sailfish(fq1, fq2, sailfish_dir, gtf_file, ref_file, strandedness, data):
    safe_makedir(sailfish_dir)
    samplename = dd.get_sample_name(data)
    out_file = os.path.join(sailfish_dir, "quant.sf")
    if file_exists(out_file):
        return out_file
    sailfish_idx = sailfish_index(gtf_file, ref_file, data)
    num_cores = dd.get_num_cores(data)
    sailfish = config_utils.get_program("sailfish", data["config"])
    cmd = "{sailfish} quant -i {sailfish_idx} -p {num_cores} "
    cmd += _libtype_string(fq1, fq2, strandedness)
    fq1_cmd = "{fq1}" if not is_gzipped(fq1) else "<(gzip -cd {fq1})"
    fq1_cmd = fq1_cmd.format(fq1=fq1)
    if not fq2:
        cmd = " -r {fq1_cmd} "
    else:
        fq2_cmd = "{fq2}" if not is_gzipped(fq2) else "<(gzip -cd {fq2})"
        fq2_cmd = fq2_cmd.format(fq2=fq2)
        cmd += " -1 {fq1_cmd} -2 {fq2_cmd} "
        cmd += "--useVBOpt --numBootstraps 30 "
    cmd += "-o {tx_out_dir}"
    message = "Quantifying transcripts in {fq1} and {fq2}."
    with file_transaction(data, sailfish_dir) as tx_out_dir:
        do.run(cmd.format(**locals()), message.format(**locals()), None)
    return out_file

def sailfish_index(gtf_file, ref_file, data):
    sailfish = config_utils.get_program("sailfish", data["config"])
    gtf_fa_dirty = _gtf_to_fasta(gtf_file, ref_file, data)
    gtf_fa = _clean_gtf_fa(gtf_fa_dirty, data)
    out_dir = tempfile.mkdtemp(prefix="sailfish_index")
    cmd = "{sailfish} index -t {gtf_fa} -o {out_dir} -k 25"
    message = "Creating sailfish index for {gtf_fa}."
    do.run(cmd.format(**locals()), message.format(**locals()), None)
    return out_dir

def _libtype_string(fq1, fq2, strandedness):
    """
    supports just the Tophat unstranded/firstrand/secondstrand
    """
    libtype = "-l I" if fq2 else "-l "
    strand = _sailfish_strand_string(strandedness)
    return libtype + strand

def _sailfish_strand_string(strandedness):
    return {'unstranded': "U",
            'firststrand': "SR",
            'secondstrand': "SF"}.get(strandedness, "U")

def _gtf_to_fasta(gtf_file, ref_file, data):
    gtf_fa = tempfile.NamedTemporaryFile(delete=False, suffix=".fa").name
    with file_transaction(data, gtf_fa) as tx_gtf_fa:
        cmd = "gtf_to_fasta {gtf_file} {ref_file} {tx_gtf_fa}"
        message = "Extracting genomic sequences of {gtf_file}."
        do.run(cmd.format(**locals()), message.format(**locals()), None)
    return gtf_fa

def _clean_gtf_fa(gtf_fa, data):
    """
    convert the gtf_to_fasta sequence names to just the transcript ID
    >1 ENST00000389680 chrM+ 648-1601 -> >ENST00000389680
    """
    out_file = tempfile.NamedTemporaryFile(delete=False, suffix=".fa").name
    with file_transaction(data, out_file) as tx_out_file:
        with open(gtf_fa) as in_handle, open(tx_out_file, "w") as out_handle:
            for line in in_handle:
                if line.startswith(">"):
                    line = ">" + line.split()[1] + "\n"
                out_handle.write(line)
    return out_file

def combine_sailfish(samples):
    work_dir = dd.get_in_samples(samples, dd.get_work_dir)
    gtf_file = dd.get_in_samples(samples, dd.get_gtf_file)
    dont_combine, to_combine = partition(dd.get_sailfish,
                                         dd.sample_data_iterator(samples), True)
    if not to_combine:
        return samples

    tidy_file = os.path.join(work_dir, "sailfish", "combined.sf")
    transcript_tpm_file = os.path.join(work_dir, "sailfish",
                                       "combined.isoform.sf.tpm")
    gene_tpm_file = os.path.join(work_dir, "sailfish",
                                 "combined.gene.sf.tpm")
    if not all([file_exists(x) for x in [gene_tpm_file, tidy_file,
                                         transcript_tpm_file]]):
        df = pd.DataFrame()
        for data in to_combine:
            sailfish_file = dd.get_sailfish(data)
            samplename = dd.get_sample_name(data)
            new_df = _sailfish_expression_parser(sailfish_file, samplename)
            if df.empty:
                df = new_df
            else:
                df = rbind([df, new_df])
        with file_transaction(tidy_file) as tx_out_file:
            df.to_csv(tx_out_file, sep="\t", index_label="name")
        with file_transaction(transcript_tpm_file) as  tx_out_file:
            df.pivot(None, "sample", "tpm").to_csv(tx_out_file, sep="\t")
        with file_transaction(gene_tpm_file) as  tx_out_file:
            pivot = df.pivot(None, "sample", "tpm")
            tdf = pd.DataFrame.from_dict(gtf.transcript_to_gene(gtf_file),
                                         orient="index")
            tdf.columns = ["gene_id"]
            pivot = pivot.join(tdf)
            pivot = pivot.groupby("gene_id").agg(np.sum)
            pivot.to_csv(tx_out_file, sep="\t")

    updated_samples = []
    for data in dd.sample_data_iterator(samples):
        data = dd.set_sailfish_tidy(data, tidy_file)
        data = dd.set_sailfish_transcript_tpm(data, transcript_tpm_file)
        data = dd.set_sailfish_gene_tpm(data, gene_tpm_file)
        updated_samples.append([data])
    return updated_samples

def _sailfish_expression_parser(sailfish_file, samplename):
    col_names = ["name", "length", "tpm", "numreads"]
    df = pd.read_csv(sailfish_file, comment="#", header=None, index_col=0,
                     names=col_names, sep="\t")
    df["sample"] = samplename
    return df
<|endoftext|>