<commit_before>"""Forecast evaluation and skill scores for deterministic categorial forecasts.
"""

import numpy as np

def scores_det_cat_fcst(pred, obs, thr,
                        scores=['csi']):

    """ Calculate simple and skill scores for deterministic categorical forecasts

    Input:
    ------
    pred: array-like
        predictions
    obs: array-like
        verifiyinig observations
    scores : list
        names of the scores to be computed

    Return:
    ------
    result : list
        the verification results
    """

    # flatten array if 2D
    pred = pred.flatten()
    obs = obs.flatten()

    # apply threshold
    predb = pred > thr
    obsb  = obs > thr

    # calculate hits, misses, false positives, correct rejects
    H_idx = np.logical_and(predb==1, obsb==1) # correctly predicted precip
    F_idx = np.logical_and(predb==1, obsb==0) # predicted precip even though none there
    M_idx = np.logical_and(predb==0, obsb==1) # predicted no precip even though there was
    R_idx = np.logical_and(predb==0, obsb==0) # correctly predicted no precip

    H = sum(H_idx).astype(float) # hits
    M = sum(M_idx).astype(float) # misses
    F = sum(F_idx).astype(float) # false alarms
    R = sum(R_idx).astype(float) # correct rejections
    tot = H+M+F+R

    result = []
    for score in scores:

        # simple scores
        POD = H/(H+M) # probability of detection
        FAR = F/(H+F) # false alarm ratio
        FA = F/(F+R) # false alarm rate = prob of false detection
        s = (H+M)/(H+M+F+R) # base rate = freq of observed events

        if score.lower() == 'pod':
            result.append(POD)
        if score.lower() == 'far':
            result.append(FAR)
        if score.lower() == 'fa':
            result.append(FA)
        if score.lower() == 'acc':
            ACC = (H+R)/(H+M+F+R) # accuracy (fraction correct)
            result.append(ACC)
        if score.lower() == 'csi':
            CSI = H/(H+M+F) # critical success index
            result.append(CSI)

        # skill scores
        if score.lower() == 'hss':
            HSS = 2*(H*R-F*M)/((H+M)*(M+R)+(H+F)*(F+R)) # Heidke Skill Score (-1 < HSS < 1) < 0 implies no skill
            result.append(HSS)
        if score.lower() == 'hk':
            HK = POD-FA # Hanssen-Kuipers Discriminant
            result.append(HK)
        if score.lower() == 'gss':
            GSS = (POD-FA)/((1-s*POD)/(1-s)+FA*(1-s)/s) # Gilbert Skill Score
            result.append(GSS)
        if score.lower() == 'sedi':
            # Symmetric extremal dependence index
            SEDI = (np.log(FA)-np.log(POD)+np.log(1-POD)-np.log(1-FA))/(np.log(FA)
                    +np.log(POD)+np.log(1-POD)+np.log(1-FA))
            result.append(SEDI)

    return result<commit_msg>Convert bool to float before the sum<commit_after>"""Forecast evaluation and skill scores for deterministic categorial forecasts.
"""

import numpy as np
import time

def scores_det_cat_fcst(pred, obs, thr,
                        scores=['csi']):

    """ Calculate simple and skill scores for deterministic categorical forecasts

    Input:
    ------
    pred: array-like
        predictions
    obs: array-like
        verifiyinig observations
    scores : list
        names of the scores to be computed

    Return:
    ------
    result : list
        the verification results
    """

    starttime = time.time()

    # flatten array if 2D
    pred = pred.flatten()
    obs = obs.flatten()

    # apply threshold
    predb = pred > thr
    obsb  = obs > thr

    # calculate hits, misses, false positives, correct rejects
    H_idx = np.logical_and(predb==1, obsb==1) # correctly predicted precip
    F_idx = np.logical_and(predb==1, obsb==0) # predicted precip even though none there
    M_idx = np.logical_and(predb==0, obsb==1) # predicted no precip even though there was
    R_idx = np.logical_and(predb==0, obsb==0) # correctly predicted no precip

    H = sum(H_idx.astype(float)) # hits
    M = sum(M_idx.astype(float)) # misses
    F = sum(F_idx.astype(float)) # false alarms
    R = sum(R_idx.astype(float)) # correct rejections
    tot = H+M+F+R

    result = []
    for score in scores:

        # simple scores
        POD = H/(H+M) # probability of detection
        FAR = F/(H+F) # false alarm ratio
        FA = F/(F+R) # false alarm rate = prob of false detection
        s = (H+M)/(H+M+F+R) # base rate = freq of observed events

        if score.lower() == 'pod':
            result.append(POD)
        if score.lower() == 'far':
            result.append(FAR)
        if score.lower() == 'fa':
            result.append(FA)
        if score.lower() == 'acc':
            ACC = (H+R)/(H+M+F+R) # accuracy (fraction correct)
            result.append(ACC)
        if score.lower() == 'csi':
            CSI = H/(H+M+F) # critical success index
            result.append(CSI)

        # skill scores
        if score.lower() == 'hss':
            HSS = 2*(H*R-F*M)/((H+M)*(M+R)+(H+F)*(F+R)) # Heidke Skill Score (-1 < HSS < 1) < 0 implies no skill
            result.append(HSS)
        if score.lower() == 'hk':
            HK = POD-FA # Hanssen-Kuipers Discriminant
            result.append(HK)
        if score.lower() == 'gss':
            GSS = (POD-FA)/((1-s*POD)/(1-s)+FA*(1-s)/s) # Gilbert Skill Score
            result.append(GSS)
        if score.lower() == 'sedi':
            # Symmetric extremal dependence index
            SEDI = (np.log(FA)-np.log(POD)+np.log(1-POD)-np.log(1-FA))/(np.log(FA)
                    +np.log(POD)+np.log(1-POD)+np.log(1-FA))
            result.append(SEDI)

    print("done in %.2f seconds." % (time.time() - starttime))

    return result<|endoftext|>


<commit_before>"""
=================
DOMMatrixReadOnly
=================
"""
import numpy as np
import re

class DOMMatrixReadOnly(object):
    """
    [Constructor(sequence<unrestricted double> numberSequence),
        Exposed=(Window,Worker)]
    interface DOMMatrixReadOnly {
        // These attributes are simple aliases for certain elements of the 4x4
        // matrix
        readonly attribute unrestricted double a;
        readonly attribute unrestricted double b;
        readonly attribute unrestricted double c;
        readonly attribute unrestricted double d;
        readonly attribute unrestricted double e;
        readonly attribute unrestricted double f;

        readonly attribute unrestricted double m11;
        readonly attribute unrestricted double m12;
        readonly attribute unrestricted double m13;
        readonly attribute unrestricted double m14;
        readonly attribute unrestricted double m21;
        readonly attribute unrestricted double m22;
        readonly attribute unrestricted double m23;
        readonly attribute unrestricted double m24;
        readonly attribute unrestricted double m31;
        readonly attribute unrestricted double m32;
        readonly attribute unrestricted double m33;
        readonly attribute unrestricted double m34;
        readonly attribute unrestricted double m41;
        readonly attribute unrestricted double m42;
        readonly attribute unrestricted double m43;
        readonly attribute unrestricted double m44;

        readonly attribute boolean is2D;
        readonly attribute boolean isIdentity;

        // Immutable transform methods
        DOMMatrix translate(unrestricted double tx, unrestricted double ty,
            optional unrestricted double tz = 0);
        DOMMatrix scale(unrestricted double scale, optional unrestricted double
            originX = 0, optional unrestricted double originY = 0);
        DOMMatrix scale3d(unrestricted double scale, optional unrestricted
            double originX = 0, optional unrestricted double originY = 0,
            optional unrestricted double originZ = 0);
        DOMMatrix scaleNonUniform(unrestricted double scaleX, optional
            unrestricted double scaleY = 1, optional unrestricted double
            scaleZ = 1, optional unrestricted double originX = 0, optional
            unrestricted double originY = 0, optional unrestricted double
            originZ = 0);
        DOMMatrix rotate(unrestricted double angle, optional unrestricted double
            originX = 0, optional unrestricted double originY = 0);
        DOMMatrix rotateFromVector(unrestricted double x, unrestricted double y);
        DOMMatrix rotateAxisAngle(unrestricted double x, unrestricted double y,
            unrestricted double z, unrestricted double angle);
        DOMMatrix skewX(unrestricted double sx);
        DOMMatrix skewY(unrestricted double sy);
        DOMMatrix multiply(DOMMatrix other);
        DOMMatrix flipX();
        DOMMatrix flipY();
        DOMMatrix inverse();

        DOMPoint            transformPoint(optional DOMPointInit point);
        Float32Array        toFloat32Array();
        Float64Array        toFloat64Array();
                            stringifier;
    };
    """

    #_is2D = False

    def __init__(self, *args, **kwargs):
        """
        The DOMMatrixReadOnly(sequence<unrestricted double> numberSequence)
        constructor, when invoked, must run the following steps:

        If numberSequence has 6 elements
            * Set m11 element, m12 element, m21 element, m22 element, m41
              element, m42 element to the values of numberSequence in order
              starting with the first value. Furthermore, set m31 element, m32
              element, m13 element, m23 element, m43 element, m14 element, m24
              element, m34 element to 0 and m33 element, m44 element to 1.
            * Set is2D to true.
            * Return the new DOMMatrixReadOnly object.

        If numberSequence has 16 elements
            * Set m11 element to m44 element to the values of numberSequence in
              column-major order.
            * Set is2D to false.
            * Return the new DOMMatrixReadOnly object.

        Otherwise
            Throw a TypeError exception.

        Source: https://www.w3.org/TR/geometry-1/#dommatrixreadonly-constructors
        """
        object.__setattr__(self, '_matrix', {
            'm11': 1, 'm12': 0, 'm13': 0, 'm14': 0,
            'm21': 0, 'm22': 1, 'm23': 0, 'm24': 0,
            'm31': 0, 'm32': 0, 'm33': 1, 'm34': 0,
            'm41': 0, 'm42': 0, 'm43': 0, 'm44': 1
        })

        if len(args) is 6:
            self._matrix['m11'] = args[0]
            self._matrix['m12'] = args[1]
            self._matrix['m21'] = args[2]
            self._matrix['m22'] = args[3]
            self._matrix['m41'] = args[4]
            self._matrix['m42'] = args[5]

            # Due to __setattr__ overloading in DOMMatrix
            object.__setattr__(self, '_is2D', True)

        elif len(args) is 16:
            self._matrix['m11'] = args[0]
            self._matrix['m12'] = args[1]
            self._matrix['m13'] = args[2]
            self._matrix['m14'] = args[3]
            self._matrix['m21'] = args[4]
            self._matrix['m22'] = args[5]
            self._matrix['m23'] = args[6]
            self._matrix['m24'] = args[7]
            self._matrix['m31'] = args[8]
            self._matrix['m32'] = args[9]
            self._matrix['m33'] = args[10]
            self._matrix['m34'] = args[11]
            self._matrix['m41'] = args[12]
            self._matrix['m42'] = args[13]
            self._matrix['m43'] = args[14]
            self._matrix['m44'] = args[15]

            # Due to __setattr__ overloading in DOMMatrix
            object.__setattr__(self, '_is2D', False)

        elif kwargs is not None and bool(kwargs):
            for key, value in kwargs.items():
                match = re.search(r'^m[1-4][1-4]$', key)
                if match and key in self._matrix:
                    self._matrix[key] = value
                else:
                    raise TypeError()

        else:
            raise TypeError()

    @property
    def m11(self):
        return self._matrix['m11']

    @property
    def m12(self):
        return self._matrix['m12']

    @property
    def m13(self):
        return self._matrix['m13']

    @property
    def m14(self):
        return self._matrix['m14']

    @property
    def m21(self):
        return self._matrix['m21']

    @property
    def m22(self):
        return self._matrix['m22']

    @property
    def m23(self):
        return self._matrix['m23']

    @property
    def m24(self):
        return self._matrix['m24']

    @property
    def m31(self):
        return self._matrix['m31']

    @property
    def m32(self):
        return self._matrix['m32']

    @property
    def m33(self):
        return self._matrix['m33']

    @property
    def m34(self):
        return self._matrix['m34']

    @property
    def m41(self):
        return self._matrix['m41']

    @property
    def m42(self):
        return self._matrix['m42']

    @property
    def m43(self):
        return self._matrix['m43']

    @property
    def m44(self):
        return self._matrix['m44']

    @property
    def a(self):
        return self._matrix['m11']

    @property
    def b(self):
        return self._matrix['m12']

    @property
    def c(self):
        return self._matrix['m21']

    @property
    def d(self):
        return self._matrix['m22']

    @property
    def e(self):
        return self._matrix['m41']

    @property
    def f(self):
        return self._matrix['m42']

    @property
    def is2D(self):
        return self._is2D

    @property
    def isIdentity(self):
        """
        Returns true if m12, m13, m14, m21, m23, m24, m31, m32, m34, m41, m42,
        m43 are 0 and m11, m22, m33, m44 are 1. Otherwise returns false.
        """
        _m = self._matrix

        if (_m['m12'] == _m['m13'] == _m['m14'] == _m['m21'] ==
            _m['m23'] == _m['m24'] == _m['m31'] == _m['m32'] ==
            _m['m34'] == _m['m41'] == _m['m42'] == _m['m42'] == 0) and (
            _m['m11'] == _m['m22'] == _m['m33'] == _m['m44'] == 1):
            return True

        return False

    def _np_array(self, other=None):
        if other is not None:
            _m = other._matrix
        else:
            _m = self._matrix

        return np.array([
            [ _m['m11'], _m['m12'], _m['m13'], _m['m14'] ],
            [ _m['m21'], _m['m22'], _m['m23'], _m['m24'] ],
            [ _m['m31'], _m['m32'], _m['m33'], _m['m34'] ],
            [ _m['m41'], _m['m42'], _m['m43'], _m['m44'] ]
        ])

    def multiply(self, other):
        from dommatrix import DOMMatrix
        _m = np.dot(self._np_array(other), self._np_array())
        return DOMMatrix(*_m.flatten())

    def translateSelf(self, tx, ty, tz=0):
        pass

    def scale(self, scale, ox=0, oy=0):
        pass

    def scale3d(self, scale, ox=0, oy=0, oz=0):
        pass

    def scaleNonUniform(self, sx, sy=1, sz=1, ox=0, oy=0, oz=0):
        pass

    def rotateSelf(self, angle, originX=0, originY=0):
        pass

    def rotateFromVectorSelf(self, x, y):
        pass

    def rotateAxisAngleSelf(self, x, y, z, angle):
        pass

    def skewXSelf(self, sx):
        pass

    def skewYSelf(self, sy):
        pass

    def invertSelf(self):
        pass

    def setMatrixValue(self, transformList):
        pass

    def __str__(self):
        r = ""
        r = r + "[ %s, %s, %s, %s ],\n" % (self.m11, self.m12, self.m13, self.m14)
        r = r + "[ %s, %s, %s, %s ],\n" % (self.m21, self.m22, self.m23, self.m24)
        r = r + "[ %s, %s, %s, %s ],\n" % (self.m31, self.m32, self.m33, self.m34)
        r = r + "[ %s, %s, %s, %s ]\n" % (self.m41, self.m42, self.m43, self.m44)
        r = r + "\n"
        r = r + "a: %s\n" % self.a
        r = r + "b: %s\n" % self.b
        r = r + "c: %s\n" % self.c
        r = r + "d: %s\n" % self.d
        r = r + "e: %s\n" % self.e
        r = r + "f: %s\n" % self.f
        return r
<commit_msg>:facepalm:<commit_after>"""
=================
DOMMatrixReadOnly
=================
"""
import numpy as np
import re

class DOMMatrixReadOnly(object):
    """
    [Constructor(sequence<unrestricted double> numberSequence),
        Exposed=(Window,Worker)]
    interface DOMMatrixReadOnly {
        // These attributes are simple aliases for certain elements of the 4x4
        // matrix
        readonly attribute unrestricted double a;
        readonly attribute unrestricted double b;
        readonly attribute unrestricted double c;
        readonly attribute unrestricted double d;
        readonly attribute unrestricted double e;
        readonly attribute unrestricted double f;

        readonly attribute unrestricted double m11;
        readonly attribute unrestricted double m12;
        readonly attribute unrestricted double m13;
        readonly attribute unrestricted double m14;
        readonly attribute unrestricted double m21;
        readonly attribute unrestricted double m22;
        readonly attribute unrestricted double m23;
        readonly attribute unrestricted double m24;
        readonly attribute unrestricted double m31;
        readonly attribute unrestricted double m32;
        readonly attribute unrestricted double m33;
        readonly attribute unrestricted double m34;
        readonly attribute unrestricted double m41;
        readonly attribute unrestricted double m42;
        readonly attribute unrestricted double m43;
        readonly attribute unrestricted double m44;

        readonly attribute boolean is2D;
        readonly attribute boolean isIdentity;

        // Immutable transform methods
        DOMMatrix translate(unrestricted double tx, unrestricted double ty,
            optional unrestricted double tz = 0);
        DOMMatrix scale(unrestricted double scale, optional unrestricted double
            originX = 0, optional unrestricted double originY = 0);
        DOMMatrix scale3d(unrestricted double scale, optional unrestricted
            double originX = 0, optional unrestricted double originY = 0,
            optional unrestricted double originZ = 0);
        DOMMatrix scaleNonUniform(unrestricted double scaleX, optional
            unrestricted double scaleY = 1, optional unrestricted double
            scaleZ = 1, optional unrestricted double originX = 0, optional
            unrestricted double originY = 0, optional unrestricted double
            originZ = 0);
        DOMMatrix rotate(unrestricted double angle, optional unrestricted double
            originX = 0, optional unrestricted double originY = 0);
        DOMMatrix rotateFromVector(unrestricted double x, unrestricted double y);
        DOMMatrix rotateAxisAngle(unrestricted double x, unrestricted double y,
            unrestricted double z, unrestricted double angle);
        DOMMatrix skewX(unrestricted double sx);
        DOMMatrix skewY(unrestricted double sy);
        DOMMatrix multiply(DOMMatrix other);
        DOMMatrix flipX();
        DOMMatrix flipY();
        DOMMatrix inverse();

        DOMPoint            transformPoint(optional DOMPointInit point);
        Float32Array        toFloat32Array();
        Float64Array        toFloat64Array();
                            stringifier;
    };
    """

    #_is2D = False

    def __init__(self, *args, **kwargs):
        """
        The DOMMatrixReadOnly(sequence<unrestricted double> numberSequence)
        constructor, when invoked, must run the following steps:

        If numberSequence has 6 elements
            * Set m11 element, m12 element, m21 element, m22 element, m41
              element, m42 element to the values of numberSequence in order
              starting with the first value. Furthermore, set m31 element, m32
              element, m13 element, m23 element, m43 element, m14 element, m24
              element, m34 element to 0 and m33 element, m44 element to 1.
            * Set is2D to true.
            * Return the new DOMMatrixReadOnly object.

        If numberSequence has 16 elements
            * Set m11 element to m44 element to the values of numberSequence in
              column-major order.
            * Set is2D to false.
            * Return the new DOMMatrixReadOnly object.

        Otherwise
            Throw a TypeError exception.

        Source: https://www.w3.org/TR/geometry-1/#dommatrixreadonly-constructors
        """
        object.__setattr__(self, '_matrix', {
            'm11': 1, 'm12': 0, 'm13': 0, 'm14': 0,
            'm21': 0, 'm22': 1, 'm23': 0, 'm24': 0,
            'm31': 0, 'm32': 0, 'm33': 1, 'm34': 0,
            'm41': 0, 'm42': 0, 'm43': 0, 'm44': 1
        })

        if len(args) is 6:
            self._matrix['m11'] = args[0]
            self._matrix['m12'] = args[1]
            self._matrix['m21'] = args[2]
            self._matrix['m22'] = args[3]
            self._matrix['m41'] = args[4]
            self._matrix['m42'] = args[5]

            # Due to __setattr__ overloading in DOMMatrix
            object.__setattr__(self, '_is2D', True)

        elif len(args) is 16:
            self._matrix['m11'] = args[0]
            self._matrix['m12'] = args[1]
            self._matrix['m13'] = args[2]
            self._matrix['m14'] = args[3]
            self._matrix['m21'] = args[4]
            self._matrix['m22'] = args[5]
            self._matrix['m23'] = args[6]
            self._matrix['m24'] = args[7]
            self._matrix['m31'] = args[8]
            self._matrix['m32'] = args[9]
            self._matrix['m33'] = args[10]
            self._matrix['m34'] = args[11]
            self._matrix['m41'] = args[12]
            self._matrix['m42'] = args[13]
            self._matrix['m43'] = args[14]
            self._matrix['m44'] = args[15]

            # Due to __setattr__ overloading in DOMMatrix
            object.__setattr__(self, '_is2D', False)

        elif kwargs is not None and bool(kwargs):
            for key, value in kwargs.items():
                match = re.search(r'^m[1-4][1-4]$', key)
                if match and key in self._matrix:
                    self._matrix[key] = value
                else:
                    raise TypeError()

        else:
            raise TypeError()

    @property
    def m11(self):
        return self._matrix['m11']

    @property
    def m12(self):
        return self._matrix['m12']

    @property
    def m13(self):
        return self._matrix['m13']

    @property
    def m14(self):
        return self._matrix['m14']

    @property
    def m21(self):
        return self._matrix['m21']

    @property
    def m22(self):
        return self._matrix['m22']

    @property
    def m23(self):
        return self._matrix['m23']

    @property
    def m24(self):
        return self._matrix['m24']

    @property
    def m31(self):
        return self._matrix['m31']

    @property
    def m32(self):
        return self._matrix['m32']

    @property
    def m33(self):
        return self._matrix['m33']

    @property
    def m34(self):
        return self._matrix['m34']

    @property
    def m41(self):
        return self._matrix['m41']

    @property
    def m42(self):
        return self._matrix['m42']

    @property
    def m43(self):
        return self._matrix['m43']

    @property
    def m44(self):
        return self._matrix['m44']

    @property
    def a(self):
        return self._matrix['m11']

    @property
    def b(self):
        return self._matrix['m12']

    @property
    def c(self):
        return self._matrix['m21']

    @property
    def d(self):
        return self._matrix['m22']

    @property
    def e(self):
        return self._matrix['m41']

    @property
    def f(self):
        return self._matrix['m42']

    @property
    def is2D(self):
        return self._is2D

    @property
    def isIdentity(self):
        """
        Returns true if m12, m13, m14, m21, m23, m24, m31, m32, m34, m41, m42,
        m43 are 0 and m11, m22, m33, m44 are 1. Otherwise returns false.
        """
        _m = self._matrix

        if (_m['m12'] == _m['m13'] == _m['m14'] == _m['m21'] ==
            _m['m23'] == _m['m24'] == _m['m31'] == _m['m32'] ==
            _m['m34'] == _m['m41'] == _m['m42'] == _m['m42'] == 0) and (
            _m['m11'] == _m['m22'] == _m['m33'] == _m['m44'] == 1):
            return True

        return False

    def _np_array(self, other=None):
        if other is not None:
            _m = other._matrix
        else:
            _m = self._matrix

        return np.array([
            [ _m['m11'], _m['m12'], _m['m13'], _m['m14'] ],
            [ _m['m21'], _m['m22'], _m['m23'], _m['m24'] ],
            [ _m['m31'], _m['m32'], _m['m33'], _m['m34'] ],
            [ _m['m41'], _m['m42'], _m['m43'], _m['m44'] ]
        ])

    def multiply(self, other):
        from .dommatrix import DOMMatrix
        _m = np.dot(self._np_array(other), self._np_array())
        return DOMMatrix(*_m.flatten())

    def translateSelf(self, tx, ty, tz=0):
        pass

    def scale(self, scale, ox=0, oy=0):
        pass

    def scale3d(self, scale, ox=0, oy=0, oz=0):
        pass

    def scaleNonUniform(self, sx, sy=1, sz=1, ox=0, oy=0, oz=0):
        pass

    def rotateSelf(self, angle, originX=0, originY=0):
        pass

    def rotateFromVectorSelf(self, x, y):
        pass

    def rotateAxisAngleSelf(self, x, y, z, angle):
        pass

    def skewXSelf(self, sx):
        pass

    def skewYSelf(self, sy):
        pass

    def invertSelf(self):
        pass

    def setMatrixValue(self, transformList):
        pass

    def __str__(self):
        r = ""
        r = r + "[ %s, %s, %s, %s ],\n" % (self.m11, self.m12, self.m13, self.m14)
        r = r + "[ %s, %s, %s, %s ],\n" % (self.m21, self.m22, self.m23, self.m24)
        r = r + "[ %s, %s, %s, %s ],\n" % (self.m31, self.m32, self.m33, self.m34)
        r = r + "[ %s, %s, %s, %s ]\n" % (self.m41, self.m42, self.m43, self.m44)
        r = r + "\n"
        r = r + "a: %s\n" % self.a
        r = r + "b: %s\n" % self.b
        r = r + "c: %s\n" % self.c
        r = r + "d: %s\n" % self.d
        r = r + "e: %s\n" % self.e
        r = r + "f: %s\n" % self.f
        return r
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

#
# Xibo - Digitial Signage - http://www.xibo.org.uk
# Copyright (C) 2009 Alex Harrington
#
# This file is part of Xibo.
#
# Xibo is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# any later version.
#
# Xibo is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Xibo.  If not, see <http://www.gnu.org/licenses/>.
#

from XiboMedia import XiboMedia
from threading import Thread
import urllib
import math

class WebpageMedia(XiboMedia):
    def add(self):
        self.retryCount = 0
        zoomLevel = self.calculateZoomLevel()
        tmpXML = '<browser id="' + self.mediaNodeName + '" opacity="0" width="' + str(self.width) + '" height="' + str(self.height) + \
                '" zoomLevel="' + str(zoomLevel) + '"/>'
        self.p.enqueue('add',(tmpXML,self.regionNodeName))

    def run(self):
        self.p.enqueue('browserNavigate',(self.mediaNodeName,urllib.unquote(str(self.options['uri'])),self.finishedRendering))
        self.startStats()

    def requiredFiles(self):
        return []

    def dispose(self):
        self.returnStats()
        self.p.enqueue('del',self.mediaNodeName)
        self.parent.tNext()

    def calculateZoomLevel(self):
        # Calling the zoomIn/zoomOut methods of a browser causes a 20% zoom in/out
        # Calculate how many calls to make
        n = 100
        count = 0
        try:
            n = int(self.options['scaling'])
        except:
            pass

        if n == 100:
            pass
        else:
            if n < 100:
                # Zoom Out
                count = -int(math.ceil((100 - n) / 20.0))
            else:
                # Zoom In
                count = int(math.ceil((n - 100) / 20.0))
        return count

    def finishedRendering(self):
        bo = self.browserOptions()
        optionsTuple = (self.mediaNodeName,bo[0],bo[1])
        self.p.enqueue('browserOptions',optionsTuple)
        currentNode = self.p.getElementByID(self.mediaNodeName)
        if currentNode.painted():
            # Make the browser visible
            self.p.enqueue('setOpacity',(self.mediaNodeName,1))
            self.p.enqueue('timer',(int(self.duration) * 1000,self.parent.next))
        else:
            if self.retryCount > 3:
                print "Error rendering %s. Skipping" % self.mediaNodeName
                self.parent.next()
            else:
                if self.retryCount > 1:
                    print "Error rendering %s. Re-rendering" % self.mediaNodeName

                self.retryCount = self.retryCount + 1
                self.p.enqueue('browserNavigate',(self.mediaNodeName,urllib.unquote(str(self.options['uri'])),self.finishedRendering))

    def browserOptions(self):
        scroll = False
        trans = False

        # Decide if the browser background should be transparent
        try:
            if self.options['transparency'] == "1":
                trans = True
        except:
            pass

        return (trans,scroll)
<commit_msg>[pyclient] Implemented webpage zoom using css zoom instead of Berkelium's broken zoom function<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

#
# Xibo - Digitial Signage - http://www.xibo.org.uk
# Copyright (C) 2009 Alex Harrington
#
# This file is part of Xibo.
#
# Xibo is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# any later version.
#
# Xibo is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Xibo.  If not, see <http://www.gnu.org/licenses/>.
#

from XiboMedia import XiboMedia
from threading import Thread
import urllib
import math
import os
import sys
import codecs

class WebpageMedia(XiboMedia):
    def add(self):
        self.tmpPath = os.path.join(self.libraryDir,self.mediaNodeName + "-zoom-tmp.html")
        self.retryCount = 0
        tmpXML = '<browser id="' + self.mediaNodeName + '" opacity="0" width="' + str(self.width) + '" height="' + str(self.height) + '"/>'
        self.p.enqueue('add',(tmpXML,self.regionNodeName))

    def run(self):
        self.zoomOffsetWrapper()
        self.p.enqueue('browserNavigate',(self.mediaNodeName,self.options['uri'],self.finishedRendering))
        self.startStats()

    def requiredFiles(self):
        return []

    def dispose(self):
        self.returnStats()
        self.p.enqueue('del',self.mediaNodeName)

        try:
            os.remove(self.tmpPath)
        except:
            pass

        self.parent.tNext()

    def finishedRendering(self):
        bo = self.browserOptions()
        optionsTuple = (self.mediaNodeName,bo[0],bo[1])
        self.p.enqueue('browserOptions',optionsTuple)
        currentNode = self.p.getElementByID(self.mediaNodeName)
        if currentNode.painted():
            # Make the browser visible
            self.p.enqueue('setOpacity',(self.mediaNodeName,1))
            self.p.enqueue('timer',(int(self.duration) * 1000,self.parent.next))
        else:
            if self.retryCount > 3:
                print "Error rendering %s. Skipping" % self.mediaNodeName
                self.parent.next()
            else:
                if self.retryCount > 1:
                    print "Error rendering %s. Re-rendering" % self.mediaNodeName

                self.retryCount = self.retryCount + 1
                self.p.enqueue('browserNavigate',(self.mediaNodeName,self.options['uri'],self.finishedRendering))

    def browserOptions(self):
        scroll = False
        trans = False

        # Decide if the browser background should be transparent
        try:
            if self.options['transparency'] == "1":
                trans = True
        except:
            pass

        return (trans,scroll)

    def zoomOffsetWrapper(self):
        # Decode the URI
        self.options['uri'] = urllib.unquote(str(self.options['uri']))

        try:
            scaling = self.options['scaling']
        except:
            scaling = "100"

        w = self.width
        h = self.height
        zoom = ""
        iframe = ""
        offsetLeft = ""
        offsetTop = ""

        if (scaling != "100"):
            scaling = int(scaling) / 100.0
            w = w * (1 / scaling)
            h = h * (1 / scaling)
            zoom = "-webkit-transform: scale(%s); -webkit-transform-origin: 0 0;" % scaling

        try:
            offsetTop = self.options['offsetTop']
        except:
            offsetTop = "0"

        try:
            offsetLeft = self.options['offsetLeft']
        except:
            offsetLeft = "0"

        # The layout is zoomed or moved
        # Wrap it in an iframe
        if zoom != "" or offsetTop != "0" or offsetLeft != "0":
            iframe = ""
            iframe = iframe % (offsetLeft, offsetTop, zoom, int(w) + int(offsetLeft), int(h) + int(offsetTop), self.options['uri'])

            # Write the iframe to a file and overwrite the uri option to point to our temporary location
            try:
                try:
                    f = codecs.open(self.tmpPath,mode='w',encoding="utf-8")
                    f.write(iframe)
                    iframe = None
                finally:
                    f.close()
            except:
                self.log.log(0,"error","Unable to write " + self.tmpPath)
                self.parent.next()
                return

            # Navigate the browser to the temporary file
            self.options['uri'] = "file://" + os.path.abspath(self.tmpPath)
<|endoftext|>


<commit_before>from xml.etree import ElementTree as et
import os, sys

def reportfordir(path):
    """
    Usage: python3 exemplarlistreport.py directorypath
    """
    print("Report for " + path)
    try:
        for file in sorted(os.listdir(path)):
            cf = os.path.join(path, file)
            print("\n" + file)
            r = et.parse(cf).getroot()
            e = set(r.findall(".//exemplarCharacters[@draft='generated']"))
            if len(e) == 0:
                print("No generated exemplar lists found")
                continue
            d = set(r.findall(".//exemplarCharacters[@draft='generated'][@type]"))
            m = list(e-d)
            if len(m) == 0:
                print("No generated main exemplar list found")
                continue
            else:
                print("Generated main exemplar list: " + m[0].text)
            a = r.findall(".//exemplarCharacters[@draft='generated'][@type='auxiliary']")
            if len(a) == 0:
                print("No generated auxiliary list found")
            else:
                print("Generated auxiliary exemplar list: " + a[0].text)
    except OSError:
        print("Directory not found or empty")

if __name__ == '__main__':
    if len(sys.argv) > 1:
        reportfordir(sys.argv[1])
    else:
        print(reportfordir.__doc__)
<commit_msg>write output to file<commit_after>from xml.etree import ElementTree as et
import os, sys

def reportfordir(path, reportfilename):
    """
    Usage: python3 exemplarlistreport.py directorypath reportfile.txt
    """
    with open(reportfilename, mode="w") as ofile:
        chars = ofile.write("Report for " + path)
        try:
            for file in sorted(os.listdir(path)):
                cf = os.path.join(path, file)
                chars = ofile.write("\n" + file)
                r = et.parse(cf).getroot()
                e = set(r.findall(".//exemplarCharacters[@draft='generated']"))
                if len(e) == 0:
                    chars = ofile.write("\nNo generated exemplar lists found")
                    continue
                d = set(r.findall(".//exemplarCharacters[@draft='generated'][@type]"))
                m = list(e-d)
                if len(m) == 0:
                    chars = ofile.write("\nNo generated main exemplar list found")
                    continue
                else:
                    chars = ofile.write("\nGenerated main exemplar list: " + m[0].text)
                a = r.findall(".//exemplarCharacters[@draft='generated'][@type='auxiliary']")
                if len(a) == 0:
                    chars = ofile.write("\nNo generated auxiliary list found")
                else:
                    chars = ofile.write("\nGenerated auxiliary exemplar list: " + a[0].text)
        except OSError:
            chars = ofile.write("\nDirectory not found or empty")

if __name__ == '__main__':
    if len(sys.argv) > 2:
        reportfordir(sys.argv[1], sys.argv[2])
    else:
        print(reportfordir.__doc__)
<|endoftext|>


<commit_before>
from functools import lru_cache

from pymorphy2 import MorphAnalyzer as PymorphyAnalyzer

from .record import Record

class Gender(Record):
    __attributes__ = ['male', 'female', 'neutral', 'bi', 'general']

    def __init__(self, grams):
        self.male = 'masc' in grams
        self.female = 'femn' in grams
        self.neutral = 'neut' in grams
        # https://github.com/OpenCorpora/opencorpora/issues/795
        self.bi = 'Ms-f' in grams or 'ms-f' in grams
        self.general = 'GNdr' in grams

class Number(Record):
    __attributes__ = ['single', 'plural', 'only_single', 'only_plural']

    def __init__(self, grams):
        self.single = 'sing' in grams
        self.plural = 'plur' in grams
        self.only_single = 'Sgtm' in grams
        self.only_plural = 'Pltm' in grams

class Case(Record):
    __attributes__ = ['mask', 'fixed']

    def __init__(self, grams):
        self.mask = [
            (_ in grams)
            for _ in ['nomn', 'gent', 'datv', 'accs', 'ablt', 'loct', 'voct']
        ]
        self.fixed = 'Fixd' in grams

class Grams(Record):
    __attributes__ = ['values']

    def __init__(self, values):
        self.values = values

    @property
    def gender(self):
        return Gender(self)

    @property
    def number(self):
        return Number(self)

    @property
    def case(self):
        return Case(self)

    def __contains__(self, value):
        return value in self.values

    def __repr__(self):
        values = sorted(self.values)
        return 'Grams({values})'.format(
            values=','.join(values)
        )

    def _repr_pretty_(self, printer, cycle):
        printer.text(repr(self))

class Form(Record):
    __attributes__ = ['normalized', 'grams']

    def __init__(self, normalized, grams, raw=None):
        self.normalized = normalized
        self.grams = grams
        self.raw = raw

    def inflect(self, grams={'nomn', 'sing'}):
        record = self.raw.inflect(grams)
        if not record:
            return self.normalized
        return record.word

def prepare_form(raw):
    normalized = raw.normal_form
    grams = Grams(raw.tag.grammemes)
    return Form(normalized, grams, raw=raw)

class MorphAnalyzer(object):
    def __init__(self):
        self.raw = PymorphyAnalyzer()

    def check_gram(self, gram):
        if not self.raw.TagClass.grammeme_is_known(gram):
            raise ValueError(gram)

    def __call__(self, word):
        records = self.raw.parse(word)
        return [prepare_form(_) for _ in records]

    def normalized(self, word):
        return {_.normalized for _ in self(word)}

DEFAULT_SIZE = 100000

class CachedMorphAnalyzer(MorphAnalyzer):
    def __init__(self, size=DEFAULT_SIZE):
        super(CachedMorphAnalyzer, self).__init__()
        self.__call__ = lru_cache(size)(self.__call__)
<commit_msg>Compact form repr<commit_after>
from functools import lru_cache

from pymorphy2 import MorphAnalyzer as PymorphyAnalyzer

from .record import Record

class Gender(Record):
    __attributes__ = ['male', 'female', 'neutral', 'bi', 'general']

    def __init__(self, grams):
        self.male = 'masc' in grams
        self.female = 'femn' in grams
        self.neutral = 'neut' in grams
        # https://github.com/OpenCorpora/opencorpora/issues/795
        self.bi = 'Ms-f' in grams or 'ms-f' in grams
        self.general = 'GNdr' in grams

class Number(Record):
    __attributes__ = ['single', 'plural', 'only_single', 'only_plural']

    def __init__(self, grams):
        self.single = 'sing' in grams
        self.plural = 'plur' in grams
        self.only_single = 'Sgtm' in grams
        self.only_plural = 'Pltm' in grams

class Case(Record):
    __attributes__ = ['mask', 'fixed']

    def __init__(self, grams):
        self.mask = [
            (_ in grams)
            for _ in ['nomn', 'gent', 'datv', 'accs', 'ablt', 'loct', 'voct']
        ]
        self.fixed = 'Fixd' in grams

class Grams(Record):
    __attributes__ = ['values']

    def __init__(self, values):
        self.values = values

    @property
    def gender(self):
        return Gender(self)

    @property
    def number(self):
        return Number(self)

    @property
    def case(self):
        return Case(self)

    def __contains__(self, value):
        return value in self.values

    def __repr__(self):
        values = sorted(self.values)
        return 'Grams({values})'.format(
            values=','.join(values)
        )

    def _repr_pretty_(self, printer, cycle):
        printer.text(repr(self))

class Form(Record):
    __attributes__ = ['normalized', 'grams']

    def __init__(self, normalized, grams, raw=None):
        self.normalized = normalized
        self.grams = grams
        self.raw = raw

    def inflect(self, grams={'nomn', 'sing'}):
        record = self.raw.inflect(grams)
        if not record:
            return self.normalized
        return record.word

    def __repr__(self):
        return 'Form({self.normalized!r}, {self.grams!r})'.format(self=self)

    def _repr_pretty_(self, printer, cycle):
        printer.text(repr(self))

def prepare_form(raw):
    normalized = raw.normal_form
    grams = Grams(raw.tag.grammemes)
    return Form(normalized, grams, raw=raw)

class MorphAnalyzer(object):
    def __init__(self):
        self.raw = PymorphyAnalyzer()

    def check_gram(self, gram):
        if not self.raw.TagClass.grammeme_is_known(gram):
            raise ValueError(gram)

    def __call__(self, word):
        records = self.raw.parse(word)
        return [prepare_form(_) for _ in records]

    def normalized(self, word):
        return {_.normalized for _ in self(word)}

DEFAULT_SIZE = 100000

class CachedMorphAnalyzer(MorphAnalyzer):
    def __init__(self, size=DEFAULT_SIZE):
        super(CachedMorphAnalyzer, self).__init__()
        self.__call__ = lru_cache(size)(self.__call__)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
# yyets-subdl.py - command-line downloader for YYeTs subtitles

import os
import sys
import shutil
import zipfile

# pip install requests
import requests

def query(keyword):
    request_url = 'http://www.yyets.com/php/search/api'
    request_params = dict(keyword=keyword)
    response = requests.get(request_url, params=request_params)
    if response.status_code == requests.codes.ok:
        items = response.json()['data']
        choices = [x for x in items if x['type'] == 'subtitle']
        return choices
    else:
        raise Exception('HTTP status_code=%s' % response.status_code)
    return []

def ask(choices):
    for idx, choice in enumerate(choices):
        print '%2d. %s' % (idx+1, choice['title'])
        print '    %s' % choice['version']
    answers = raw_input('What items do you want? (seperated by commas) [1] ')
    if answers: return map(lambda x: int(x)-1, answers.split(r','))
    else: return [0]

def download(items):
    files = []
    for item in items:
        # item_url = 'http://www.yyets.com/%s/%s' % (item['type'], item['itemid'])
        download_url = 'http://www.yyets.com/%s/index/download?id=%s' % (item['type'], item['itemid'])
        download_filename = '%s.zip' % (item['version'].split(';')[0] or item['title'])
        command_line = "curl -L '%s' > '%s'" % (download_url, download_filename)
        print command_line
        os.system(command_line.encode('utf-8'))
        files.append(download_filename)
    return files

def extract(files):
    for filepath in files:
        print 'Examining %s' % filepath
        if not 'zip' in os.path.splitext(filepath)[1].lower():
            continue
        extract_success = False
        with zipfile.ZipFile(filepath) as archive:
            for item in archive.namelist():
                filename = os.path.split(item)[1].decode('gb2312').encode('utf-8')
                extension = os.path.splitext(filename)[1].lower()

                found = False
                target_lang = u'繁体&英文'.encode('utf-8')
                if ('ass' in extension) and (target_lang in filename):
                    # zipfile.extract() makes filename escaped with '%xx'
                    with archive.open(item, 'r') as infile, open(filename, 'wb') as outfile:
                        shutil.copyfileobj(infile, outfile)
                        extract_success = True
                    print 'Extracted %s' % filename
        if extract_success:
            os.remove(filepath)

if __name__ == '__main__':
    if len(sys.argv) < 2:
        print 'Usage: %s <keyword>' % sys.argv[0]
        print "Example: %s 'Person of Interest S04E01'" % sys.argv[0]
        sys.exit(1)
    keyword = sys.argv[1]
    choices = query(keyword)
    chosen_ids = ask(choices)
    chosens = [choices[idx] for idx in chosen_ids]
    files = download(chosens)
    extract(files)
<commit_msg>fix: handle string encode/decode exception in extract()<commit_after>#!/usr/bin/env python
# encoding: utf-8
# yyets-subdl.py - command-line downloader for YYeTs subtitles

import os
import sys
import shutil
import zipfile

# pip install requests
import requests

def query(keyword):
    request_url = 'http://www.yyets.com/php/search/api'
    request_params = dict(keyword=keyword)
    response = requests.get(request_url, params=request_params)
    if response.status_code == requests.codes.ok:
        items = response.json()['data']
        choices = [x for x in items if x['type'] == 'subtitle']
        return choices
    else:
        raise Exception('HTTP status_code=%s' % response.status_code)
    return []

def ask(choices):
    for idx, choice in enumerate(choices):
        print '%2d. %s' % (idx+1, choice['title'])
        print '    %s' % choice['version']
    answers = raw_input('What items do you want? (seperated by commas) [1] ')
    if answers: return map(lambda x: int(x)-1, answers.split(r','))
    else: return [0]

def download(items):
    files = []
    for item in items:
        # item_url = 'http://www.yyets.com/%s/%s' % (item['type'], item['itemid'])
        download_url = 'http://www.yyets.com/%s/index/download?id=%s' % (item['type'], item['itemid'])
        download_filename = '%s.zip' % (item['version'].split(';')[0] or item['title'])
        command_line = "curl -L '%s' > '%s'" % (download_url, download_filename)
        print command_line
        os.system(command_line.encode('utf-8'))
        files.append(download_filename)
    return files

def extract(files):
    for filepath in files:
        print 'Examining %s' % filepath
        if not 'zip' in os.path.splitext(filepath)[1].lower():
            continue
        extract_success = False
        with zipfile.ZipFile(filepath) as archive:
            for item in archive.namelist():
                filename = os.path.split(item)[1]
                extension = os.path.splitext(filename)[1].lower()

                try: filename = filename.decode('gb2312')
                except: pass

                try: filename = filename.encode('utf-8')
                except: pass

                found = False
                target_lang = u'繁体&英文'.encode('utf-8')
                if ('ass' in extension) and (target_lang in filename):
                    # zipfile.extract() makes filename escaped with '%xx'
                    with archive.open(item, 'r') as infile, open(filename, 'wb') as outfile:
                        shutil.copyfileobj(infile, outfile)
                        extract_success = True
                    print 'Extracted %s' % filename
        if extract_success:
            os.remove(filepath)

if __name__ == '__main__':
    if len(sys.argv) < 2:
        print 'Usage: %s <keyword>' % sys.argv[0]
        print "Example: %s 'Person of Interest S04E01'" % sys.argv[0]
        sys.exit(1)
    keyword = sys.argv[1]
    choices = query(keyword)
    chosen_ids = ask(choices)
    chosens = [choices[idx] for idx in chosen_ids]
    files = download(chosens)
    extract(files)
<|endoftext|>


<commit_before>import logging
import raven

from datetime import timedelta
from flask import Flask
from flask_alembic import Alembic
from flask_sqlalchemy import SQLAlchemy
from raven.contrib.flask import Sentry

from zeus.utils.celery import Celery
from zeus.utils.redis import Redis

import os

ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))

alembic = Alembic()
celery = Celery()
db = SQLAlchemy()
redis = Redis()
sentry = Sentry(logging=True, level=logging.WARN)

def force_ssl(app):
    def middleware(environ, start_response):
        environ['wsgi.url_scheme'] = 'https'
        return app(environ, start_response)

    return middleware

def create_app(_read_config=True, **config):
    app = Flask(
        __name__,
        static_folder=os.path.join(ROOT, 'static'),
        template_folder=os.path.join(ROOT, 'templates'),
    )

    # support for kubernetes
    # https://kubernetes.io/docs/concepts/services-networking/service/
    if os.environ.get('GET_HOSTS_FROM') == 'env':
        REDIS_URL = 'redis://{}:{}/0'.format(
            os.environ['REDIS_MASTER_SERVICE_HOST'],
            os.environ['REDIS_MASTER_SERVICE_PORT'],
        )
        # Cloud SQL
        # https://cloud.google.com/sql/docs/postgres/connect-container-engine
        SQLALCHEMY_URI = 'postgresql://{}:{}@127.0.0.1:5432/zeus'.format(
            os.environ['DB_USER'],
            os.environ['DB_PASSWORD'],
        )
    else:
        REDIS_URL = os.environ.get('REDIS_URL', 'redis://localhost/0')
        SQLALCHEMY_URI = os.environ.get('SQLALCHEMY_DATABASE_URI', 'postgresql:///zeus')

    if os.environ.get('SERVER_NAME'):
        app.config['SERVER_NAME'] = os.environ['SERVER_NAME']

    app.config['SECRET_KEY'] = os.environ.get('SECRET_KEY')

    app.config['LOG_LEVEL'] = os.environ.get('LOG_LEVEL') or 'INFO'

    app.config['SSL'] = os.environ.get('SSL') in ('1', 'true', 'on')

    # limit sessions to one day so permissions are revalidated automatically
    app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(days=1)

    app.config['SQLALCHEMY_COMMIT_ON_TEARDOWN'] = True
    app.config['SQLALCHEMY_DATABASE_URI'] = SQLALCHEMY_URI
    app.config['SQLALCHEMY_POOL_SIZE'] = 60
    app.config['SQLALCHEMY_MAX_OVERFLOW'] = 20
    app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False

    app.config['REDIS_URL'] = REDIS_URL

    app.config['SENTRY_DSN'] = os.environ.get('SENTRY_DSN') or None
    app.config['SENTRY_DSN_FRONTEND'] = os.environ.get('SENTRY_DSN_FRONTEND') or None
    app.config['SENTRY_INCLUDE_PATHS'] = [
        'zeus',
    ]
    try:
        app.config['SENTRY_RELEASE'] = raven.fetch_git_sha(ROOT)
    except Exception:
        app.logger.warn('unable to bind sentry.release context', exc_info=True)
    app.config['SENTRY_ENVIRONMENT'] = os.environ.get('NODE_ENV', 'development')

    app.config['GITHUB_CLIENT_ID'] = os.environ.get('GITHUB_CLIENT_ID') or None
    app.config['GITHUB_CLIENT_SECRET'] = os.environ.get('GITHUB_CLIENT_SECRET') or None

    app.config['CELERY_ACCEPT_CONTENT'] = ['zeus_json', 'json']
    app.config['CELERY_ACKS_LATE'] = True
    app.config['CELERY_BROKER_URL'] = app.config['REDIS_URL']
    app.config['CELERY_DEFAULT_QUEUE'] = 'default'
    app.config['CELERY_DEFAULT_EXCHANGE'] = 'default'
    app.config['CELERY_DEFAULT_EXCHANGE_TYPE'] = 'direct'
    app.config['CELERY_DEFAULT_ROUTING_KEY'] = 'default'
    app.config['CELERY_DISABLE_RATE_LIMITS'] = True
    app.config['CELERY_EVENT_SERIALIZER'] = 'zeus_json'
    app.config['CELERY_IGNORE_RESULT'] = True
    app.config['CELERY_IMPORTS'] = ('zeus.tasks', )
    app.config['CELERY_RESULT_BACKEND'] = None
    app.config['CELERY_RESULT_SERIALIZER'] = 'zeus_json'
    app.config['CELERY_SEND_EVENTS'] = False
    app.config['CELERY_TASK_RESULT_EXPIRES'] = 1
    app.config['CELERY_TASK_SERIALIZER'] = 'zeus_json'
    app.config['CELERYD_PREFETCH_MULTIPLIER'] = 1
    app.config['CELERYD_MAX_TASKS_PER_CHILD'] = 10000

    app.config['REPO_ROOT'] = os.environ.get('REPO_ROOT', '/usr/local/cache/zeus-repos')

    # app.config['DEFAULT_FILE_STORAGE'] = ''

    if _read_config:
        if os.environ.get('ZEUS_CONF'):
            # ZEUS_CONF=/etc/zeus.conf.py
            app.config.from_envvar('ZEUS_CONF')
        else:
            # Look for ~/.zeus/zeus.conf.py
            path = os.path.normpath(os.path.expanduser('~/.zeus/zeus.config.py'))
            app.config.from_pyfile(path, silent=True)

    app.config.update(config)

    req_vars = (
        'GITHUB_CLIENT_ID', 'GITHUB_CLIENT_SECRET', 'REDIS_URL', 'SECRET_KEY',
        'SQLALCHEMY_DATABASE_URI'
    )
    for varname in req_vars:
        if not app.config.get(varname):
            raise SystemExit('Required configuration not present for {}'.format(varname))

    if app.config.get('SSL'):
        app.wgsi_app = force_ssl(app)
        app.config['PREFERRED_URL_SCHEME'] = 'https'
        app.config['SESSION_COOKIE_SECURE'] = True

    from zeus.testutils.client import ZeusTestClient
    app.test_client_class = ZeusTestClient

    if app.config.get('LOG_LEVEL'):
        app.logger.setLevel(getattr(logging, app.config['LOG_LEVEL'].upper()))

    # init sentry first
    sentry.init_app(app)

    configure_db(app)

    redis.init_app(app)

    celery.init_app(app, sentry)

    configure_api(app)
    configure_web(app)

    from . import models  # NOQA

    return app

def configure_db(app):
    from sqlalchemy import event
    from sqlalchemy.orm import mapper
    from sqlalchemy.inspection import inspect

    alembic.init_app(app)
    db.init_app(app)

    @event.listens_for(mapper, "init")
    def instant_defaults_listener(target, args, kwargs):
        for key, column in inspect(type(target)).columns.items():
            if column.default is not None:
                if callable(column.default.arg):
                    setattr(target, key, column.default.arg(target))
                else:
                    setattr(target, key, column.default.arg)

    event.listen(mapper, 'init', instant_defaults_listener)

def configure_api(app):
    from zeus import api

    app.register_blueprint(api.app, url_prefix='/api')

def configure_web(app):
    from zeus import web

    app.register_blueprint(web.app, url_prefix='')
<commit_msg>enforce psycopg2 in dsn<commit_after>import logging
import raven

from datetime import timedelta
from flask import Flask
from flask_alembic import Alembic
from flask_sqlalchemy import SQLAlchemy
from raven.contrib.flask import Sentry

from zeus.utils.celery import Celery
from zeus.utils.redis import Redis

import os

ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))

alembic = Alembic()
celery = Celery()
db = SQLAlchemy()
redis = Redis()
sentry = Sentry(logging=True, level=logging.WARN)

def force_ssl(app):
    def middleware(environ, start_response):
        environ['wsgi.url_scheme'] = 'https'
        return app(environ, start_response)

    return middleware

def create_app(_read_config=True, **config):
    app = Flask(
        __name__,
        static_folder=os.path.join(ROOT, 'static'),
        template_folder=os.path.join(ROOT, 'templates'),
    )

    # support for kubernetes
    # https://kubernetes.io/docs/concepts/services-networking/service/
    if os.environ.get('GET_HOSTS_FROM') == 'env':
        REDIS_URL = 'redis://{}:{}/0'.format(
            os.environ['REDIS_MASTER_SERVICE_HOST'],
            os.environ['REDIS_MASTER_SERVICE_PORT'],
        )
        # Cloud SQL
        # https://cloud.google.com/sql/docs/postgres/connect-container-engine
        SQLALCHEMY_URI = 'postgresql+psycopg2://{}:{}@127.0.0.1:5432/zeus'.format(
            os.environ['DB_USER'],
            os.environ['DB_PASSWORD'],
        )
    else:
        REDIS_URL = os.environ.get('REDIS_URL', 'redis://localhost/0')
        SQLALCHEMY_URI = os.environ.get('SQLALCHEMY_DATABASE_URI', 'postgresql+psycopg2:///zeus')

    if os.environ.get('SERVER_NAME'):
        app.config['SERVER_NAME'] = os.environ['SERVER_NAME']

    app.config['SECRET_KEY'] = os.environ.get('SECRET_KEY')

    app.config['LOG_LEVEL'] = os.environ.get('LOG_LEVEL') or 'INFO'

    app.config['SSL'] = os.environ.get('SSL') in ('1', 'true', 'on')

    # limit sessions to one day so permissions are revalidated automatically
    app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(days=1)

    app.config['SQLALCHEMY_COMMIT_ON_TEARDOWN'] = True
    app.config['SQLALCHEMY_DATABASE_URI'] = SQLALCHEMY_URI
    app.config['SQLALCHEMY_POOL_SIZE'] = 60
    app.config['SQLALCHEMY_MAX_OVERFLOW'] = 20
    app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False

    app.config['REDIS_URL'] = REDIS_URL

    app.config['SENTRY_DSN'] = os.environ.get('SENTRY_DSN') or None
    app.config['SENTRY_DSN_FRONTEND'] = os.environ.get('SENTRY_DSN_FRONTEND') or None
    app.config['SENTRY_INCLUDE_PATHS'] = [
        'zeus',
    ]
    try:
        app.config['SENTRY_RELEASE'] = raven.fetch_git_sha(ROOT)
    except Exception:
        app.logger.warn('unable to bind sentry.release context', exc_info=True)
    app.config['SENTRY_ENVIRONMENT'] = os.environ.get('NODE_ENV', 'development')

    app.config['GITHUB_CLIENT_ID'] = os.environ.get('GITHUB_CLIENT_ID') or None
    app.config['GITHUB_CLIENT_SECRET'] = os.environ.get('GITHUB_CLIENT_SECRET') or None

    app.config['CELERY_ACCEPT_CONTENT'] = ['zeus_json', 'json']
    app.config['CELERY_ACKS_LATE'] = True
    app.config['CELERY_BROKER_URL'] = app.config['REDIS_URL']
    app.config['CELERY_DEFAULT_QUEUE'] = 'default'
    app.config['CELERY_DEFAULT_EXCHANGE'] = 'default'
    app.config['CELERY_DEFAULT_EXCHANGE_TYPE'] = 'direct'
    app.config['CELERY_DEFAULT_ROUTING_KEY'] = 'default'
    app.config['CELERY_DISABLE_RATE_LIMITS'] = True
    app.config['CELERY_EVENT_SERIALIZER'] = 'zeus_json'
    app.config['CELERY_IGNORE_RESULT'] = True
    app.config['CELERY_IMPORTS'] = ('zeus.tasks', )
    app.config['CELERY_RESULT_BACKEND'] = None
    app.config['CELERY_RESULT_SERIALIZER'] = 'zeus_json'
    app.config['CELERY_SEND_EVENTS'] = False
    app.config['CELERY_TASK_RESULT_EXPIRES'] = 1
    app.config['CELERY_TASK_SERIALIZER'] = 'zeus_json'
    app.config['CELERYD_PREFETCH_MULTIPLIER'] = 1
    app.config['CELERYD_MAX_TASKS_PER_CHILD'] = 10000

    app.config['REPO_ROOT'] = os.environ.get('REPO_ROOT', '/usr/local/cache/zeus-repos')

    # app.config['DEFAULT_FILE_STORAGE'] = ''

    if _read_config:
        if os.environ.get('ZEUS_CONF'):
            # ZEUS_CONF=/etc/zeus.conf.py
            app.config.from_envvar('ZEUS_CONF')
        else:
            # Look for ~/.zeus/zeus.conf.py
            path = os.path.normpath(os.path.expanduser('~/.zeus/zeus.config.py'))
            app.config.from_pyfile(path, silent=True)

    app.config.update(config)

    req_vars = (
        'GITHUB_CLIENT_ID', 'GITHUB_CLIENT_SECRET', 'REDIS_URL', 'SECRET_KEY',
        'SQLALCHEMY_DATABASE_URI'
    )
    for varname in req_vars:
        if not app.config.get(varname):
            raise SystemExit('Required configuration not present for {}'.format(varname))

    if app.config.get('SSL'):
        app.wgsi_app = force_ssl(app)
        app.config['PREFERRED_URL_SCHEME'] = 'https'
        app.config['SESSION_COOKIE_SECURE'] = True

    from zeus.testutils.client import ZeusTestClient
    app.test_client_class = ZeusTestClient

    if app.config.get('LOG_LEVEL'):
        app.logger.setLevel(getattr(logging, app.config['LOG_LEVEL'].upper()))

    # init sentry first
    sentry.init_app(app)

    configure_db(app)

    redis.init_app(app)

    celery.init_app(app, sentry)

    configure_api(app)
    configure_web(app)

    from . import models  # NOQA

    return app

def configure_db(app):
    from sqlalchemy import event
    from sqlalchemy.orm import mapper
    from sqlalchemy.inspection import inspect

    alembic.init_app(app)
    db.init_app(app)

    @event.listens_for(mapper, "init")
    def instant_defaults_listener(target, args, kwargs):
        for key, column in inspect(type(target)).columns.items():
            if column.default is not None:
                if callable(column.default.arg):
                    setattr(target, key, column.default.arg(target))
                else:
                    setattr(target, key, column.default.arg)

    event.listen(mapper, 'init', instant_defaults_listener)

def configure_api(app):
    from zeus import api

    app.register_blueprint(api.app, url_prefix='/api')

def configure_web(app):
    from zeus import web

    app.register_blueprint(web.app, url_prefix='')
<|endoftext|>


<commit_before>from __future__ import unicode_literals, absolute_import
import re
import inspect
from collections import defaultdict

# Alias str to unicode with unicode_literals imported
try:
    str = unicode
except NameError:
    pass

class Uid(object):
    "Unique id generator."
    def __init__(self):
        self.i = 0

    def __call__(self):
        i = self.i
        self.i += 1
        return i

uid = Uid()

class Props(object):
    match_props = None
    update_props = None

    def __init__(self, props=None):
        if props is None:
            self.props = {}
        elif isinstance(props, dict):
            self.props = props
        else:
            self.props = dict(props)

        self.id = uid()

    def __str__(self):
        return str(self.id)

    def __repr__(self):
        return '{}({})'.format(self.__class__.__name__, self.id)

    def __getitem__(self, key):
        return self.props[key]

    def __setitem__(self, key, value):
        self.props[key] = value

    def __delitem__(self, key):
        del self.props[key]

    def __contains__(self, key):
        return key in self.props

    def __iter__(self):
        return iter(self.props)

    def update(self, props):
        self.props.update(props)

    def serialize(self, *args, **kwargs):
        "Returns a shallow copy of the properties."
        return self.props.copy()

class Rel(Props):
    __slots__ = ('id', 'start', 'end', 'type', 'props')

    def __init__(self, start, end, type, props=None):
        super(Rel, self).__init__(props)
        self.start = start
        self.end = end
        self.type = type

    def __repr__(self):
        return '{}({}-{}-{})'.format(self.__class__.__name__, self.start,
                                     self.type, self.end)

    def related(self):
        "Returns true if this relationship is currently bound."
        return self.start.related(self.end, self.type)

    def unrelate(self):
        "Disassociates this relationship from the start node."
        return self.start.unrelate(self.end, type=self.type)

    def relate(self):
        "Reassociates this relationship to the start node."
        self.start._add_rel(self)

class Node(Props):
    labels = None
    reltype = Rel

    def __init__(self, props=None, labels=None):
        super(Node, self).__init__(props)
        # Nested hash of relationships by node id then type. Currently a
        # only a single relationship of the same type can be defined between
        # the same two nodes.
        self._rels = defaultdict(dict)

        # Hash of relationship types with a set of node ids.
        self._types = defaultdict(set)

    def __repr__(self):
        return "{}('{}')".format(self.__class__.__name__, self)

    def _get_rels_for_type(self, type):
        rels = []
        for id in self._types[type]:
            rels.append(self._rels[id][type])
        return rels

    def _get_rels_for_node(self, id):
        rels = []
        for type in self._rels[id]:
            rels.append(self._rels[id][type])
        return rels

    def _get_rels_for_node_and_type(self, id, type):
        rel = self._rels[id].get(type)
        if rel:
            return [rel]
        return []

    def _get_rels(self):
        rels = []
        for id in self._rels:
            rels.extend(self._rels[id].values())
        return rels

    def _add_rel(self, rel):
        node = rel.end
        self._rels[node.id][rel.type] = rel
        self._types[rel.type].add(node.id)

    def _prune(self):
        "Removes empty entries in _rels and _types."
        for id in list(self._rels):
            if not self._rels[id]:
                del self._rels[id]

        for type in list(self._types):
            if not self._types[type]:
                del self._types[type]

    def _del_rel(self, id, type):
        del self._rels[id][type]
        self._types[type].remove(id)

    def _del_rels_for_type(self, type):
        "Deletes all relationships of type."
        count = 0
        for id in tuple(self._types[type]):
            self._del_rel(id, type)
            count += 1
        return count

    def _del_rels_for_node(self, id):
        "Deletes all relationships for node."
        count = 0
        for type in list(self._rels[id]):
            self._del_rel(id, type)
            count += 1
        return count

    def _del_rels_for_node_and_type(self, id, type):
        "Deletes relationship for node and type."
        if id not in self._rels or type not in self._rels[id]:
            return 0
        self._del_rel(id, type)
        return 1

    def _del_rels(self):
        "Deletes all relationships."
        count = 0
        for id in list(self._rels):
            for type in list(self._rels[id]):
                self._del_rel(id, type)
                count += 1
        return count

    def relate(self, node, type, props=None, reltype=None):
        "Adds a relationship to node if it does not already exist."
        if reltype is None:
            reltype = self.reltype

        if isinstance(node, (list, tuple)):
            rels = []
            for _node in node:
                rels.append(self.relate(_node, type, props))
            return Rels(rels)

        assert isinstance(node, Node), 'end node must be a Node instance'
        assert isinstance(type, (str, bytes)), 'type must be a string'

        if type in self._types and node.id in self._types[type]:
            rel = self._rels[node.id][type]
            if props:
                rel.update(props)
        else:
            rel = reltype(self, node, type, props)
            self._add_rel(rel)

        return rel

    def unrelate(self, node=None, type=None):
        "Deletes a relationship."
        if node and type:
            count = self._del_rels_for_node_and_type(node.id, type)
        elif node:
            count = self._del_rels_for_node(node.id)
        elif type:
            count = self._del_rels_for_type(type)
        else:
            count = self._del_rels()

        self._prune()
        return count

    def related(self, node, type=None):
        "Returns true if the node is related, optionally by a type."
        related = node.id in self._rels or self.id in node._rels

        if related and type:
            return type in self._rels[node.id] or type in node._rels[self.id]

        return related

    def rels(self, node=None, type=None):
        "Returns relations for the node, optionally filtered by type."
        if node and type:
            rels = self._get_rels_for_node_and_type(node.id, type)
        elif node:
            rels = self._get_rels_for_node(node.id)
        elif type:
            rels = self._get_rels_for_type(type)
        else:
            rels = self._get_rels()

        return Rels(rels)

class DictSeq(tuple):
    """Immutable sequence of items which supports dict-like access. Items are
    ordered in the order they were provided. They can be accessed by index
    or by key, where the key is the `str` representation of the item.

    Key-based accessed is case-insensitive.
    """
    def __init__(self, *args):
        self._map = {str(n).lower(): i for i, n in enumerate(self)}

    def __eq__(self, other):
        "Equality based on the items contained."
        if not hasattr(other, '__iter__'):
            return False

        if len(self) != len(other):
            return False

        for i, x in enumerate(self):
            if other[i] != x:
                return False

        return True

    def __ne__(self, other):
        "Equality based on the items contained."
        return not self == other

    def __getitem__(self, key):
        """Supports standard index/slice access, key-based access by name
        and multi-item access via passing an non-string iterable.
        """
        if isinstance(key, (int, slice)):
            return tuple.__getitem__(self, key)
        elif isinstance(key, (str, bytes)):
            key = key.lower()
        elif hasattr(key, '__iter__'):
            return Nodes(self[_key] for _key in key)

        key = self._map[key]

        return tuple.__getitem__(self, key)

    def filter(self, key, value=None):
        """Filters the items in this container and returns a new container.
        The most common filtering is by property, so a key and value can be
        supplied as a shorthand, otherwise a filter function must be passed.
        """
        if isinstance(key, (str, bytes)):
            def func(item):
                if key in item:
                    if value is not None:
                        return item[key] == value
                    return True
                return False
        else:
            func = key
            if not inspect.isfunction(func):
                raise TypeError('filter requires key/value or function')

        return self.__class__(filter(func, self))

    def match(self, regexp, flags=re.I):
        "Returns one or more items that match the regexp on the key."
        r = re.compile(regexp, flags)
        return self.__class__(self[key] for key in self._map if r.match(key))

class Rels(DictSeq):
    "Tuple of relationships."
    def nodes(self):
        return Nodes({rel.end for rel in self})

class Nodes(DictSeq):
    "Tuple of nodes."
<commit_msg>Add support for taking match_props and update_props in constructor<commit_after>from __future__ import unicode_literals, absolute_import
import re
import inspect
from collections import defaultdict

# Alias str to unicode with unicode_literals imported
try:
    str = unicode
except NameError:
    pass

class Uid(object):
    "Unique id generator."
    def __init__(self):
        self.i = 0

    def __call__(self):
        i = self.i
        self.i += 1
        return i

uid = Uid()

class Props(object):
    match_props = None
    update_props = None

    def __init__(self, props=None, match_props=None, update_props=None):
        if props is None:
            self.props = {}
        elif isinstance(props, dict):
            self.props = props
        else:
            self.props = dict(props)

        # Override class-defined properties
        if match_props:
            self.match_props = match_props
        if update_props:
            self.update_props = update_props

        self.id = uid()

    def __str__(self):
        return str(self.id)

    def __repr__(self):
        return '{}({})'.format(self.__class__.__name__, self.id)

    def __getitem__(self, key):
        return self.props[key]

    def __setitem__(self, key, value):
        self.props[key] = value

    def __delitem__(self, key):
        del self.props[key]

    def __contains__(self, key):
        return key in self.props

    def __iter__(self):
        return iter(self.props)

    def update(self, props):
        self.props.update(props)

    def serialize(self, *args, **kwargs):
        "Returns a shallow copy of the properties."
        return self.props.copy()

class Rel(Props):
    def __init__(self, start, end, type, *args, **kwargs):
        self.start = start
        self.end = end
        self.type = type
        super(Rel, self).__init__(*args, **kwargs)

    def __repr__(self):
        return '{}({}-{}-{})'.format(self.__class__.__name__, self.start,
                                     self.type, self.end)

    def related(self):
        "Returns true if this relationship is currently bound."
        return self.start.related(self.end, self.type)

    def unrelate(self):
        "Disassociates this relationship from the start node."
        return self.start.unrelate(self.end, type=self.type)

    def relate(self):
        "Reassociates this relationship to the start node."
        self.start._add_rel(self)

class Node(Props):
    labels = None
    reltype = Rel

    def __init__(self, *args, **kwargs):
        labels = kwargs.pop('labels', None)

        # Override class-defined labels
        if labels:
            self.labels = labels

        # Nested hash of relationships by node id then type. Currently a
        # only a single relationship of the same type can be defined between
        # the same two nodes.
        self._rels = defaultdict(dict)

        # Hash of relationship types with a set of node ids.
        self._types = defaultdict(set)

        super(Node, self).__init__(*args, **kwargs)

    def __repr__(self):
        return "{}('{}')".format(self.__class__.__name__, self)

    def _get_rels_for_type(self, type):
        rels = []
        for id in self._types[type]:
            rels.append(self._rels[id][type])
        return rels

    def _get_rels_for_node(self, id):
        rels = []
        for type in self._rels[id]:
            rels.append(self._rels[id][type])
        return rels

    def _get_rels_for_node_and_type(self, id, type):
        rel = self._rels[id].get(type)
        if rel:
            return [rel]
        return []

    def _get_rels(self):
        rels = []
        for id in self._rels:
            rels.extend(self._rels[id].values())
        return rels

    def _add_rel(self, rel):
        node = rel.end
        self._rels[node.id][rel.type] = rel
        self._types[rel.type].add(node.id)

    def _prune(self):
        "Removes empty entries in _rels and _types."
        for id in list(self._rels):
            if not self._rels[id]:
                del self._rels[id]

        for type in list(self._types):
            if not self._types[type]:
                del self._types[type]

    def _del_rel(self, id, type):
        del self._rels[id][type]
        self._types[type].remove(id)

    def _del_rels_for_type(self, type):
        "Deletes all relationships of type."
        count = 0
        for id in tuple(self._types[type]):
            self._del_rel(id, type)
            count += 1
        return count

    def _del_rels_for_node(self, id):
        "Deletes all relationships for node."
        count = 0
        for type in list(self._rels[id]):
            self._del_rel(id, type)
            count += 1
        return count

    def _del_rels_for_node_and_type(self, id, type):
        "Deletes relationship for node and type."
        if id not in self._rels or type not in self._rels[id]:
            return 0
        self._del_rel(id, type)
        return 1

    def _del_rels(self):
        "Deletes all relationships."
        count = 0
        for id in list(self._rels):
            for type in list(self._rels[id]):
                self._del_rel(id, type)
                count += 1
        return count

    def relate(self, node, type, props=None, **kwargs):
        "Adds a relationship to node if it does not already exist."
        if isinstance(node, (list, tuple)):
            rels = []
            for _node in node:
                rels.append(self.relate(_node, type, props, **kwargs))
            return Rels(rels)

        reltype = kwargs.pop('reltype', self.reltype)

        assert isinstance(node, Node), 'end node must be a Node instance'
        assert isinstance(type, (str, bytes)), 'type must be a string'

        if type in self._types and node.id in self._types[type]:
            rel = self._rels[node.id][type]
            if props:
                rel.update(props)
        else:
            rel = reltype(self, node, type, props=props, **kwargs)
            self._add_rel(rel)

        return rel

    def unrelate(self, node=None, type=None):
        "Deletes a relationship."
        if node and type:
            count = self._del_rels_for_node_and_type(node.id, type)
        elif node:
            count = self._del_rels_for_node(node.id)
        elif type:
            count = self._del_rels_for_type(type)
        else:
            count = self._del_rels()

        self._prune()
        return count

    def related(self, node, type=None):
        "Returns true if the node is related, optionally by a type."
        related = node.id in self._rels or self.id in node._rels

        if related and type:
            return type in self._rels[node.id] or type in node._rels[self.id]

        return related

    def rels(self, node=None, type=None):
        "Returns relations for the node, optionally filtered by type."
        if node and type:
            rels = self._get_rels_for_node_and_type(node.id, type)
        elif node:
            rels = self._get_rels_for_node(node.id)
        elif type:
            rels = self._get_rels_for_type(type)
        else:
            rels = self._get_rels()

        return Rels(rels)

class DictSeq(tuple):
    """Immutable sequence of items which supports dict-like access. Items are
    ordered in the order they were provided. They can be accessed by index
    or by key, where the key is the `str` representation of the item.

    Key-based accessed is case-insensitive.
    """
    def __init__(self, *args):
        self._map = {str(n).lower(): i for i, n in enumerate(self)}

    def __eq__(self, other):
        "Equality based on the items contained."
        if not hasattr(other, '__iter__'):
            return False

        if len(self) != len(other):
            return False

        for i, x in enumerate(self):
            if other[i] != x:
                return False

        return True

    def __ne__(self, other):
        "Equality based on the items contained."
        return not self == other

    def __getitem__(self, key):
        """Supports standard index/slice access, key-based access by name
        and multi-item access via passing an non-string iterable.
        """
        if isinstance(key, (int, slice)):
            return tuple.__getitem__(self, key)
        elif isinstance(key, (str, bytes)):
            key = key.lower()
        elif hasattr(key, '__iter__'):
            return Nodes(self[_key] for _key in key)

        key = self._map[key]

        return tuple.__getitem__(self, key)

    def filter(self, key, value=None):
        """Filters the items in this container and returns a new container.
        The most common filtering is by property, so a key and value can be
        supplied as a shorthand, otherwise a filter function must be passed.
        """
        if isinstance(key, (str, bytes)):
            def func(item):
                if key in item:
                    if value is not None:
                        return item[key] == value
                    return True
                return False
        else:
            func = key
            if not inspect.isfunction(func):
                raise TypeError('filter requires key/value or function')

        return self.__class__(filter(func, self))

    def match(self, regexp, flags=re.I):
        "Returns one or more items that match the regexp on the key."
        r = re.compile(regexp, flags)
        return self.__class__(self[key] for key in self._map if r.match(key))

class Rels(DictSeq):
    "Tuple of relationships."
    def nodes(self):
        return Nodes({rel.end for rel in self})

class Nodes(DictSeq):
    "Tuple of nodes."
<|endoftext|>


<commit_before>POSTCODES = [70173, 70174, 70176, 70178, 70180, 70182, 70184, 70186, 70188,
             70190, 70191, 70192, 70193, 70195, 70197, 70199, 70327, 70329,
             70372, 70374, 70378, 70435, 70437, 70439, 70469, 70476, 70499,
             70563, 70565, 70567, 70569, 70597, 70599, 70619, 70629]

SCRAPE_SLEEP = 1 # pause between each GET request, in seconds

OUTPUT_DIR = '/var/www/gsack/calendar'

<commit_msg>New default OUTPUT_DIR<commit_after>POSTCODES = [70173, 70174, 70176, 70178, 70180, 70182, 70184, 70186, 70188,
             70190, 70191, 70192, 70193, 70195, 70197, 70199, 70327, 70329,
             70372, 70374, 70378, 70435, 70437, 70439, 70469, 70476, 70499,
             70563, 70565, 70567, 70569, 70597, 70599, 70619, 70629]

SCRAPE_SLEEP = 1 # pause between each GET request, in seconds

OUTPUT_DIR = '/home/kopf/gsack-output'

<|endoftext|>


<commit_before><commit_msg>Change key for exchange rates<commit_after><|endoftext|>


<commit_before><commit_msg>Small typo fix<commit_after><|endoftext|>


<commit_before>import re
import json
import shapely
import binascii as ba

from geopandas import GeoSeries, GeoDataFrame, points_from_xy

ENC_SHAPELY = 'shapely'
ENC_WKB = 'wkb'
ENC_WKB_HEX = 'wkb-hex'
ENC_WKB_BHEX = 'wkb-bhex'
ENC_WKT = 'wkt'
ENC_EWKT = 'ewkt'
SPHERICAL_TOLERANCE = 0.0001
SIMPLIFY_TOLERANCE = 0.001

def set_geometry(gdf, col, drop=False, inplace=False, crs=None):
    """Set the GeoDataFrame geometry using either an existing column or the specified input.
    By default yields a new object. The original geometry column is replaced with the input.
    It detects the geometry encoding and it decodes the column if required. Supported geometry
    encodings are:

        - `WKB` (Bytes, Hexadecimal String, Hexadecimal Bytestring)
        - `Extended WKB` (Bytes, Hexadecimal String, Hexadecimal Bytestring)
        - `WKT` (String)
        - `Extended WKT` (String)

    Args:
        col (column label or array): Name of the column or column containing the geometry.
        drop (boolean, default False): Delete the column to be used as the new geometry.
        inplace (boolean, default False): Modify the GeoDataFrame in place (do not create a new object).
        crs (str/result of fion.get_crs, optional): Coordinate system to use. If passed, overrides both
            DataFrame and col's crs. Otherwise, tries to get crs from passed col values or DataFrame.

    Example:
        >>> set_geometry(gdf, 'the_geom', drop=True, inplace=True)

    """
    if not isinstance(gdf, GeoDataFrame):
        raise ValueError('gdf must be an instance of geopandas.GeoDataFrame.')

    if inplace:
        frame = gdf
    else:
        frame = gdf.copy()

    # Decode geometry
    if isinstance(col, str):
        if col not in frame:
            raise Exception('Column "{0}" does not exist.'.format(col))
        frame[col] = decode_geometry(frame[col])
    else:
        col = decode_geometry(col)

    # Call set_geometry with decoded column
    frame.set_geometry(col, drop=drop, inplace=True, crs=crs)

    if not inplace:
        return frame

def set_geometry_from_xy(gdf, x, y, drop=False, inplace=False, crs=None):
    """Set the GeoDataFrame geometry using either existing lng/lat columns or the specified inputs.
    By default yields a new object. The original geometry column is replaced with the new one.

    Args:
        x (column label or array): Name of the x (longitude) column or column containing the x coordinates.
        y (column label or array): Name of the y (latitude) column or column containing the y coordinates.
        drop (boolean, default False): Delete the columns to be used to generate the new geometry.
        inplace (boolean, default False): Modify the GeoDataFrame in place (do not create a new object).
        crs (str/result of fion.get_crs, optional): Coordinate system to use. If passed, overrides both
            DataFrame and col's crs. Otherwise, tries to get crs from passed col values or DataFrame.

    Example:
        >>> set_geometry_from_xy(gdf, 'lng', 'lat', drop=True, inplace=True)

    """
    if not isinstance(gdf, GeoDataFrame):
        raise ValueError('gdf must be an instance of geopandas.GeoDataFrame.')

    if isinstance(x, str) and x in gdf and isinstance(y, str) and y in gdf:
        x_col = gdf[x]
        y_col = gdf[y]
    else:
        x_col = x
        y_col = y

    # Generate geometry
    geom_col = points_from_xy(x_col, y_col)

    # Call set_geometry with generated column
    frame = gdf.set_geometry(geom_col, inplace=inplace, crs=crs)

    if drop:
        if frame is None:
            frame = gdf
        del frame[x]
        del frame[y]

    return frame

def has_geometry(gdf):
    """Method to check if the GeoDataFrame contains a valid geometry column.
    If there is no valid geometry, you can use the following methods:

        - `set_geometry`: to create a decoded geometry column from any raw geometry column.
        - `set_geometry_from_xy`: to create a geometry column from `longitude` and `latitude` columns.
    """
    return hasattr(gdf, '_geometry_column_name') and gdf._geometry_column_name in gdf

def decode_geometry(geom_col):
    """Decodes a DataFrame column. It detects the geometry encoding and it decodes the column if required.
    Supported geometry encodings are:

        - `WKB` (Bytes, Hexadecimal String, Hexadecimal Bytestring)
        - `Extended WKB` (Bytes, Hexadecimal String, Hexadecimal Bytestring)
        - `WKT` (String)
        - `Extended WKT` (String)

    Args:
        geom_col (array): Column containing the encoded geometry.

    Example:
        >>> decode_geometry(df['the_geom'])

    """
    if geom_col.size > 0:
        enc_type = None
        if any(geom_col):
            first_geom = next(item for item in geom_col if item is not None)
            enc_type = detect_encoding_type(first_geom)
        return GeoSeries(geom_col.apply(lambda g: decode_geometry_item(g, enc_type)))
    else:
        return geom_col

def detect_encoding_type(input_geom):
    """
    Detect geometry encoding type:
    - ENC_WKB: b'\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00H\x93@\x00\x00\x00\x00\x00\x9d\xb6@'
    - ENC_EWKB: b'\x01\x01\x00\x00 \xe6\x10\x00\x00\x00\x00\x00\x00\x00H\x93@\x00\x00\x00\x00\x00\x9d\xb6@'
    - ENC_WKB_HEX: '0101000000000000000048934000000000009DB640'
    - ENC_EWKB_HEX: '0101000020E6100000000000000048934000000000009DB640'
    - ENC_WKB_BHEX: b'0101000000000000000048934000000000009DB640'
    - ENC_EWKB_BHEX: b'0101000020E6100000000000000048934000000000009DB640'
    - ENC_WKT: 'POINT (1234 5789)'
    - ENC_EWKT: 'SRID=4326;POINT (1234 5789)'
    """
    if isinstance(input_geom, shapely.geometry.base.BaseGeometry):
        return ENC_SHAPELY

    if isinstance(input_geom, str):
        if _is_hex(input_geom):
            return ENC_WKB_HEX
        else:
            srid, geom = _extract_srid(input_geom)
            if not geom:
                return None
            if srid:
                return ENC_EWKT
            else:
                return ENC_WKT

    if isinstance(input_geom, bytes):
        try:
            ba.unhexlify(input_geom)
            return ENC_WKB_BHEX
        except Exception:
            return ENC_WKB

    return None

def decode_geometry_item(geom, enc_type):
    """Decode any geometry into a shapely geometry."""
    if geom:
        func = {
            ENC_SHAPELY: lambda: geom,
            ENC_WKB: lambda: _load_wkb(geom),
            ENC_WKB_HEX: lambda: _load_wkb_hex(geom),
            ENC_WKB_BHEX: lambda: _load_wkb_bhex(geom),
            ENC_WKT: lambda: _load_wkt(geom),
            ENC_EWKT: lambda: _load_ewkt(geom)
        }.get(enc_type)
        return func() if func else geom
    return shapely.geometry.base.BaseGeometry()

def _load_wkb(geom):
    """Load WKB or EWKB geometry."""
    return shapely.wkb.loads(geom)

def _load_wkb_hex(geom):
    """Load WKB_HEX or EWKB_HEX geometry."""
    return shapely.wkb.loads(geom, hex=True)

def _load_wkb_bhex(geom):
    """Load WKB_BHEX or EWKB_BHEX geometry.
    The geom must be converted to WKB/EWKB before loading.
    """
    return shapely.wkb.loads(ba.unhexlify(geom))

def _load_wkt(geom):
    """Load WKT geometry."""
    return shapely.wkt.loads(geom)

def _load_ewkt(egeom):
    """Load EWKT geometry.
    The SRID must be removed before loading and added after loading.
    """
    srid, geom = _extract_srid(egeom)
    ogeom = _load_wkt(geom)
    if srid:
        shapely.geos.lgeos.GEOSSetSRID(ogeom._geom, int(srid))
    return ogeom

def _is_hex(input_geom):
    return re.match(r'^[0-9a-fA-F]+$', input_geom)

def _extract_srid(egeom):
    result = re.match(r'^SRID=(\d+);(.*)$', egeom)
    if result:
        return (result.group(1), result.group(2))
    else:
        return (0, egeom)

def encode_geometry_ewkt(geom, srid=4326):
    if isinstance(geom, shapely.geometry.base.BaseGeometry):
        return 'SRID={0};{1}'.format(srid, geom.wkt)

def encode_geometry_ewkb(geom, srid=4326):
    if isinstance(geom, shapely.geometry.base.BaseGeometry):
        shapely.geos.lgeos.GEOSSetSRID(geom._geom, srid)
        return shapely.wkb.dumps(geom, hex=True, include_srid=True)

def to_geojson(geom, buffer_simplify=True):
    if geom is not None and str(geom) != 'GEOMETRYCOLLECTION EMPTY':
        if buffer_simplify and geom.geom_type in (Polygon, Multipolygon):
            return json.dumps(shapely.geometry.mapping(
                geom.buffer(SPHERICAL_TOLERANCE).simplify(SIMPLIFY_TOLERANCE)
            ), sort_keys=True)
        else:
            return json.dumps(shapely.geometry.mapping(geom), sort_keys=True)
<commit_msg>Get Polygon and Multipolygon definitions<commit_after>import re
import json
import shapely
import binascii as ba

from geopandas import GeoSeries, GeoDataFrame, points_from_xy
from shapely.geometry.polygon import Polygon
from shapely.geometry.multipolygon import MultiPolygon

ENC_SHAPELY = 'shapely'
ENC_WKB = 'wkb'
ENC_WKB_HEX = 'wkb-hex'
ENC_WKB_BHEX = 'wkb-bhex'
ENC_WKT = 'wkt'
ENC_EWKT = 'ewkt'
SPHERICAL_TOLERANCE = 0.0001
SIMPLIFY_TOLERANCE = 0.001

def set_geometry(gdf, col, drop=False, inplace=False, crs=None):
    """Set the GeoDataFrame geometry using either an existing column or the specified input.
    By default yields a new object. The original geometry column is replaced with the input.
    It detects the geometry encoding and it decodes the column if required. Supported geometry
    encodings are:

        - `WKB` (Bytes, Hexadecimal String, Hexadecimal Bytestring)
        - `Extended WKB` (Bytes, Hexadecimal String, Hexadecimal Bytestring)
        - `WKT` (String)
        - `Extended WKT` (String)

    Args:
        col (column label or array): Name of the column or column containing the geometry.
        drop (boolean, default False): Delete the column to be used as the new geometry.
        inplace (boolean, default False): Modify the GeoDataFrame in place (do not create a new object).
        crs (str/result of fion.get_crs, optional): Coordinate system to use. If passed, overrides both
            DataFrame and col's crs. Otherwise, tries to get crs from passed col values or DataFrame.

    Example:
        >>> set_geometry(gdf, 'the_geom', drop=True, inplace=True)

    """
    if not isinstance(gdf, GeoDataFrame):
        raise ValueError('gdf must be an instance of geopandas.GeoDataFrame.')

    if inplace:
        frame = gdf
    else:
        frame = gdf.copy()

    # Decode geometry
    if isinstance(col, str):
        if col not in frame:
            raise Exception('Column "{0}" does not exist.'.format(col))
        frame[col] = decode_geometry(frame[col])
    else:
        col = decode_geometry(col)

    # Call set_geometry with decoded column
    frame.set_geometry(col, drop=drop, inplace=True, crs=crs)

    if not inplace:
        return frame

def set_geometry_from_xy(gdf, x, y, drop=False, inplace=False, crs=None):
    """Set the GeoDataFrame geometry using either existing lng/lat columns or the specified inputs.
    By default yields a new object. The original geometry column is replaced with the new one.

    Args:
        x (column label or array): Name of the x (longitude) column or column containing the x coordinates.
        y (column label or array): Name of the y (latitude) column or column containing the y coordinates.
        drop (boolean, default False): Delete the columns to be used to generate the new geometry.
        inplace (boolean, default False): Modify the GeoDataFrame in place (do not create a new object).
        crs (str/result of fion.get_crs, optional): Coordinate system to use. If passed, overrides both
            DataFrame and col's crs. Otherwise, tries to get crs from passed col values or DataFrame.

    Example:
        >>> set_geometry_from_xy(gdf, 'lng', 'lat', drop=True, inplace=True)

    """
    if not isinstance(gdf, GeoDataFrame):
        raise ValueError('gdf must be an instance of geopandas.GeoDataFrame.')

    if isinstance(x, str) and x in gdf and isinstance(y, str) and y in gdf:
        x_col = gdf[x]
        y_col = gdf[y]
    else:
        x_col = x
        y_col = y

    # Generate geometry
    geom_col = points_from_xy(x_col, y_col)

    # Call set_geometry with generated column
    frame = gdf.set_geometry(geom_col, inplace=inplace, crs=crs)

    if drop:
        if frame is None:
            frame = gdf
        del frame[x]
        del frame[y]

    return frame

def has_geometry(gdf):
    """Method to check if the GeoDataFrame contains a valid geometry column.
    If there is no valid geometry, you can use the following methods:

        - `set_geometry`: to create a decoded geometry column from any raw geometry column.
        - `set_geometry_from_xy`: to create a geometry column from `longitude` and `latitude` columns.
    """
    return hasattr(gdf, '_geometry_column_name') and gdf._geometry_column_name in gdf

def decode_geometry(geom_col):
    """Decodes a DataFrame column. It detects the geometry encoding and it decodes the column if required.
    Supported geometry encodings are:

        - `WKB` (Bytes, Hexadecimal String, Hexadecimal Bytestring)
        - `Extended WKB` (Bytes, Hexadecimal String, Hexadecimal Bytestring)
        - `WKT` (String)
        - `Extended WKT` (String)

    Args:
        geom_col (array): Column containing the encoded geometry.

    Example:
        >>> decode_geometry(df['the_geom'])

    """
    if geom_col.size > 0:
        enc_type = None
        if any(geom_col):
            first_geom = next(item for item in geom_col if item is not None)
            enc_type = detect_encoding_type(first_geom)
        return GeoSeries(geom_col.apply(lambda g: decode_geometry_item(g, enc_type)))
    else:
        return geom_col

def detect_encoding_type(input_geom):
    """
    Detect geometry encoding type:
    - ENC_WKB: b'\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00H\x93@\x00\x00\x00\x00\x00\x9d\xb6@'
    - ENC_EWKB: b'\x01\x01\x00\x00 \xe6\x10\x00\x00\x00\x00\x00\x00\x00H\x93@\x00\x00\x00\x00\x00\x9d\xb6@'
    - ENC_WKB_HEX: '0101000000000000000048934000000000009DB640'
    - ENC_EWKB_HEX: '0101000020E6100000000000000048934000000000009DB640'
    - ENC_WKB_BHEX: b'0101000000000000000048934000000000009DB640'
    - ENC_EWKB_BHEX: b'0101000020E6100000000000000048934000000000009DB640'
    - ENC_WKT: 'POINT (1234 5789)'
    - ENC_EWKT: 'SRID=4326;POINT (1234 5789)'
    """
    if isinstance(input_geom, shapely.geometry.base.BaseGeometry):
        return ENC_SHAPELY

    if isinstance(input_geom, str):
        if _is_hex(input_geom):
            return ENC_WKB_HEX
        else:
            srid, geom = _extract_srid(input_geom)
            if not geom:
                return None
            if srid:
                return ENC_EWKT
            else:
                return ENC_WKT

    if isinstance(input_geom, bytes):
        try:
            ba.unhexlify(input_geom)
            return ENC_WKB_BHEX
        except Exception:
            return ENC_WKB

    return None

def decode_geometry_item(geom, enc_type):
    """Decode any geometry into a shapely geometry."""
    if geom:
        func = {
            ENC_SHAPELY: lambda: geom,
            ENC_WKB: lambda: _load_wkb(geom),
            ENC_WKB_HEX: lambda: _load_wkb_hex(geom),
            ENC_WKB_BHEX: lambda: _load_wkb_bhex(geom),
            ENC_WKT: lambda: _load_wkt(geom),
            ENC_EWKT: lambda: _load_ewkt(geom)
        }.get(enc_type)
        return func() if func else geom
    return shapely.geometry.base.BaseGeometry()

def _load_wkb(geom):
    """Load WKB or EWKB geometry."""
    return shapely.wkb.loads(geom)

def _load_wkb_hex(geom):
    """Load WKB_HEX or EWKB_HEX geometry."""
    return shapely.wkb.loads(geom, hex=True)

def _load_wkb_bhex(geom):
    """Load WKB_BHEX or EWKB_BHEX geometry.
    The geom must be converted to WKB/EWKB before loading.
    """
    return shapely.wkb.loads(ba.unhexlify(geom))

def _load_wkt(geom):
    """Load WKT geometry."""
    return shapely.wkt.loads(geom)

def _load_ewkt(egeom):
    """Load EWKT geometry.
    The SRID must be removed before loading and added after loading.
    """
    srid, geom = _extract_srid(egeom)
    ogeom = _load_wkt(geom)
    if srid:
        shapely.geos.lgeos.GEOSSetSRID(ogeom._geom, int(srid))
    return ogeom

def _is_hex(input_geom):
    return re.match(r'^[0-9a-fA-F]+$', input_geom)

def _extract_srid(egeom):
    result = re.match(r'^SRID=(\d+);(.*)$', egeom)
    if result:
        return (result.group(1), result.group(2))
    else:
        return (0, egeom)

def encode_geometry_ewkt(geom, srid=4326):
    if isinstance(geom, shapely.geometry.base.BaseGeometry):
        return 'SRID={0};{1}'.format(srid, geom.wkt)

def encode_geometry_ewkb(geom, srid=4326):
    if isinstance(geom, shapely.geometry.base.BaseGeometry):
        shapely.geos.lgeos.GEOSSetSRID(geom._geom, srid)
        return shapely.wkb.dumps(geom, hex=True, include_srid=True)

def to_geojson(geom, buffer_simplify=True):
    if geom is not None and str(geom) != 'GEOMETRYCOLLECTION EMPTY':
        if buffer_simplify and geom.geom_type in (Polygon, Multipolygon):
            return json.dumps(shapely.geometry.mapping(
                geom.buffer(SPHERICAL_TOLERANCE).simplify(SIMPLIFY_TOLERANCE)
            ), sort_keys=True)
        else:
            return json.dumps(shapely.geometry.mapping(geom), sort_keys=True)
<|endoftext|>


<commit_before><commit_msg>[viewers/power_scan_h5] reorganized functions to simplify their meaning. Data is now shifted to prevent negative values to enter the loglog-plot which had caused issues.<commit_after><|endoftext|>