<commit_before>#!/bin/python3
import csv
import sys
import math
from collections import OrderedDict
from graphviz import Digraph
from cyclefind import CycleFind

class CycleDetect(object):
    def __init__(self, use_last=False):
        self.reset()
        self.use_last = use_last

    def reset(self):
        self.graph = OrderedDict()
        self.origins = []
        self.delay = OrderedDict()
        self.limit = OrderedDict()
        self.trade = OrderedDict()
        self.cyclelimit = None

    def add_link(self, f, t, v, d=None, l=None):
        if f not in self.graph:
            self.graph[f] = {}
        if f not in self.delay:
            self.delay[f] = {}
        if f not in self.limit:
            self.limit[f] = {}
        if t not in self.graph[f]:
            self.graph[f][t] = 0.0
        self.graph[f][t] += v
        if d is not None:
            if t not in self.delay[f]:
                self.delay[f][t] = 0.0
            self.delay[f][t] += d
        if l is not None:
            if t not in self.limit[f]:
                self.limit[f][t] = l
            else:
                self.limit[f][t] = min(self.limit[f][t], l)

    def load(self, fplist):
        tfee = []
        for fp in fplist:
            reader = csv.reader(fp, delimiter=',', quotechar='"')
            for row in reader:
                if len(row) == 0:
                    continue
                lformat = row[0]
                if len(row[0]) == 0 or row[0][0] == '#':
                    continue
                if lformat == 'origin':
                    f = row[1]
                    if f not in self.origins:
                        self.origins.append(f)
                if lformat == 'link':
                    f = row[1]
                    t = row[2]
                    v = float(row[3])
                    self.add_link(f, t, v)
                elif lformat == 'cycle-limit':
                    self.cyclelimit = int(row[1])
                elif lformat == 'node':
                    f = row[1]
                    if f not in self.graph:
                        self.graph[f] = {}
                elif lformat == 'bid-ask':
                    f = row[1]
                    t = row[2]
                    b = math.log(float(row[3]))
                    a = math.log(float(row[4]))
                    if len(row) > 5 and row[5] != "":
                        l = math.log(float(row[5]))
                    else:
                        l = None
                    if self.use_last and l is not None \
                       and b < l and l < a:
                        b = l
                        a = l
                    self.add_link(f, t, -b)
                    self.add_link(t, f, a)
                    if (f, t) in self.trade:
                        trade = self.trade[(f, t)]
                        self.add_link(f, t, -v, d, l)
                    elif (t, f) in self.trade:
                        trade = self.trade[(t, f)]
                        self.add_link(t, f, -v, d, l)
                elif lformat == 'trade':
                    f = row[1]
                    t = row[2]
                    v = math.log(1.0 - float(row[3])/100.0)
                    if len(row) > 4 and row[4] != "":
                        d = float(row[4])
                    else:
                        d = None
                    if len(row) > 5 and row[5] != "":
                        l = float(row[5])
                    else:
                        l = None
                    self.trade[(f,t)] = [v, d, l]
                    if f in self.graph and t in self.graph[f]:
                        self.add_link(f, t, -v, d, l)
                    elif t in self.graph and f in self.graph[t]:
                        self.add_link(t, f, -v, d, l)
                elif lformat == 'transfer':
                    f = row[1]
                    t = row[2]
                    v = math.log(1.0 - float(row[3])/100.0)
                    if len(row) > 4 and row[4] != "":
                        d = float(row[4])
                    else:
                        d = None
                    if len(row) > 5 and row[5] != "":
                        l = float(row[5])
                    else:
                        l = None
                    self.add_link(f, t, -v, d, l)

    def graphviz(self, fp=None):
        if fp is not None:
            self.load(fp);
        nl = self.run()
        show = {}
        dot = Digraph(format='png')
        for ik, iv in self.graph.items():
            ikid = ik.replace(":", "__")
            dot.node(ikid, ik)
        for i in nl:
            i1 = i[0]
            for j in zip(i1, i1[1::]):
                ikid = j[0].replace(":", "__")
                jkid = j[1].replace(":", "__")
                if (ikid, jkid) not in show:
                    show[(ikid, jkid)] = True
                    dot.edge(ikid, jkid, color='red')
        for ik, iv in self.graph.items():
            for jk, jv in iv.items():
                ikid = ik.replace(":", "__")
                jkid = jk.replace(":", "__")
                if (ikid, jkid) not in show:
                    dot.edge(ikid, jkid)
        return dot.pipe()

    def run(self, fp=None):
        if fp is not None:
            self.load(fp)
        retval = []
        cf = CycleFind(self.graph, self.origins, cyclelimit=self.cyclelimit)
        cycles = cf.run()
        negative_cycle_lists = cf.filter_negative(cycles)
        for i in negative_cycle_lists:
            total = 0.0
            d = 0.0
            l = None
            i1 = list(i)
            i1.append(i[0])
            for j in zip(i1, i1[1::]):
                total -= self.graph[j[0]][j[1]]
                d += self.delay[j[0]].get(j[1], 0.0)
                new_limit = self.limit[j[0]].get(j[1], None)
                if new_limit is not None:
                    if l is None or new_limit < l:
                        l = new_limit
            retval.append([i1, math.exp(total), d, l])
        return retval

    def format(self, cycle_list, threshold = 0.0):
        retval = ''
        for i in cycle_list:
            percent = (i[1] - 1.0) * 100
            if percent > threshold:
                retval += " -> ".join(i[0]) + "\n"
                retval += "Expected return: " + str(percent) + "\n"
                retval += "Delay: " + str(i[2]) + "\n"
                if i[3] is not None:
                    retval += "Limit: " + str(i[3]) + "\n"
                retval += "\n"
        return retval

if __name__ == '__main__':
    cd = CycleDetect()
    for i in sys.argv[1:]:
        with open(i, 'r') as csvfile:
            print("Loading: ", i)
            cd.load([csvfile])
    cycles = cd.run()
    cycles.sort(key=lambda a: -a[1])
    import time
    print(time.strftime("%Y-%m-%d %H:%M:%S"))
    print()
    print(cd.format(cycles, 2))
<commit_msg>move threshold into object<commit_after>#!/bin/python3
import csv
import sys
import math
from collections import OrderedDict
from graphviz import Digraph
from cyclefind import CycleFind

class CycleDetect(object):
    def __init__(self, use_last=False, threshold=1.5):
        self.reset()
        self.use_last = use_last
        self.threshold = threshold

    def reset(self):
        self.graph = OrderedDict()
        self.origins = []
        self.delay = OrderedDict()
        self.limit = OrderedDict()
        self.trade = OrderedDict()
        self.cyclelimit = None

    def add_link(self, f, t, v, d=None, l=None):
        if f not in self.graph:
            self.graph[f] = {}
        if f not in self.delay:
            self.delay[f] = {}
        if f not in self.limit:
            self.limit[f] = {}
        if t not in self.graph[f]:
            self.graph[f][t] = 0.0
        self.graph[f][t] += v
        if d is not None:
            if t not in self.delay[f]:
                self.delay[f][t] = 0.0
            self.delay[f][t] += d
        if l is not None:
            if t not in self.limit[f]:
                self.limit[f][t] = l
            else:
                self.limit[f][t] = min(self.limit[f][t], l)

    def load(self, fplist):
        tfee = []
        for fp in fplist:
            reader = csv.reader(fp, delimiter=',', quotechar='"')
            for row in reader:
                if len(row) == 0:
                    continue
                lformat = row[0]
                if len(row[0]) == 0 or row[0][0] == '#':
                    continue
                if lformat == 'origin':
                    f = row[1]
                    if f not in self.origins:
                        self.origins.append(f)
                if lformat == 'link':
                    f = row[1]
                    t = row[2]
                    v = float(row[3])
                    self.add_link(f, t, v)
                elif lformat == 'cycle-limit':
                    self.cyclelimit = int(row[1])
                elif lformat == 'node':
                    f = row[1]
                    if f not in self.graph:
                        self.graph[f] = {}
                elif lformat == 'bid-ask':
                    f = row[1]
                    t = row[2]
                    b = math.log(float(row[3]))
                    a = math.log(float(row[4]))
                    if len(row) > 5 and row[5] != "":
                        l = math.log(float(row[5]))
                    else:
                        l = None
                    if self.use_last and l is not None \
                       and b < l and l < a:
                        b = l
                        a = l
                    self.add_link(f, t, -b)
                    self.add_link(t, f, a)
                    if (f, t) in self.trade:
                        trade = self.trade[(f, t)]
                        self.add_link(f, t, -v, d, l)
                    elif (t, f) in self.trade:
                        trade = self.trade[(t, f)]
                        self.add_link(t, f, -v, d, l)
                elif lformat == 'trade':
                    f = row[1]
                    t = row[2]
                    v = math.log(1.0 - float(row[3])/100.0)
                    if len(row) > 4 and row[4] != "":
                        d = float(row[4])
                    else:
                        d = None
                    if len(row) > 5 and row[5] != "":
                        l = float(row[5])
                    else:
                        l = None
                    self.trade[(f,t)] = [v, d, l]
                    if f in self.graph and t in self.graph[f]:
                        self.add_link(f, t, -v, d, l)
                    elif t in self.graph and f in self.graph[t]:
                        self.add_link(t, f, -v, d, l)
                elif lformat == 'transfer':
                    f = row[1]
                    t = row[2]
                    v = math.log(1.0 - float(row[3])/100.0)
                    if len(row) > 4 and row[4] != "":
                        d = float(row[4])
                    else:
                        d = None
                    if len(row) > 5 and row[5] != "":
                        l = float(row[5])
                    else:
                        l = None
                    self.add_link(f, t, -v, d, l)

    def graphviz(self, fp=None):
        if fp is not None:
            self.load(fp);
        nl = self.run()
        show = {}
        dot = Digraph(format='png')
        for ik, iv in self.graph.items():
            ikid = ik.replace(":", "__")
            dot.node(ikid, ik)
        for i in nl:
            i1 = i[0]
            for j in zip(i1, i1[1::]):
                ikid = j[0].replace(":", "__")
                jkid = j[1].replace(":", "__")
                if (ikid, jkid) not in show:
                    show[(ikid, jkid)] = True
                    dot.edge(ikid, jkid, color='red')
        for ik, iv in self.graph.items():
            for jk, jv in iv.items():
                ikid = ik.replace(":", "__")
                jkid = jk.replace(":", "__")
                if (ikid, jkid) not in show:
                    dot.edge(ikid, jkid)
        return dot.pipe()

    def run(self, fp=None):
        if fp is not None:
            self.load(fp)
        retval = []
        cf = CycleFind(self.graph, self.origins, cyclelimit=self.cyclelimit)
        cycles = cf.run()
        negative_cycle_lists = cf.filter_negative(cycles)
        for i in negative_cycle_lists:
            total = 0.0
            d = 0.0
            l = None
            i1 = list(i)
            i1.append(i[0])
            for j in zip(i1, i1[1::]):
                total -= self.graph[j[0]][j[1]]
                d += self.delay[j[0]].get(j[1], 0.0)
                new_limit = self.limit[j[0]].get(j[1], None)
                if new_limit is not None:
                    if l is None or new_limit < l:
                        l = new_limit
            retval.append([i1, math.exp(total), d, l])
        return retval

    def format(self, cycle_list):
        retval = ''
        for i in cycle_list:
            percent = (i[1] - 1.0) * 100
            if percent > self.threshold:
                retval += " -> ".join(i[0]) + "\n"
                retval += "Expected return: " + str(percent) + "\n"
                retval += "Delay: " + str(i[2]) + "\n"
                if i[3] is not None:
                    retval += "Limit: " + str(i[3]) + "\n"
                retval += "\n"
        return retval

if __name__ == '__main__':
    cd = CycleDetect(threshold=1.5)
    for i in sys.argv[1:]:
        with open(i, 'r') as csvfile:
            print("Loading: ", i)
            cd.load([csvfile])
    cycles = cd.run()
    cycles.sort(key=lambda a: -a[1])
    import time
    print(time.strftime("%Y-%m-%d %H:%M:%S"))
    print()
    print(cd.format(cycles))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from sympy.core import Basic, Rational
from sympy.core.numbers import igcd
from .cartan_type import CartanType
from sympy.mpmath import fac
from operator import itemgetter
from itertools import groupby
from sympy.matrices import Matrix, eye

class WeylGroup(Basic):

    """
    For each semisimple Lie group, we have a Weyl group.  It is a subgroup of
    the isometry group of the root system.  Specifically, it’s the subgroup
    that is generated by reflections through the hyperplanes orthogonal to
    the roots.  Therefore, Weyl groups are reflection groups, and so a Weyl
    group is a finite Coxeter group.

    """

    def __new__(cls, cartantype):
        obj = Basic.__new__(cls, cartantype)
        obj.cartan_type = CartanType(cartantype)
        return obj

    def generators(self):
        """
        This method creates the generating reflections of the Weyl group for
        a given Lie algebra.  For a Lie algebra of rank n, there are n
        different generating reflections.  This function returns them as
        a list.

        Example
        =======
        >>> from sympy.liealgebras.weyl_group import WeylGroup
        >>> c = WeylGroup("F4")
        >>> c.generators()
        ['r1', 'r2', 'r3', 'r4']
        """
        n = self.cartan_type.rank()
        generators = []
        for i in range(1, n+1):
            reflection = "r"+str(i)
            generators.append(reflection)
        return generators

    def group_order(self):
        """
        This method returns the order of the Weyl group.
        For types A, B, C, D, and E the order depends on
        the rank of the Lie algebra.  For types F and G,
        the order is fixed.

        Example
        =======
        >>> from sympy.liealgebras.weyl_group import WeylGroup
        >>> c = WeylGroup("D4")
        >>> c.group_order()
        192.0
        """
        n = self.cartan_type.rank()
        if self.cartan_type.series == "A":
            return fac(n+1)

        if self.cartan_type.series == "B" or self.cartan_type.series ==  "C":
            return fac(n)*(2**n)

        if self.cartan_type.series == "D":
            return fac(n)*(2**(n-1))

        if self.cartan_type.series == "E":
            if n == 6:
                return 51840
            if n == 7:
                return 2903040
            if n == 8:
                return 696729600
        if self.cartan_type.series == "F":
            return 1152

        if self.cartan_type.series == "G":
            return 12

    def group_name(self):
        n = self.cartan_type.rank()
        if self.cartan_type.series == "A":
            return "S"+str(n+1) + ": the symmetric group acting on " + str(n+1) + " elements."

        if self.cartan_type.series == "B" or self.cartan_type.series ==  "C":
            return "The hyperoctahedral group acting on " + str(2*n) + " elements."

        if self.cartan_type.series == "D":
            return "The symmetry group of the " + str(n) + "-dimensional demihypercube."

        if self.cartan_type.series == "E":
            if n == 6:
                return "The symmetry group of the 6-polytope."

            if n == 7:
                return "The symmetry group of the 7-polytope."

            if n == 8:
                return "The symmetry group of the 8-polytope."

        if self.cartan_type.series == "F":
            return "The symmetry group of the 24-cell, or icositetrachoron."

        if self.cartan_type.series == "G":
            return "D6, the dihedral group of order 12, and symmetry group of the hexagon."

    def element_order(self, reflection):
        n = self.cartan_type.rank()
        if self.cartan_type.series == "A":
            a = self.matrix_form(reflection)
            order = 1
            while a != eye(n+1):
                a *= self.matrix_form(reflection)
                order += 1
            return order

        if self.cartan_type.series == "D":
            a = self.matrix_form(reflection)
            order = 1
            while a != eye(n):
                a *= self.matrix_form(reflection)
                order += 1
            return order

        if self.cartan_type.series == "E":
            a = self.matrix_form(reflection)
            order = 1
            while a != eye(8):
                a *= self.matrix_form(reflection)
                order += 1
            return order

        if self.cartan_type.series == "G":
            elts = list(reflection)
            reflections = elts[1::3]
            m = self.delete_doubles(reflections)
            while self.delete_doubles(m) != m:
                m = self.delete_doubles(m)
                reflections = m
            if len(reflections) % 2 == 1:
                return 2

            elif len(reflections) == 0:
                return 1

            else:
                if len(reflections) == 1:
                    return 2
                else:
                    m = len(reflections) / 2
                    lcm = (6 * m)/ igcd(m, 6)
                order = lcm / m
                return order

        if self.cartan_type.series == 'F':
            a = self.matrix_form(reflection)
            order = 1
            while a != eye(4):
                a *= self.matrix_form(reflection)
                order += 1
            return order

        if self.cartan_type.series == "B" or self.cartan_type.series == "C":
            a = self.matrix_form(reflection)
            order = 1
            while a != eye(n):
                a *= self.matrix_form(reflection)
                order += 1
            return order

    def delete_doubles(self, reflections):
        """
        This is a helper method for determining the order of an element in the Weyl group of G2
        """
        counter = 0
        copy = list(reflections)
        for elt in copy:
            if counter < len(copy)-1:
                if copy[counter + 1] == elt:
                    del copy[counter]
                    del copy[counter]
            counter += 1

        return copy

    def matrix_form(self, reflection):
        elts = list(reflection)
        reflections = elts[1::3]
        n = self.cartan_type.rank()
        if self.cartan_type.series == 'A':
            matrixform = eye(n+1)
            for elt in reflections:
                a = int(elt)
                mat = eye(n+1)
                mat[a-1, a-1] = 0
                mat[a-1, a] = 1
                mat[a, a-1] = 1
                mat[a, a] = 0
                matrixform *= mat
            return matrixform

        if self.cartan_type.series == 'D':
            matrixform = eye(n)
            for elt in reflections:
                a = int(elt)
                mat = eye(n)
                if a < n:
                    mat[a-1, a-1] = 0
                    mat[a-1, a] = 1
                    mat[a, a-1] = 1
                    mat[a, a] = 0
                    matrixform *= mat
                else:
                    mat[n-2, n-1] = -1
                    mat[n-2, n-2] = 0
                    mat[n-1, n-2] = -1
                    mat[n-1, n-1] = 0
                    matrixform *= mat
            return matrixform

        if self.cartan_type.series == 'G':
            matrixform = eye(3)
            for elt in reflections:
                a = int(elt)
                if a == 1:
                    gen1 = Matrix([[1, 0, 0], [0, 0, 1], [0, 1, 0]])
                    matrixform *= gen1
                else:
                    gen2 = Matrix([[Rational(2, 3), Rational(2, 3), -Rational(1, 3)],
                        [Rational(2, 3), Rational(-1, 3), Rational(2, 3)], [Rational(-1, 3),
                            Rational(2, 3), Rational(2, 3)]])
                    matrixform *= gen2
            return matrixform

        if self.cartan_type.series == 'F':
            matrixform = eye(4)
            for elt in reflections:
                a = int(elt)
                if a == 1:
                    mat = Matrix([[1, 0, 0, 0], [0, 0, 1, 0], [0, 1, 0, 0], [0, 0, 0, 1]])
                    matrixform *= mat
                elif a == 2:
                    mat = Matrix([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 0, 1], [0, 0, 1, 0]])
                    matrixform *= mat
                elif a == 3:
                    mat = Matrix([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, -1]])
                    matrixform *= mat
                else:

                    mat = Matrix([[Rational(1, 2), Rational(1, 2), Rational(1, 2), Rational(1, 2)],
                        [Rational(1, 2), Rational(1, 2), Rational(-1, 2), Rational(-1, 2)],
                        [Rational(1, 2), Rational(-1, 2), Rational(1, 2), Rational(-1, 2)],
                        [Rational(1, 2), Rational(-1, 2), Rational(-1, 2), Rational(1, 2)]])
                    matrixform *= mat
            return matrixform

        if self.cartan_type.series == 'E':
            matrixform = eye(8)
            for elt in reflections:
                a = int(elt)
                if a == 1:
                    mat = Matrix([[Rational(3, 4), Rational(1, 4), Rational(1, 4), Rational(1, 4),
                        Rational(1, 4), Rational(1, 4), Rational(1, 4), Rational(-1, 4)],
                        [Rational(1, 4), Rational(3, 4), Rational(-1, 4), Rational(-1, 4),
                            Rational(-1, 4), Rational(-1, 4), Rational(1, 4), Rational(-1, 4)],
                        [Rational(1, 4), Rational(-1, 4), Rational(3, 4), Rational(-1, 4),
                        Rational(-1, 4), Rational(-1, 4), Rational(-1, 4), Rational(1, 4)],
                        [Rational(1, 4), Rational(-1, 4), Rational(-1, 4), Rational(3, 4),
                        Rational(-1, 4), Rational(-1, 4), Rational(-1, 4), Rational(1, 4)],
                        [Rational(1, 4), Rational(-1, 4), Rational(-1, 4), Rational(-1, 4),
                        Rational(3, 4), Rational(-1, 4), Rational(-1, 4), Rational(1, 4)],
                        [Rational(1, 4), Rational(-1, 4), Rational(-1, 4), Rational(-1, 4),
                        Rational(-1, 4), Rational(3, 4), Rational(-1, 4), Rational(1, 4)],
                        [Rational(1, 4), Rational(-1, 4), Rational(-1, 4), Rational(-1, 4),
                        Rational(-1, 4), Rational(-1, 4), Rational(-3, 4), Rational(1, 4)],
                        [Rational(1, 4), Rational(-1, 4), Rational(-1, 4), Rational(-1, 4),
                        Rational(-1, 4), Rational(-1, 4), Rational(-1, 4), Rational(3, 4)]])
                    matrixform *= mat
                elif a == 2:
                    mat = eye(8)
                    mat[0, 0] = 0
                    mat[0, 1] = -1
                    mat[1, 0] = -1
                    mat[1, 1] = 0
                    matrixform *= mat
                else:
                    mat = eye(8)
                    mat[a-3, a-3] = 0
                    mat[a-3, a-2] = 1
                    mat[a-2, a-3] = 1
                    mat[a-2, a-2] = 0
                    matrixform *= mat
            return matrixform

        if self.cartan_type.series == 'B' or self.cartan_type.series == 'C':
            matrixform = eye(n)
            for elt in reflections:
                a = int(elt)
                mat = eye(n)
                if a == 1:
                    mat[0, 0] = -1
                    matrixform *= mat
                else:
                    mat[a - 2, a - 2] = 0
                    mat[a-2, a-1] = 1
                    mat[a - 1, a - 2] = 1
                    mat[a -1, a - 1] = 0
                    matrixform *= mat
            return matrixform

    def coxeter_diagram(self):
        """
        This method returns the Coxeter diagram corresponding to a Weyl group.
        The Coxeter diagram can be obtained from a Lie algebra's Dynkin diagram
        by deleting all arrows; the Coxeter diagram is the undirected graph.
        The vertices of the Coxeter diagram represent the generating reflections
        of the Weyl group, , s_i.  An edge is drawn between s_i and s_j if the order
        m(i, j) of s_i*s_j is greater than two.  If there is one edge, the order
        m(i, j) is 3.  If there are two edges, the order m(i, j) is 4, and if there
        are three edges, the order m(i, j) is 6.

        Example
        ========
        >>> from sympy.liealgebras.weyl_group import WeylGroup
        >>> c = WeylGroup("B3")
        >>> print c.coxeter_diagram()
        0---0===0
        1   2   3
        """
        n = self.cartan_type.rank()
        if self.cartan_type.series == "A" or self.cartan_type.series == "D" or self.cartan_type.series == "E":
            return self.cartan_type.dynkin_diagram()

        if self.cartan_type.series == "B" or self.cartan_type.series ==  "C":
            diag = "---".join("0" for i in range (1, n)) + "===0\n"
            diag += "   ".join(str(i) for i in range (1, n+1))
            return diag

        if self.cartan_type.series == "F":
            diag = "0---0===0---0\n"
            diag += "   ".join(str(i) for i in range(1, 5))
            return diag

        if self.cartan_type.series == "G":
            diag = "0≡≡≡0\n1   2"
            return diag

<commit_msg>Adding docstrings and doctests<commit_after># -*- coding: utf-8 -*-

from sympy.core import Basic, Rational
from sympy.core.numbers import igcd
from .cartan_type import CartanType
from sympy.mpmath import fac
from operator import itemgetter
from itertools import groupby
from sympy.matrices import Matrix, eye

class WeylGroup(Basic):

    """
    For each semisimple Lie group, we have a Weyl group.  It is a subgroup of
    the isometry group of the root system.  Specifically, it’s the subgroup
    that is generated by reflections through the hyperplanes orthogonal to
    the roots.  Therefore, Weyl groups are reflection groups, and so a Weyl
    group is a finite Coxeter group.

    """

    def __new__(cls, cartantype):
        obj = Basic.__new__(cls, cartantype)
        obj.cartan_type = CartanType(cartantype)
        return obj

    def generators(self):
        """
        This method creates the generating reflections of the Weyl group for
        a given Lie algebra.  For a Lie algebra of rank n, there are n
        different generating reflections.  This function returns them as
        a list.

        Example
        =======
        >>> from sympy.liealgebras.weyl_group import WeylGroup
        >>> c = WeylGroup("F4")
        >>> c.generators()
        ['r1', 'r2', 'r3', 'r4']
        """
        n = self.cartan_type.rank()
        generators = []
        for i in range(1, n+1):
            reflection = "r"+str(i)
            generators.append(reflection)
        return generators

    def group_order(self):
        """
        This method returns the order of the Weyl group.
        For types A, B, C, D, and E the order depends on
        the rank of the Lie algebra.  For types F and G,
        the order is fixed.

        Example
        =======
        >>> from sympy.liealgebras.weyl_group import WeylGroup
        >>> c = WeylGroup("D4")
        >>> c.group_order()
        192.0
        """
        n = self.cartan_type.rank()
        if self.cartan_type.series == "A":
            return fac(n+1)

        if self.cartan_type.series == "B" or self.cartan_type.series ==  "C":
            return fac(n)*(2**n)

        if self.cartan_type.series == "D":
            return fac(n)*(2**(n-1))

        if self.cartan_type.series == "E":
            if n == 6:
                return 51840
            if n == 7:
                return 2903040
            if n == 8:
                return 696729600
        if self.cartan_type.series == "F":
            return 1152

        if self.cartan_type.series == "G":
            return 12

    def group_name(self):
        """
        This method returns some general information about the Weyl group for
        a given Lie algebra.  It returns the name of the group and the elements
        it acts on, if relevant.
        """
        n = self.cartan_type.rank()
        if self.cartan_type.series == "A":
            return "S"+str(n+1) + ": the symmetric group acting on " + str(n+1) + " elements."

        if self.cartan_type.series == "B" or self.cartan_type.series ==  "C":
            return "The hyperoctahedral group acting on " + str(2*n) + " elements."

        if self.cartan_type.series == "D":
            return "The symmetry group of the " + str(n) + "-dimensional demihypercube."

        if self.cartan_type.series == "E":
            if n == 6:
                return "The symmetry group of the 6-polytope."

            if n == 7:
                return "The symmetry group of the 7-polytope."

            if n == 8:
                return "The symmetry group of the 8-polytope."

        if self.cartan_type.series == "F":
            return "The symmetry group of the 24-cell, or icositetrachoron."

        if self.cartan_type.series == "G":
            return "D6, the dihedral group of order 12, and symmetry group of the hexagon."

    def element_order(self, weylelt):
        """
        This method returns the order of a given Weyl group element, which should
        be specified by the user in the form of products of the generating
        reflections, i.e. of the form r1*r2 etc.

        For types A-F, this method current works by taking the matrix form of
        the specified element, and then finding what power of the matrix is the
        identity.  It then returns this power.

        Example
        ========
        >>> from sympy.liealgebras.weyl_group import WeylGroup
        >>> b = WeylGroup("B4")
        >>> b.element_order('r1*r4*r2')
        4
        """
        n = self.cartan_type.rank()
        if self.cartan_type.series == "A":
            a = self.matrix_form(weylelt)
            order = 1
            while a != eye(n+1):
                a *= self.matrix_form(weylelt)
                order += 1
            return order

        if self.cartan_type.series == "D":
            a = self.matrix_form(weylelt)
            order = 1
            while a != eye(n):
                a *= self.matrix_form(weylelt)
                order += 1
            return order

        if self.cartan_type.series == "E":
            a = self.matrix_form(weylelt)
            order = 1
            while a != eye(8):
                a *= self.matrix_form(weylelt)
                order += 1
            return order

        if self.cartan_type.series == "G":
            elts = list(weylelt)
            reflections = elts[1::3]
            m = self.delete_doubles(weylelt)
            while self.delete_doubles(m) != m:
                m = self.delete_doubles(m)
                reflections = m
            if len(reflections) % 2 == 1:
                return 2

            elif len(reflections) == 0:
                return 1

            else:
                if len(reflections) == 1:
                    return 2
                else:
                    m = len(reflections) / 2
                    lcm = (6 * m)/ igcd(m, 6)
                order = lcm / m
                return order

        if self.cartan_type.series == 'F':
            a = self.matrix_form(weylelt)
            order = 1
            while a != eye(4):
                a *= self.matrix_form(weylelt)
                order += 1
            return order

        if self.cartan_type.series == "B" or self.cartan_type.series == "C":
            a = self.matrix_form(weylelt)
            order = 1
            while a != eye(n):
                a *= self.matrix_form(weylelt)
                order += 1
            return order

    def delete_doubles(self, reflections):
        """
        This is a helper method for determining the order of an element in the
        Weyl group of G2.  It takes a Weyl element and if repeated simple reflections
        in it, it deletes them.
        """
        counter = 0
        copy = list(reflections)
        for elt in copy:
            if counter < len(copy)-1:
                if copy[counter + 1] == elt:
                    del copy[counter]
                    del copy[counter]
            counter += 1

        return copy

    def matrix_form(self, weylelt):
        """
        This method takes input from the user in the form of products of the
        generating reflections, and returns the matrix corresponding to the
        element of the Weyl group.  Since each element of the Weyl group is
        a reflection of some type, there is a corresponding matrix representation.
        This method uses the standard representation for all the generating
        reflections.

        Example
        =======
        >>> from sympy.liealgebras.weyl_group import WeylGroup
        >>> f = WeylGroup("F4")
        >>> f.matrix_form('r2*r3')
        Matrix([
        [1, 0, 0,  0],
        [0, 1, 0,  0],
        [0, 0, 0, -1],
        [0, 0, 1,  0]])

        """
        elts = list(weylelt)
        reflections = elts[1::3]
        n = self.cartan_type.rank()
        if self.cartan_type.series == 'A':
            matrixform = eye(n+1)
            for elt in reflections:
                a = int(elt)
                mat = eye(n+1)
                mat[a-1, a-1] = 0
                mat[a-1, a] = 1
                mat[a, a-1] = 1
                mat[a, a] = 0
                matrixform *= mat
            return matrixform

        if self.cartan_type.series == 'D':
            matrixform = eye(n)
            for elt in reflections:
                a = int(elt)
                mat = eye(n)
                if a < n:
                    mat[a-1, a-1] = 0
                    mat[a-1, a] = 1
                    mat[a, a-1] = 1
                    mat[a, a] = 0
                    matrixform *= mat
                else:
                    mat[n-2, n-1] = -1
                    mat[n-2, n-2] = 0
                    mat[n-1, n-2] = -1
                    mat[n-1, n-1] = 0
                    matrixform *= mat
            return matrixform

        if self.cartan_type.series == 'G':
            matrixform = eye(3)
            for elt in reflections:
                a = int(elt)
                if a == 1:
                    gen1 = Matrix([[1, 0, 0], [0, 0, 1], [0, 1, 0]])
                    matrixform *= gen1
                else:
                    gen2 = Matrix([[Rational(2, 3), Rational(2, 3), -Rational(1, 3)],
                        [Rational(2, 3), Rational(-1, 3), Rational(2, 3)], [Rational(-1, 3),
                            Rational(2, 3), Rational(2, 3)]])
                    matrixform *= gen2
            return matrixform

        if self.cartan_type.series == 'F':
            matrixform = eye(4)
            for elt in reflections:
                a = int(elt)
                if a == 1:
                    mat = Matrix([[1, 0, 0, 0], [0, 0, 1, 0], [0, 1, 0, 0], [0, 0, 0, 1]])
                    matrixform *= mat
                elif a == 2:
                    mat = Matrix([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 0, 1], [0, 0, 1, 0]])
                    matrixform *= mat
                elif a == 3:
                    mat = Matrix([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, -1]])
                    matrixform *= mat
                else:

                    mat = Matrix([[Rational(1, 2), Rational(1, 2), Rational(1, 2), Rational(1, 2)],
                        [Rational(1, 2), Rational(1, 2), Rational(-1, 2), Rational(-1, 2)],
                        [Rational(1, 2), Rational(-1, 2), Rational(1, 2), Rational(-1, 2)],
                        [Rational(1, 2), Rational(-1, 2), Rational(-1, 2), Rational(1, 2)]])
                    matrixform *= mat
            return matrixform

        if self.cartan_type.series == 'E':
            matrixform = eye(8)
            for elt in reflections:
                a = int(elt)
                if a == 1:
                    mat = Matrix([[Rational(3, 4), Rational(1, 4), Rational(1, 4), Rational(1, 4),
                        Rational(1, 4), Rational(1, 4), Rational(1, 4), Rational(-1, 4)],
                        [Rational(1, 4), Rational(3, 4), Rational(-1, 4), Rational(-1, 4),
                            Rational(-1, 4), Rational(-1, 4), Rational(1, 4), Rational(-1, 4)],
                        [Rational(1, 4), Rational(-1, 4), Rational(3, 4), Rational(-1, 4),
                        Rational(-1, 4), Rational(-1, 4), Rational(-1, 4), Rational(1, 4)],
                        [Rational(1, 4), Rational(-1, 4), Rational(-1, 4), Rational(3, 4),
                        Rational(-1, 4), Rational(-1, 4), Rational(-1, 4), Rational(1, 4)],
                        [Rational(1, 4), Rational(-1, 4), Rational(-1, 4), Rational(-1, 4),
                        Rational(3, 4), Rational(-1, 4), Rational(-1, 4), Rational(1, 4)],
                        [Rational(1, 4), Rational(-1, 4), Rational(-1, 4), Rational(-1, 4),
                        Rational(-1, 4), Rational(3, 4), Rational(-1, 4), Rational(1, 4)],
                        [Rational(1, 4), Rational(-1, 4), Rational(-1, 4), Rational(-1, 4),
                        Rational(-1, 4), Rational(-1, 4), Rational(-3, 4), Rational(1, 4)],
                        [Rational(1, 4), Rational(-1, 4), Rational(-1, 4), Rational(-1, 4),
                        Rational(-1, 4), Rational(-1, 4), Rational(-1, 4), Rational(3, 4)]])
                    matrixform *= mat
                elif a == 2:
                    mat = eye(8)
                    mat[0, 0] = 0
                    mat[0, 1] = -1
                    mat[1, 0] = -1
                    mat[1, 1] = 0
                    matrixform *= mat
                else:
                    mat = eye(8)
                    mat[a-3, a-3] = 0
                    mat[a-3, a-2] = 1
                    mat[a-2, a-3] = 1
                    mat[a-2, a-2] = 0
                    matrixform *= mat
            return matrixform

        if self.cartan_type.series == 'B' or self.cartan_type.series == 'C':
            matrixform = eye(n)
            for elt in reflections:
                a = int(elt)
                mat = eye(n)
                if a == 1:
                    mat[0, 0] = -1
                    matrixform *= mat
                else:
                    mat[a - 2, a - 2] = 0
                    mat[a-2, a-1] = 1
                    mat[a - 1, a - 2] = 1
                    mat[a -1, a - 1] = 0
                    matrixform *= mat
            return matrixform

    def coxeter_diagram(self):
        """
        This method returns the Coxeter diagram corresponding to a Weyl group.
        The Coxeter diagram can be obtained from a Lie algebra's Dynkin diagram
        by deleting all arrows; the Coxeter diagram is the undirected graph.
        The vertices of the Coxeter diagram represent the generating reflections
        of the Weyl group, , s_i.  An edge is drawn between s_i and s_j if the order
        m(i, j) of s_i*s_j is greater than two.  If there is one edge, the order
        m(i, j) is 3.  If there are two edges, the order m(i, j) is 4, and if there
        are three edges, the order m(i, j) is 6.

        Example
        ========
        >>> from sympy.liealgebras.weyl_group import WeylGroup
        >>> c = WeylGroup("B3")
        >>> print c.coxeter_diagram()
        0---0===0
        1   2   3
        """
        n = self.cartan_type.rank()
        if self.cartan_type.series == "A" or self.cartan_type.series == "D" or self.cartan_type.series == "E":
            return self.cartan_type.dynkin_diagram()

        if self.cartan_type.series == "B" or self.cartan_type.series ==  "C":
            diag = "---".join("0" for i in range (1, n)) + "===0\n"
            diag += "   ".join(str(i) for i in range (1, n+1))
            return diag

        if self.cartan_type.series == "F":
            diag = "0---0===0---0\n"
            diag += "   ".join(str(i) for i in range(1, 5))
            return diag

        if self.cartan_type.series == "G":
            diag = "0≡≡≡0\n1   2"
            return diag

<|endoftext|>


<commit_before>#! python3
# -*- coding: utf-8 -*-
"""Performs cleanup operations.

This module implements functions designed to clean up various data sets:

* Sends and deletes stored messages in the MySQL-compatible database.
* Checks bot's comments and removes comments below a certain threshold.
* TODO: Checks bot's messages for keywords and deletes them after:

    * Adding message author to call reminder.
    * PMing list of message author's calls.
    * Deleting message author from call.

"""

import praw
import praw.exceptions

import logging.config
import datetime
import time

from clashcallerbotreddit.database import ClashCallerDatabase
from clashcallerbotreddit.search import is_recent
from clashcallerbotreddit import LOGGING, config

# Logger
logging.config.dictConfig(LOGGING)
logging.raiseExceptions = True  # Production mode if False (no console sys.stderr output)
logger = logging.getLogger('reply')

# Generate reddit instance
reddit = praw.Reddit('clashcallerreply')  # Section name in praw.ini
subreddit = reddit.subreddit('ClashCallerBot')  # Limit scope for testing purposes

# Make database instance
db = ClashCallerDatabase(config, root_user=False)

# Time constants
start_time = datetime.datetime.now(datetime.timezone.utc)
archive_time = start_time - datetime.timedelta(weeks=12)  # 6 months archival time

def main():
    logger.info('Start reply.py...')
    while True:
        time.sleep(120)  # 2 minutes

        # Check saved messages
        check_database()

        # Check for comments below threshold
        check_comments('ClashCallerBot')

def check_comments(usr: str, limit: int = -4)-> None:
    """Checks comments and deletes if below threshold.

    Checks given user's last 100 comments and deletes each one below the given threshold.

    Args:
        usr: User to check comments of.
        limit: Threshold below which comment will be deleted. Defaults to ``-4``.

    Returns:
         None. Comments below threshold are deleted.

    Note:
        Skips archived comments (> 6 months from start time).

    """
    try:
        comments = reddit.redditor(usr).comments.new()
        for comment in comments:
            if not is_recent(comment.created_utc, archive_time):
                logger.info(f'Skipping comments after {archive_time}.')
                return None

            if comment.score < limit:
                logger.info(f'Deleting comment below threshold of {limit}: {comment.id}.')
                comment.delete()
            logger.debug(f'Comment#{comment.id}, karma:{comment.score} is above threshold of {limit}.')
            continue
    except praw.exceptions.PRAWException as err:
        logger.exception(f'check_comments: {err}')

def check_database()-> None:
    """Checks messages in database and sends PM if expiration time passed.

    Checks messages saved in a MySQL-compatible database and sends a reminder
    via PM if the expiration time has passed. If so, the message is removed from the
    database.

    Returns:
         None. Message is removed from database if expiration time has passed.
    """
    db.open_connections()
    # Get list of messages older than current datetime
    now = datetime.datetime.now(datetime.timezone.utc)
    messages = db.get_messages(now)

    if not messages:
        logger.debug(f'No messages before: {now}.')
        db.close_connections()
        return None

    # Send reminder PM
    for message in messages:
        tid, link, msg, _exp, usr = message
        logger.debug(f'Found message: {tid}, {msg}')
        send_reminder(link, msg, usr)
        logger.info(f'Reminder sent: {link}.')

        # Delete message from database
        db.delete_message(tid)
        logger.info(f'Message deleted.')
    db.close_connections()

def send_reminder(link: str, msg: str, usr: str)-> None:
    """Sends reminder PM to username.

    Function sends given permalink and message to given username.

    Args:
         link:  Permalink to comment.
         msg:   Message in comment that was saved.
         usr:   User to send reminder to.

    """
    subject = 'ClashCallerBot Private Message Here!'
    permalink = 'https://np.reddit.com' + link  # Permalinks are missing prefix
    parent = get_parent(link)
    message = f"""**The message:** {msg}
              **The original comment:** {permalink}
              **The parent comment or submission:** {parent}

              Thank you for entrusting us with your warring needs,
              - ClashCallerBot

              [^(More info)](https://www.reddit.com/r/ClashCallerBot/comments/4e9vo7/clashcallerbot_info/)
              """
    try:
        reddit.redditor(usr).message(subject, message.replace('              ', ''))

    except praw.exceptions.PRAWException as err:
        logger.exception(f'send_reminder: {err}')

def get_parent(link: str) -> str:
    """Fetch parent comment or submission.

    Function gets parent comment of given permalink or submission if top level comment.

    Args:
         link:  Permalink to get parent of.

    Returns:
        Parent comment link or default string.
    """
    permalink = 'https://www.reddit.com' + link  # Permalink is missing prefix
    parent = 'Parent comment not found.'  # Default string
    try:
        comment = reddit.comment(url=permalink)  # Fetch comment by URL
        parent = 'https://np.reddit.com' + comment.submission.permalink
    except praw.exceptions as err:
        logger.exception(f'get_parent: {err}')
    return parent

# If run directly, instead of imported as a module, run main():
if __name__ == '__main__':
    main()
<commit_msg>Update reply.py moved sleep timer to end of while loop in main()<commit_after>#! python3
# -*- coding: utf-8 -*-
"""Performs cleanup operations.

This module implements functions designed to clean up various data sets:

* Sends and deletes stored messages in the MySQL-compatible database.
* Checks bot's comments and removes comments below a certain threshold.
* TODO: Checks bot's messages for keywords and deletes them after:

    * Adding message author to call reminder.
    * PMing list of message author's calls.
    * Deleting message author from call.

"""

import praw
import praw.exceptions

import logging.config
import datetime
import time

from clashcallerbotreddit.database import ClashCallerDatabase
from clashcallerbotreddit.search import is_recent
from clashcallerbotreddit import LOGGING, config

# Logger
logging.config.dictConfig(LOGGING)
logging.raiseExceptions = True  # Production mode if False (no console sys.stderr output)
logger = logging.getLogger('reply')

# Generate reddit instance
reddit = praw.Reddit('clashcallerreply')  # Section name in praw.ini
subreddit = reddit.subreddit('ClashCallerBot')  # Limit scope for testing purposes

# Make database instance
db = ClashCallerDatabase(config, root_user=False)

# Time constants
start_time = datetime.datetime.now(datetime.timezone.utc)
archive_time = start_time - datetime.timedelta(weeks=12)  # 6 months archival time

def main():
    logger.info('Start reply.py...')
    while True:
        # Check saved messages
        check_database()

        # Check for comments below threshold
        check_comments('ClashCallerBot')

        time.sleep(120)  # 2 minutes

def check_comments(usr: str, limit: int = -4)-> None:
    """Checks comments and deletes if below threshold.

    Checks given user's last 100 comments and deletes each one below the given threshold.

    Args:
        usr: User to check comments of.
        limit: Threshold below which comment will be deleted. Defaults to ``-4``.

    Returns:
         None. Comments below threshold are deleted.

    Note:
        Skips archived comments (> 6 months from start time).

    """
    try:
        comments = reddit.redditor(usr).comments.new()
        for comment in comments:
            if not is_recent(comment.created_utc, archive_time):
                logger.info(f'Skipping comments after {archive_time}.')
                return None

            if comment.score < limit:
                logger.info(f'Deleting comment below threshold of {limit}: {comment.id}.')
                comment.delete()
            logger.debug(f'Comment#{comment.id}, karma:{comment.score} is above threshold of {limit}.')
            continue
    except praw.exceptions.PRAWException as err:
        logger.exception(f'check_comments: {err}')

def check_database()-> None:
    """Checks messages in database and sends PM if expiration time passed.

    Checks messages saved in a MySQL-compatible database and sends a reminder
    via PM if the expiration time has passed. If so, the message is removed from the
    database.

    Returns:
         None. Message is removed from database if expiration time has passed.
    """
    db.open_connections()
    # Get list of messages older than current datetime
    now = datetime.datetime.now(datetime.timezone.utc)
    messages = db.get_messages(now)

    if not messages:
        logger.debug(f'No messages before: {now}.')
        db.close_connections()
        return None

    # Send reminder PM
    for message in messages:
        tid, link, msg, _exp, usr = message
        logger.debug(f'Found message: {tid}, {msg}')
        send_reminder(link, msg, usr)
        logger.info(f'Reminder sent: {link}.')

        # Delete message from database
        db.delete_message(tid)
        logger.info(f'Message deleted.')
    db.close_connections()

def send_reminder(link: str, msg: str, usr: str)-> None:
    """Sends reminder PM to username.

    Function sends given permalink and message to given username.

    Args:
         link:  Permalink to comment.
         msg:   Message in comment that was saved.
         usr:   User to send reminder to.

    """
    subject = 'ClashCallerBot Private Message Here!'
    permalink = 'https://np.reddit.com' + link  # Permalinks are missing prefix
    parent = get_parent(link)
    message = f"""**The message:** {msg}
              **The original comment:** {permalink}
              **The parent comment or submission:** {parent}

              Thank you for entrusting us with your warring needs,
              - ClashCallerBot

              [^(More info)](https://www.reddit.com/r/ClashCallerBot/comments/4e9vo7/clashcallerbot_info/)
              """
    try:
        reddit.redditor(usr).message(subject, message.replace('              ', ''))

    except praw.exceptions.PRAWException as err:
        logger.exception(f'send_reminder: {err}')

def get_parent(link: str) -> str:
    """Fetch parent comment or submission.

    Function gets parent comment of given permalink or submission if top level comment.

    Args:
         link:  Permalink to get parent of.

    Returns:
        Parent comment link or default string.
    """
    permalink = 'https://www.reddit.com' + link  # Permalink is missing prefix
    parent = 'Parent comment not found.'  # Default string
    try:
        comment = reddit.comment(url=permalink)  # Fetch comment by URL
        parent = 'https://np.reddit.com' + comment.submission.permalink
    except praw.exceptions as err:
        logger.exception(f'get_parent: {err}')
    return parent

# If run directly, instead of imported as a module, run main():
if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

# package version
__version__="2.2.1"
<commit_msg>Version push to 2.3.0<commit_after>#!/usr/bin/env python

# package version
__version__="2.3.0"
<|endoftext|>


<commit_before>#!/usr/bin/env python
#-*- coding: utf-8 -*-

# rss is the interface to rss parsing,
# processing and storing.
#
# @author Jin Yuan
# @email jinyuan@baidu.com
# @created Jul. 20, 2013

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append('..')

import database
import random
import rss_parser
from iamge_processor import image_helper
from image_processor import thumbnail
from data_processor import transcoder
from data_processor import tts_provider

from administration.config import LANGUAGES
from administration.config import THUMBNAIL_SIZE

def _value_added_process(entries=None, language=None):
    """
    add more value to an entry
    tts, transcode, images, redis_entry_expiration, database_entry_expiration
    """
    if not entries:
        return None
    if not language or language not in LANGUAGES:
        raise Exception("ERROR: language not found or not supported!")

    entries_new = []
    for entry in entries:
        # get a random int from 100 million possibilities
        try:
            rand = random.randint(0, 100000000)
            transcoded_relative_path = '%s_%s_%s_%i' % (entry['language'], entry['feed_id'], entry['updated'], rand)
            # high chances transcoder cannot work properly
            entry['transcoded'] = transcoder.transcode(entry['language'], entry['title'], entry['link'], transcoded_relative_path)

            # find big images
            big_images = image_helper.find_images(entry['transocded'])
            if big_images:
                entry['big_images'] = entry['big_images'] if entry.has_key('big_images') else []
                entry['big_images'].extend(entry['transcoded'])
                entry['big_images'] = list(set(entry['big_images']))

            # only for English, at present
            if entry['language'] == 'en':
                try:
                    tts_relative_path = '%s_%s_%s_%i' % (entry['language'], entry['feed_id'], entry['updated'], rand)
                    entry['mp3'] = tts_provider.google(entry['language'], entry['title'], tts_relative_path)
                except Exception as k:
                    print k, '... cannot generate TTS for %s' % entry['link']
                    entry['mp3'] = "None"

            entries_new.append(entry)
        except Exception as k:
            if k.startswith('ERROR'):
                print k
    except Exception as e:
        print str(e)
    added_entries.append((entry, REDIS_ENTRY_EXPIRATION))
    added_entries.append((entry, MONGODB_ENTRY_EXPIRATION))
    return entries_new

def update(feed_link=None, feed_id=None, feed_title=None, language=None, etag=None, modified=None):
    """
    update could be called
    1. from task procedure: all parameters included
    2. after an rss is added: all parameters included
    3. manually for testing purpose: feed_link, language
    """
    if not feed_link or not feed_link or not language:
        raise Exception(
            "ERROR: Method signature not well formed for %s!" % feed_link)
    if language not in LANGUAGES:
        raise Exception("ERROR: Language not supported for %s!" % feed_link)
    # parameters striped
    feed_link = feed_link.strip()
    language = language.strip()

    # parse rss reading from remote rss servers
    try:
        entries, feed_title_new, etag_new, modified_new = rss_parser.parse(
            feed_link, feed_id, feed_title, language, etag, modified)
    except Exception as k:
        # deal with exceptions
        if k.startswith('WARNING'):
            print k
        else:
            print k
            raise k

    # filter out existing entries in database
    # there are some possible exceptions -- yet let it be
    entries = database.dedup(entries, language)
    # and do tts, big_images, image as well as transcode.
    entries = _value_added_process(entries, language)

    # update new entries to database
    entries = database.update(entries, language)
    # and some data, like feed_title, etag and modified to database
    database.update_feed()

    # store in both database and memory
    if entries_new:
        added_entries = database.update(entries, language)
        print 'Nothing updated' if not added_entries else '    2/3 .. updated %i database items' % len(added_entries)
        if added_entries:
            updated_entries = memory.update(
                added_entries, language, category, feed_id)
            print '' '    3/3 .. updated memory'
            return updated_entries
    return None
<commit_msg>updated finding the biggest image segment<commit_after>#!/usr/bin/env python
#-*- coding: utf-8 -*-

# rss is the interface to rss parsing,
# processing and storing.
#
# @author Jin Yuan
# @email jinyuan@baidu.com
# @created Jul. 20, 2013

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append('..')

import database
import random
import rss_parser
from iamge_processor import image_helper
from image_processor import thumbnail
from data_processor import transcoder
from data_processor import tts_provider

from administration.config import LANGUAGES
from administration.config import THUMBNAIL_SIZE

def _value_added_process(entries=None, language=None):
    """
    add more value to an entry
    tts, transcode, images, redis_entry_expiration, database_entry_expiration
    """
    if not entries:
        return None
    if not language or language not in LANGUAGES:
        raise Exception("ERROR: language not found or not supported!")

    entries_new = []
    for entry in entries:
        # get a random int from 100 million possibilities
        try:
            rand = random.randint(0, 100000000)
            transcoded_relative_path = '%s_%s_%s_%i' % (entry['language'], entry['feed_id'], entry['updated'], rand)
            # high chances transcoder cannot work properly
            entry['transcoded'] = transcoder.transcode(entry['language'], entry['title'], entry['link'], transcoded_relative_path)

            # find big images
            big_images = image_helper.find_images(entry['transocded'])
            if big_images:
                entry['big_images'] = entry['big_images'] if entry.has_key('big_images') else []
                entry['big_images'].extend(entry['transcoded'])
                entry['big_images'] = list(set(entry['big_images']))

            # find biggest image
            if entry['big_images']:
                entry['image'] = image_helper.find_biggest_image(entry['big_images'])

            # tts only for English, at present
            if entry['language'] == 'en':
                try:
                    tts_relative_path = '%s_%s_%s_%i' % (entry['language'], entry['feed_id'], entry['updated'], rand)
                    entry['mp3'] = tts_provider.google(entry['language'], entry['title'], tts_relative_path)
                except Exception as k:
                    print k, '... cannot generate TTS for %s' % entry['link']
                    entry['mp3'] = "None"

            entries_new.append(entry)
        except Exception as k:
            if k.startswith('ERROR'):
                print k
    except Exception as e:
        print str(e)
    added_entries.append((entry, REDIS_ENTRY_EXPIRATION))
    added_entries.append((entry, MONGODB_ENTRY_EXPIRATION))
    return entries_new

def update(feed_link=None, feed_id=None, feed_title=None, language=None, etag=None, modified=None):
    """
    update could be called
    1. from task procedure: all parameters included
    2. after an rss is added: all parameters included
    3. manually for testing purpose: feed_link, language
    """
    if not feed_link or not feed_link or not language:
        raise Exception(
            "ERROR: Method signature not well formed for %s!" % feed_link)
    if language not in LANGUAGES:
        raise Exception("ERROR: Language not supported for %s!" % feed_link)
    # parameters striped
    feed_link = feed_link.strip()
    language = language.strip()

    # parse rss reading from remote rss servers
    try:
        entries, feed_title_new, etag_new, modified_new = rss_parser.parse(
            feed_link, feed_id, feed_title, language, etag, modified)
    except Exception as k:
        # deal with exceptions
        if k.startswith('WARNING'):
            print k
        else:
            print k
            raise k

    # filter out existing entries in database
    # there are some possible exceptions -- yet let it be
    entries = database.dedup(entries, language)
    # and do tts, big_images, image as well as transcode.
    entries = _value_added_process(entries, language)

    # update new entries to database
    entries = database.update(entries, language)
    # and some data, like feed_title, etag and modified to database
    database.update_feed()

    # store in both database and memory
    if entries_new:
        added_entries = database.update(entries, language)
        print 'Nothing updated' if not added_entries else '    2/3 .. updated %i database items' % len(added_entries)
        if added_entries:
            updated_entries = memory.update(
                added_entries, language, category, feed_id)
            print '' '    3/3 .. updated memory'
            return updated_entries
    return None
<|endoftext|>


<commit_before>"""Decorates command lines with docker arguments."""
from dodo_commands.framework.args_tree import ArgsTreeNode
from dodo_commands.framework.config import merge_into_config
from dodo_commands.framework import Dodo, CommandError
from fnmatch import fnmatch
from plumbum import local
import os

def _is_tuple(x):
    return hasattr(x, "__len__") and not isinstance(x, type(str()))

class Decorator:  # noqa
    @classmethod
    def _add_docker_volume_list(cls, docker_config, root_node):
        volume_list = docker_config.get('volume_list', [])
        volume_map = docker_config.get('volume_map', {})

        volume_map_strict = docker_config.get('volume_map_strict', {})
        for key, val in volume_map_strict.items():
            if not os.path.exists(key):
                raise CommandError(
                    "Path in volume_map_strict not found: %s" % key)
            volume_map[key] = val

        options = (
            ['--volume=%s:%s' % (x, x) for x in volume_list] +
            ['--volume=%s:%s' % key_val for key_val in volume_map.items()])
        for x in options:
            root_node['volume'].append(x)

    @classmethod
    def _add_docker_publish_list(cls, docker_config, root_node):
        publish_list = docker_config.get('publish_list', [])
        publish_map = docker_config.get('publish_map', {})
        options = (
            ['--publish=%s:%s' % (x, x) for x in publish_list] +
            ['--publish=%s:%s' % key_val for key_val in publish_map.items()])
        for x in options:
            root_node['publish'].append(x)

    @classmethod
    def _add_docker_volumes_from_list(cls, docker_config, root_node):
        volumes_from_list = docker_config.get('volumes_from_list', [])
        options = (['--volumes-from=%s' % x for x in volumes_from_list])
        for x in options:
            root_node['volumes-from'].append(x)

    @classmethod
    def _add_docker_variable_list(cls, docker_config, root_node):
        variable_list = docker_config.get('variable_list', [])
        variable_map = docker_config.get('variable_map', {})
        options = (
            ['--env=%s' % x for x in variable_list
             ] + ['--env=%s=%s' % key_val for key_val in variable_map.items()])
        for x in options:
            root_node['env'].append(x)

    @classmethod
    def _add_linked_container_list(cls, docker_config, root_node):
        options = ['--link=%s' % x for x in docker_config.get('link_list', [])]
        for x in options:
            root_node['link'].append(x)

    @classmethod
    def merged_options(cls, get_config, command_name):
        merged = {}
        for patterns, docker_config in get_config('/DOCKER/options',
                                                  {}).items():
            should_merge = False
            for pattern in (patterns if _is_tuple(patterns) else [patterns]):
                if pattern.startswith('!'):
                    should_merge = should_merge and not fnmatch(
                        command_name, pattern[1:])
                else:
                    should_merge = should_merge or fnmatch(
                        command_name, pattern)
            if should_merge:
                merge_into_config(merged, docker_config)
        return merged

    @classmethod
    def docker_node(cls, get_config, command_name, cwd):
        merged = cls.merged_options(get_config, command_name)

        docker_node = ArgsTreeNode("docker", args=['docker', 'run'])
        docker_node.add_child(ArgsTreeNode("name"))
        docker_node.add_child(ArgsTreeNode("basic"))
        docker_node.add_child(ArgsTreeNode("link"))
        docker_node.add_child(ArgsTreeNode("publish"))
        docker_node.add_child(ArgsTreeNode("volume", is_horizontal=False))
        docker_node.add_child(ArgsTreeNode("volumes-from"))
        docker_node.add_child(ArgsTreeNode("env", is_horizontal=False))
        docker_node.add_child(ArgsTreeNode("other"))
        docker_node.add_child(ArgsTreeNode("workdir"))
        docker_node.add_child(ArgsTreeNode("image"))

        cls._add_linked_container_list(merged, docker_node)
        cls._add_docker_publish_list(merged, docker_node)
        cls._add_docker_volume_list(merged, docker_node)
        cls._add_docker_volumes_from_list(merged, docker_node)
        cls._add_docker_variable_list(merged, docker_node)

        if merged.get('rm', True):
            docker_node['basic'].append('--rm')

        if merged.get('is_interactive', True):
            docker_node['basic'].append('--interactive')
            docker_node['basic'].append('--tty')

        if merged.get('is_daemon', False):
            docker_node['basic'].append('-d')

        name = merged.get('name', command_name)
        if name:
            docker_node["name"].append("--name=%s" % name)

        for key_val in get_config('/ENVIRONMENT/variable_map', {}).items():
            docker_node['env'].append("--env=%s=%s" % key_val)

        for x in merged.get('extra_options', []):
            docker_node['other'].append(x)

        if cwd:
            docker_node['workdir'].append('--workdir=' + cwd)

        image = merged.get('image')
        if not image:
            raise CommandError(
                "No docker image found in /DOCKER/options for command %s" %
                command_name)
        docker_node['image'].append(image)

        return docker_node, image, name

    def add_arguments(self, parser):  # noqa
        parser.add_argument(
            '--kill-existing',
            action='store_true',
            help="Kill and remove existing docker container with the same name"
        )

    def _kill_existing_container(self, name):
        docker = local['docker']
        live_containers = docker("ps", "--format={{.Names}}",
                                 "--filter=name=^/%s$" % name)
        if len(live_containers):
            docker("stop", name)

        exited_containers = docker("ps", "-a", "--format={{.Names}}",
                                   "--filter=name=^/%s$" % name)
        if len(exited_containers):
            docker("rm", name)

    def modify_args(self, dodo_args, root_node, cwd):  # noqa
        docker_node, _, docker_name = self.docker_node(Dodo.get_config,
                                                       Dodo.command_name, cwd)

        if getattr(dodo_args, 'kill_existing', False):
            self._kill_existing_container(docker_name)

        docker_node.add_child(root_node)
        return docker_node, local.cwd
<commit_msg>Support option workdir in /DOCKER/options<commit_after>"""Decorates command lines with docker arguments."""
from dodo_commands.framework.args_tree import ArgsTreeNode
from dodo_commands.framework.config import merge_into_config
from dodo_commands.framework import Dodo, CommandError
from fnmatch import fnmatch
from plumbum import local
import os

def _is_tuple(x):
    return hasattr(x, "__len__") and not isinstance(x, type(str()))

class Decorator:  # noqa
    @classmethod
    def _add_docker_volume_list(cls, docker_config, root_node):
        volume_list = docker_config.get('volume_list', [])
        volume_map = docker_config.get('volume_map', {})

        volume_map_strict = docker_config.get('volume_map_strict', {})
        for key, val in volume_map_strict.items():
            if not os.path.exists(key):
                raise CommandError("Path in volume_map_strict not found: %s" %
                                   key)
            volume_map[key] = val

        options = (
            ['--volume=%s:%s' % (x, x) for x in volume_list] +
            ['--volume=%s:%s' % key_val for key_val in volume_map.items()])
        for x in options:
            root_node['volume'].append(x)

    @classmethod
    def _add_docker_publish_list(cls, docker_config, root_node):
        publish_list = docker_config.get('publish_list', [])
        publish_map = docker_config.get('publish_map', {})
        options = (
            ['--publish=%s:%s' % (x, x) for x in publish_list] +
            ['--publish=%s:%s' % key_val for key_val in publish_map.items()])
        for x in options:
            root_node['publish'].append(x)

    @classmethod
    def _add_docker_volumes_from_list(cls, docker_config, root_node):
        volumes_from_list = docker_config.get('volumes_from_list', [])
        options = (['--volumes-from=%s' % x for x in volumes_from_list])
        for x in options:
            root_node['volumes-from'].append(x)

    @classmethod
    def _add_docker_variable_list(cls, docker_config, root_node):
        variable_list = docker_config.get('variable_list', [])
        variable_map = docker_config.get('variable_map', {})
        options = (
            ['--env=%s' % x for x in variable_list] +
            ['--env=%s=%s' % key_val for key_val in variable_map.items()])
        for x in options:
            root_node['env'].append(x)

    @classmethod
    def _add_linked_container_list(cls, docker_config, root_node):
        options = ['--link=%s' % x for x in docker_config.get('link_list', [])]
        for x in options:
            root_node['link'].append(x)

    @classmethod
    def merged_options(cls, get_config, command_name):
        merged = {}
        for patterns, docker_config in get_config('/DOCKER/options',
                                                  {}).items():
            should_merge = False
            for pattern in (patterns if _is_tuple(patterns) else [patterns]):
                if pattern.startswith('!'):
                    should_merge = should_merge and not fnmatch(
                        command_name, pattern[1:])
                else:
                    should_merge = should_merge or fnmatch(
                        command_name, pattern)
            if should_merge:
                merge_into_config(merged, docker_config)
        return merged

    @classmethod
    def docker_node(cls, get_config, command_name, cwd):
        merged = cls.merged_options(get_config, command_name)

        docker_node = ArgsTreeNode("docker", args=['docker', 'run'])
        docker_node.add_child(ArgsTreeNode("name"))
        docker_node.add_child(ArgsTreeNode("basic"))
        docker_node.add_child(ArgsTreeNode("link"))
        docker_node.add_child(ArgsTreeNode("publish"))
        docker_node.add_child(ArgsTreeNode("volume", is_horizontal=False))
        docker_node.add_child(ArgsTreeNode("volumes-from"))
        docker_node.add_child(ArgsTreeNode("env", is_horizontal=False))
        docker_node.add_child(ArgsTreeNode("other"))
        docker_node.add_child(ArgsTreeNode("workdir"))
        docker_node.add_child(ArgsTreeNode("image"))

        cls._add_linked_container_list(merged, docker_node)
        cls._add_docker_publish_list(merged, docker_node)
        cls._add_docker_volume_list(merged, docker_node)
        cls._add_docker_volumes_from_list(merged, docker_node)
        cls._add_docker_variable_list(merged, docker_node)

        if merged.get('rm', True):
            docker_node['basic'].append('--rm')

        if merged.get('is_interactive', True):
            docker_node['basic'].append('--interactive')
            docker_node['basic'].append('--tty')

        if merged.get('is_daemon', False):
            docker_node['basic'].append('-d')

        name = merged.get('name', command_name)
        if name:
            docker_node["name"].append("--name=%s" % name)

        for key_val in get_config('/ENVIRONMENT/variable_map', {}).items():
            docker_node['env'].append("--env=%s=%s" % key_val)

        for x in merged.get('extra_options', []):
            docker_node['other'].append(x)

        if cwd:
            docker_node['workdir'].append('--workdir=' + cwd)
        elif merged.get('workdir', ''):
            docker_node['workdir'].append('--workdir=' + merged.get('workdir'))

        image = merged.get('image')
        if not image:
            raise CommandError(
                "No docker image found in /DOCKER/options for command %s" %
                command_name)
        docker_node['image'].append(image)

        return docker_node, image, name

    def add_arguments(self, parser):  # noqa
        parser.add_argument(
            '--kill-existing',
            action='store_true',
            help="Kill and remove existing docker container with the same name"
        )

    def _kill_existing_container(self, name):
        docker = local['docker']
        live_containers = docker("ps", "--format={{.Names}}",
                                 "--filter=name=^/%s$" % name)
        if len(live_containers):
            docker("stop", name)

        exited_containers = docker("ps", "-a", "--format={{.Names}}",
                                   "--filter=name=^/%s$" % name)
        if len(exited_containers):
            docker("rm", name)

    def modify_args(self, dodo_args, root_node, cwd):  # noqa
        docker_node, _, docker_name = self.docker_node(Dodo.get_config,
                                                       Dodo.command_name, cwd)

        if getattr(dodo_args, 'kill_existing', False):
            self._kill_existing_container(docker_name)

        docker_node.add_child(root_node)
        return docker_node, local.cwd
<|endoftext|>


<commit_before><commit_msg>Don't delete temporary directory unless kill_buckd succeeds<commit_after><|endoftext|>


<commit_before>__author__ = 'mrichardson'

from math import fmod, sin, pi, cos, asin

from atmPy.constants import a2r, r2a

__julian = {"day": 0., "cent": 0.}

class solar(object):

    def __init__(self, ltime):

        julian = solar.juliandates(ltime)
        self.__jday = julian["day"]
        self.__jcent = julian["cent"]

        self.lon = 0
        self.lat = 0

    sinrad = lambda x: sin(a2r(x))
    cosrad = lambda x: cos(a2r(x))

    def juliandates(self, ltime):
        """
        Calculate a Julian date for a given local time

        Parameters
        ----------
        ltime:      float
                    Local time calculated as seconds since Jan 1 1904
        Returns
        --------
        dictionary
            Returns a dictionary of two floats containing julian day and century.
        """
        # Julian day is the continuous count of days since the beginning of the Julian period.
        self.__jday = ltime/(3600*24)+1462+2415018.5
        self.__jcent = (self.__jday-2451545)/36525
        return None

    def __oblelip(self):

        return ((21.448-self.__jcent*(self.__jcent*
                                     (0.00059-(self.__jcent*0.001813))+46.815))/60+26)/60+23

    def __gemeanlon(self):
        return fmod((self.__jcent*0.0003032+36000.76983)*self.__jcent+280.46646, 360)

    def __meananom(self):
        return self.__jcent*(self.__jcent*0.0001537-35999.05029)+357.52911

    def __eartheccen(self):
        return self.__jcent*(self.__jcent*1.267e-7+4.2037e-5)-0.016708634

    def __centsun(self):

        f = lambda x: sin(a2r(x))

        a = f(3)*0.000289
        b = f(2)*(0.019993-self.__jcent*0.000101)
        c = f(1)*(self.__jcent*(self.__jcent**1.45e-5+0.004817)-1.914602)

        return a+b+c

    def __oblcorr(self):
        return self.cosrad(self.__jcent*1934.136-125.04)*0.00256+self.__oblelip()

    def __truelon(self):
        return self.__gemeanlon() + self.__centsun()

    def __app(self):
        a = self.__truelon()-0.00569
        a -= self.sinrad(self.__jcent*1934.136-125.04)*0.00478
        return a

    def __declang(self):
        return r2a(asin(self.sinrad(self.__oblcorr())*self.sinrad(self.__app())))

    def __eq_time(self):
        return None
<commit_msg>added sun position as function of time, lon and lat<commit_after>__author__ = 'mrichardson, Hagen Telg'

from math import fmod, sin, pi, cos, asin

from atmPy.constants import a2r, r2a

import numpy as np
import pandas as pd
import ephem

__julian = {"day": 0., "cent": 0.}

class solar(object):

    def __init__(self, ltime):

        julian = solar.juliandates(ltime)
        self.__jday = julian["day"]
        self.__jcent = julian["cent"]

        self.lon = 0
        self.lat = 0

    sinrad = lambda x: sin(a2r(x))
    cosrad = lambda x: cos(a2r(x))

    def juliandates(self, ltime):
        """
        Calculate a Julian date for a given local time

        Parameters
        ----------
        ltime:      float
                    Local time calculated as seconds since Jan 1 1904
        Returns
        --------
        dictionary
            Returns a dictionary of two floats containing julian day and century.
        """
        # Julian day is the continuous count of days since the beginning of the Julian period.
        self.__jday = ltime/(3600*24)+1462+2415018.5
        self.__jcent = (self.__jday-2451545)/36525
        return None

    def __oblelip(self):

        return ((21.448-self.__jcent*(self.__jcent*
                                     (0.00059-(self.__jcent*0.001813))+46.815))/60+26)/60+23

    def __gemeanlon(self):
        return fmod((self.__jcent*0.0003032+36000.76983)*self.__jcent+280.46646, 360)

    def __meananom(self):
        return self.__jcent*(self.__jcent*0.0001537-35999.05029)+357.52911

    def __eartheccen(self):
        return self.__jcent*(self.__jcent*1.267e-7+4.2037e-5)-0.016708634

    def __centsun(self):

        f = lambda x: sin(a2r(x))

        a = f(3)*0.000289
        b = f(2)*(0.019993-self.__jcent*0.000101)
        c = f(1)*(self.__jcent*(self.__jcent**1.45e-5+0.004817)-1.914602)

        return a+b+c

    def __oblcorr(self):
        return self.cosrad(self.__jcent*1934.136-125.04)*0.00256+self.__oblelip()

    def __truelon(self):
        return self.__gemeanlon() + self.__centsun()

    def __app(self):
        a = self.__truelon()-0.00569
        a -= self.sinrad(self.__jcent*1934.136-125.04)*0.00478
        return a

    def __declang(self):
        return r2a(asin(self.sinrad(self.__oblcorr())*self.sinrad(self.__app())))

    def __eq_time(self):
        return None

def get_sun_position(lat, lon, datetime_UTC, elevation=0):
    """returns elevation and azimuth angle of the sun
    Arguments:
    ----------
    lat, lon: float
        latitude and longitude of the observer (e.g. Denver, lat = 39.7392, lon = -104.9903)
    datetime_UTC: datetime instance or strint ('2015/7/6 19:00:00')
        time of interestes in UTC
    elevation: float, optional.
        elevation of observer.

    Returns
    -------
    tuple of two floats
        elevation and azimuth angle in radians.
    """
    obs = ephem.Observer()
    obs.lat = lat
    obs.long = lon
    obs.elevation = elevation
    # obs.date = '2015/7/6 19:00:00'
    obs.date = datetime_UTC  # datetime.datetime.now() + datetime.timedelta(hours = 6)
    #     print(obs)
    sun = ephem.Sun()
    sun.compute(obs)
    return sun.alt, sun.az

def get_sun_position_TS(timeseries):
    """Returns the position, polar and azimuth angle, of the sun in the sky for a given time and location.

    Arguments
    ---------
    timeseries: pandas.DataFrame instance with the index being of type datetime (e.g. atmPy.timeseries).
        This is typically a housekeeping/telemetry timeseries. It must contain the columns
        Lat, Lon, and Height

    Returns
    -------
    pandas.DataFram with two collums for the elevation and azimuth angle
    Furthermore the timeseries gets two new collumns with the two angles
    """
    lat = timeseries.data.Lat.values.astype(str)
    lon = timeseries.data.Lon.values.astype(str)
    alti = timeseries.data.Height.values
    t = timeseries.data.Lat.index
    sunpos = np.zeros((lat.shape[0], 2))
    # sunpos = np.zeros((2,2))
    for e, i in enumerate(lat):
        if 0 == 1:
            break
        sunpos[e] = get_sun_position(lat[e], lon[e], t[e], elevation=alti[e])
    #     return sunpos
    timeseries.data['Solar_position_elevation'] = pd.Series(sunpos[:, 0], index=timeseries.data.index)
    timeseries.data['Solar_position_azimuth'] = pd.Series(sunpos[:, 1], index=timeseries.data.index)
    return pd.DataFrame(sunpos, columns=['elevation', 'azimuth'], index=timeseries.data.index)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# SubstructureLib.py
#   Copyright (C) 2006 CCLRC, Graeme Winter
#
#   This code is distributed under the terms and conditions of the
#   CCP4 Program Suite Licence Agreement as a CCP4 Library.
#   A copy of the CCP4 licence can be obtained by writing to the
#   CCP4 Secretary, Daresbury Laboratory, Warrington WA4 4AD, UK.
#
# 16th November 2006
#
# A library of things pertaining to substructure manipulation:
#
# .pdb file -> fractional coordinates
# invert hand

import sys
import os

if not os.path.join(os.environ['XIA2CORE_ROOT'], 'Python') in sys.path:
    sys.path.append(os.path.join(os.environ['XIA2CORE_ROOT'], 'Python'))

if not os.environ['DPA_ROOT'] in sys.path:
    sys.path.append(os.environ['DPA_ROOT'])

def _dot(a, b):
    '''Compute a.b. For converting with the aid of a SCALEN record in a
    pdb file...'''

    if not len(a) == len(b):
        raise RuntimeError, 'different length vectors'

    result = 0.0

    for i in range(len(a)):
        result += a[i] * b[i]

    return result

def parse_pdb_sites_file(pdb_file):
    '''Parse a pdb file full of heavy atoms and transmogrify this into
    a form suitable for input to e.g. bp3 (with fractional coordinates
    and occupancies.)'''

    data = open(pdb_file, 'r').readlines()

    scales = { }

    results = []

    for d in data:
        if 'SCALE' in d[:5]:
            scale = map(float, d.split()[1:4])
            scales[int(d.split()[0].replace('SCALE', '')) - 1] = scale

    if not scales.has_key(0):
        raise RuntimeError, 'SCALE1 record missing'

    if not scales.has_key(1):
        raise RuntimeError, 'SCALE2 record missing'

    if not scales.has_key(2):
        raise RuntimeError, 'SCALE3 record missing'

    for d in data:
        if 'ATOM' in d[:4]:
            cartesian = map(float, d.split()[5:8])
            occ = float(d.split()[8])
            atom = d.split()[2].lower()

            fractional = [_dot(scales[i], cartesian) for i in range(3)]

            results.append({'atom':atom,
                            'occupancy':occ,
                            'cartesian':cartesian,
                            'fractional':fractional})

    return results

if __name__ == '__main__':
    if len(sys.argv) < 2:
        raise RuntimeError, '%s pdb_file' % sys.argv[0]

    print parse_pdb_sites_file(sys.argv[1])

<commit_msg>This now gathers more information from the pdb file from e.g. the CRYST1 record, and also returns a dictionary containing all of the information.<commit_after>#!/usr/bin/env python
# SubstructureLib.py
#   Copyright (C) 2006 CCLRC, Graeme Winter
#
#   This code is distributed under the terms and conditions of the
#   CCP4 Program Suite Licence Agreement as a CCP4 Library.
#   A copy of the CCP4 licence can be obtained by writing to the
#   CCP4 Secretary, Daresbury Laboratory, Warrington WA4 4AD, UK.
#
# 16th November 2006
#
# A library of things pertaining to substructure manipulation:
#
# .pdb file -> fractional coordinates
# invert hand

import sys
import os

if not os.path.join(os.environ['XIA2CORE_ROOT'], 'Python') in sys.path:
    sys.path.append(os.path.join(os.environ['XIA2CORE_ROOT'], 'Python'))

if not os.environ['DPA_ROOT'] in sys.path:
    sys.path.append(os.environ['DPA_ROOT'])

def _dot(a, b):
    '''Compute a.b. For converting with the aid of a SCALEN record in a
    pdb file...'''

    if not len(a) == len(b):
        raise RuntimeError, 'different length vectors'

    result = 0.0

    for i in range(len(a)):
        result += a[i] * b[i]

    return result

def parse_pdb_sites_file(pdb_file):
    '''Parse a pdb file full of heavy atoms and transmogrify this into
    a form suitable for input to e.g. bp3 (with fractional coordinates
    and occupancies.)'''

    data = open(pdb_file, 'r').readlines()

    scales = { }

    sites = []
    cell = ()
    symm = ''

    for d in data:
        if 'SCALE' in d[:5]:
            scale = map(float, d.split()[1:4])
            scales[int(d.split()[0].replace('SCALE', '')) - 1] = scale

        # need to store this to handle the inversion...

        if 'CRYST1' in d[:6]:
            cell = tuple(map(float, d.split()[1:7]))
            symm = d[55:].strip()

    if not scales.has_key(0):
        raise RuntimeError, 'SCALE1 record missing'

    if not scales.has_key(1):
        raise RuntimeError, 'SCALE2 record missing'

    if not scales.has_key(2):
        raise RuntimeError, 'SCALE3 record missing'

    for d in data:
        if 'ATOM' in d[:4]:
            cartesian = map(float, d.split()[5:8])
            occ = float(d.split()[8])
            atom = d.split()[2].lower()

            fractional = [_dot(scales[i], cartesian) for i in range(3)]

            sites.append({'atom':atom,
                          'occupancy':occ,
                          'cartesian':cartesian,
                          'fractional':fractional})

    results = { }
    results['sites'] = sites
    results['cell'] = cell
    results['spacegroup'] = symm
    results['scale'] = scales

    return results

def invert_hand(sites):
    '''Invert the hand (and perhaps the spacegroup) of substructure sites.'''

if __name__ == '__main__':
    if len(sys.argv) < 2:
        pdb = os.path.join(os.environ['SS_ROOT'],
                           'Data', 'Test', 'Sites', 'hyss-sites.pdb')
    else:
        pdb = sys.argv[1]

    print parse_pdb_sites_file(pdb)

<|endoftext|>


<commit_before>from datetime import datetime

import six
import pytz
import logging

logger = logging.getLogger()

def timestamp():
    return pytz.utc.localize(datetime.utcnow()).isoformat()

def copy_to_unicode(element):
    """ used to transform the lxml version of unicode to a
    standard version of unicode that can be pickalable -
    necessary for linting """

    if isinstance(element, dict):
        for key, val in element.items():
            element[key] = copy_to_unicode(val)
    elif isinstance(element, list):
        for idx, item in enumerate(element):
            element[idx] = copy_to_unicode(item)
    else:
        try:
            # A dirty way to convert to unicode in python 2 + 3.3+
            element = u''.join(element)
        except TypeError:
            pass
    return element

def stamp_from_raw(raw_doc, **kwargs):
    kwargs['normalizeFinished'] = timestamp()
    stamps = raw_doc['timestamps']
    stamps.update(kwargs)
    return stamps

def format_date_with_slashes(date):
    return date.strftime('%m/%d/%Y')

def json_without_bytes(jobj):
    """
        An ugly hack.

        Before we treat a structure as JSON, ensure that bytes are decoded to str.
    """
    # Create a JSON-compatible copy of the attributes for validation
    jobj = jobj.copy()
    for k, v in jobj.items():
        if isinstance(v, six.binary_type):
            jobj[k] = v.decode('utf8')
    return jobj
<commit_msg>Move try_n_times to util.py<commit_after>from datetime import datetime

import six
import pytz
import time
import logging

logger = logging.getLogger()

def timestamp():
    return pytz.utc.localize(datetime.utcnow()).isoformat()

def copy_to_unicode(element):
    """ used to transform the lxml version of unicode to a
    standard version of unicode that can be pickalable -
    necessary for linting """

    if isinstance(element, dict):
        for key, val in element.items():
            element[key] = copy_to_unicode(val)
    elif isinstance(element, list):
        for idx, item in enumerate(element):
            element[idx] = copy_to_unicode(item)
    else:
        try:
            # A dirty way to convert to unicode in python 2 + 3.3+
            element = u''.join(element)
        except TypeError:
            pass
    return element

def stamp_from_raw(raw_doc, **kwargs):
    kwargs['normalizeFinished'] = timestamp()
    stamps = raw_doc['timestamps']
    stamps.update(kwargs)
    return stamps

def format_date_with_slashes(date):
    return date.strftime('%m/%d/%Y')

def json_without_bytes(jobj):
    """
        An ugly hack.

        Before we treat a structure as JSON, ensure that bytes are decoded to str.
    """
    # Create a JSON-compatible copy of the attributes for validation
    jobj = jobj.copy()
    for k, v in jobj.items():
        if isinstance(v, six.binary_type):
            jobj[k] = v.decode('utf8')
    return jobj

def try_n_times(n, action, *args, **kwargs):
    for _ in xrange(n):
        try:
            return action(*args, **kwargs)
        except Exception as e:
            logger.exception(e)
            time.sleep(15)
    if e:
        raise e
<|endoftext|>


<commit_before># import chainer
import glob
import os
import random

import chainer
import numpy as np
import skimage
from skimage import io
from skimage.transform import resize as imresize
from .basedataset import BaseLabeledImageDataset

#class YoloPreprocessedDataset(chainer.dataset.DatasetMixin):
#    def __init__(self, dirs=('JPEGImages/', 'labels/'), root='data/',
#                 resize=224, tags='voc.names', random=True):
#        self.root = root
#        self.dirs = dirs
#        self.image_dir = self.root + self.dirs[0]
#        self.label_dir = self.root + self.dirs[1]
#        self.paths = [os.path.split(f)[-1] for f in glob.glob(
#            self.image_dir + '*') if ('png'in f or 'jpg' in f)]
#        self.tags_path = self.root + tags
#        if isinstance(resize, int):
#            resize = [resize]
#        self.crop_size = resize
#        self.random = random
#        self.init_tag()
#
#    def __len__(self):
#        return len(self.paths)
#
#    def init_tag(self):
#        with open(self.tags_path, 'r') as f:
#            self.tags = [item.strip() for item in f.readlines()]
#
#    def convert2tuple(self, line):
#        line = line.strip().split()
#        line[0] = int(line[0])
#        line[1] = float(line[1])
#        line[2] = float(line[2])
#        line[3] = float(line[3])
#        line[4] = float(line[4])
#        return tuple(line)
#
#    def label_reader(self, labelfile):
#        with open(labelfile, 'r') as f:
#            lines = [self.convert2tuple(line) for line in f.readlines()]
#        ground_truths = []
#        for item in lines:
#            one_hot_label = np.zeros(len(self.tags))
#            one_hot_label[item[0]] = 1
#            ground_truths.append({
#                'x': item[1],
#                'y': item[2],
#                'w': item[3],
#                'h': item[4],
#                'label': item[0],
#                'one_hot_label': one_hot_label
#            })
#        return ground_truths
#
#    def get_example(self, i):
#        resize = int(*random.sample(self.crop_size, 1))
#        crop_size = (resize, resize)
#        imagefile = self.image_dir + self.paths[i]
#        labelfile = self.label_dir + self.paths[i].split('.')[0] + '.txt'
#        image = io.imread(imagefile)
#        labels = self.label_reader(labelfile)
#
#        image = imresize(image, crop_size, mode='reflect')
#        image = skimage.img_as_float(image)
#        return image.transpose(2, 0, 1), labels

class YoloPreprocessedDataset(BaseLabeledImageDataset):
#    def __init__(self, pairs, label_dict, resize=224):
#        super().__init__(pairs, label_dict)
#        if isinstance(self.resize, int):
#            resize = [resize]
#        self.crop_size = resize
#
#        self.root = root
#        self.dirs = dirs
#        self.image_dir = self.root + self.dirs[0]
#        self.label_dir = self.root + self.dirs[1]
#        self.paths = [os.path.split(f)[-1] for f in glob.glob(
#            self.image_dir + '*') if ('png'in f or 'jpg' in f)]
#        self.tags_path = self.root + tags
#        self.random = random
#        self.init_tag()

    def get_example(self, i):
        if isinstance(self.resize, int):
            resize = [resize]
            self.crop_size = resize
        else:
            self.crop_size = self.resize

        resize = int(*random.sample(self.crop_size, 1))
        crop_size = (resize, resize)
        full_path, label = self._pairs[i]
        print('full_path: ', full_path)
        print('label_dict: ', self.label_dict)
        image = io.imread(full_path)
        labels = self.label_reader(label)
#
#        imagefile = self.image_dir + self.paths[i]
#        labelfile = self.label_dir + self.paths[i].split('.')[0] + '.txt'
#
        image = imresize(image, crop_size, mode='reflect')
        image = skimage.img_as_float(image)
        return image.transpose(2, 0, 1), labels

    def init_tag(self):
        with open(self.tags_path, 'r') as f:
            self.tags = [item.strip() for item in f.readlines()]

    def convert2tuple(self, line):
        line = line.strip().split()
        line[0] = int(line[0])
        line[1] = float(line[1])
        line[2] = float(line[2])
        line[3] = float(line[3])
        line[4] = float(line[4])
        return tuple(line)

    def label_reader(self, labelfile):
        with open(labelfile, 'r') as f:
            lines = [self.convert2tuple(line) for line in f.readlines()]
        ground_truths = []
        for item in lines:
            one_hot_label = np.zeros(len(self.label_dict))
            one_hot_label[item[0]] = 1
            ground_truths.append({
                'x': item[1],
                'y': item[2],
                'w': item[3],
                'h': item[4],
                'label': item[0],
                'one_hot_label': one_hot_label
            })
        return ground_truths
<commit_msg>[remove]: useless comment.<commit_after># import chainer
import glob
import os
import random

import chainer
import numpy as np
import skimage
from skimage import io
from skimage.transform import resize as imresize
from .basedataset import BaseLabeledImageDataset

class YoloPreprocessedDataset(BaseLabeledImageDataset):
    def get_example(self, i):
        if isinstance(self.resize, int):
            resize = [resize]
            self.crop_size = resize
        else:
            self.crop_size = self.resize

        resize = int(*random.sample(self.crop_size, 1))
        crop_size = (resize, resize)
        full_path, label = self._pairs[i]
        image = io.imread(full_path)
        labels = self.label_reader(label)
        image = imresize(image, crop_size, mode='reflect')
        image = skimage.img_as_float(image)
        return image.transpose(2, 0, 1), labels

    def convert2tuple(self, line):
        line = line.strip().split()
        line[0] = int(line[0])
        line[1] = float(line[1])
        line[2] = float(line[2])
        line[3] = float(line[3])
        line[4] = float(line[4])
        return tuple(line)

    def label_reader(self, labelfile):
        with open(labelfile, 'r') as f:
            lines = [self.convert2tuple(line) for line in f.readlines()]
        ground_truths = []
        for item in lines:
            one_hot_label = np.zeros(len(self.label_dict))
            one_hot_label[item[0]] = 1
            ground_truths.append({
                'x': item[1],
                'y': item[2],
                'w': item[3],
                'h': item[4],
                'label': item[0],
                'one_hot_label': one_hot_label
            })
        return ground_truths
<|endoftext|>