<commit_before># -*- encoding: utf-8 -*-
"""\
Some benchmarks running against QR Code generators
"""
import sys
import timeit
import segno
try:
    import qrcode
    from qrcode import QRCode, ERROR_CORRECT_M
    from qrcode.image.svg import SvgImage, SvgPathImage
except ImportError:
    QRCode = None
try:
    from qrcodegen import QrCode
except ImportError:
    QrCode = None
try:
    import pyqrcode
except ImportError:
    pyqrcode = None

if QRCode:
    def create_qrcode(data='QR Code Symbol'):
        """qrcode create"""
        qr = QRCode(error_correction=ERROR_CORRECT_M)
        qr.add_data(data)
        qr.make()

    def svg_qrcode_path(data='QR Code Symbol'):
        """qrcode SVG path"""
        qrcode.make(data, error_correction=qrcode.ERROR_CORRECT_M,
                    box_size=10,
                    image_factory=SvgPathImage) \
            .save('out/qrcode_path_%s.svg' % data)

    def svg_qrcode_rects(data='QR Code Symbol'):
        """qrcode SVG rects"""
        qrcode.make(data, error_correction=qrcode.ERROR_CORRECT_M,
                    box_size=10,
                    image_factory=SvgImage) \
            .save('out/qrcode_rects_%s.svg' % data)

    def png_qrcode(data='QR Code Symbol'):
        """qrcode PNG"""
        qrcode.make(data, error_correction=qrcode.ERROR_CORRECT_M,
                    box_size=10) \
            .save('out/qrcode_%s.png' % data)

if pyqrcode:
    def create_pyqrcode(data='QR Code Symbol'):
        """PyQRCode create"""
        pyqrcode.create(data, error='m')

    def svg_pyqrcode(data='QR Code Symbol'):
        """PyQRCode SVG"""
        pyqrcode.create(data, error='m').svg('out/pyqrcode_%s.svg' % data, scale=10)

    def png_pyqrcode(data='QR Code Symbol'):
        """PyQRCode PNG"""
        pyqrcode.create(data, error='m').png('out/pyqrcode_%s.png' % data, scale=10)

if QrCode:
    def create_qrcodegen(data='QR Code Symbol'):
        """qrcodegen create"""
        QrCode.encode_text(data, QrCode.Ecc.MEDIUM)

    def svg_qrcodegen(data='QR Code Symbol'):
        """qrcodegen SVG"""
        with open('out/qrcodegen_%s.svg' % data, 'wt') as f:
            f.write(QrCode.encode_text(data, QrCode.Ecc.MEDIUM).to_svg_str(border=4))

def create_segno(data='QR Code Symbol'):
    """Segno create"""
    segno.make_qr(data, error='m')

def svg_segno(data='QR Code Symbol'):
    """Segno SVG"""
    segno.make_qr(data, error='m').save('out/segno_%s.svg' % data, scale=10)

def png_segno(data='QR Code Symbol'):
    """Segno PNG"""
    segno.make_qr(data, error='m').save('out/segno_%s.png' % data, scale=10)

def run_create_tests(which=None, number=500):
    tests = ('create_segno', 'create_pyqrcode', 'create_qrcodegen',
             'create_qrcode')
    if which:
        tests = filter(lambda n: n[len('create_'):] in which, tests)
    _run_tests(tests, number)

def run_svg_tests(which=None, number=500):
    tests = ('svg_segno', 'svg_pyqrcode', 'svg_qrcodegen',
             'svg_qrcode_path', 'svg_qrcode_rects')

    if which:
        tests = filter(lambda n: n[len('svg_'):] in which, tests)
    _run_tests(tests, number)

def run_png_tests(which=None, number=500):
    tests = ('png_segno', 'png_pyqrcode', 'png_qrcode')
    if which:
        tests = filter(lambda n: n[len('png_'):] in which, tests)
    _run_tests(tests, number)

def _run_tests(tests, number):
    # Code taken from <https://genshi.edgewall.org/browser/trunk/examples/bench/bigtable.py>
    # Author: Jonas Borgström <jonas@edgewall.com>
    # License: BSD (I'd think since Genshi uses BSD as well)
    for test in [t for t in tests if hasattr(sys.modules[__name__], t)]:
        t = timeit.Timer(setup='from __main__ import %s;' % test, stmt='%s()' % test)
        time = t.timeit(number=number) / number
        if time < 0.00001:
            result = '   (not installed?)'
        else:
            result = '%16.2f ms' % (1000 * time)
        print '%-35s %s' % (getattr(sys.modules[__name__], test).__doc__, result)

if __name__ == '__main__':
    run_create_tests()
    run_svg_tests()
    run_png_tests()
<commit_msg>Updated benchmark script<commit_after># -*- encoding: utf-8 -*-
"""\
Some benchmarks running against QR Code generators
"""
from __future__ import print_function
import sys
import timeit
import segno
try:
    import qrcode
    from qrcode import QRCode, ERROR_CORRECT_M
    from qrcode.image.svg import SvgImage, SvgPathImage
except ImportError:
    QRCode = None
try:
    from qrcodegen import QrCode
except ImportError:
    QrCode = None
try:
    import pyqrcode
except ImportError:
    pyqrcode = None

if QRCode:
    def create_qrcode(data='QR Code Symbol'):
        """qrcode create"""
        qr = QRCode(error_correction=ERROR_CORRECT_M)
        qr.add_data(data, optimize=False)
        qr.make()

    def svg_qrcode_path(data='QR Code Symbol'):
        """qrcode SVG path"""
        qr = QRCode(error_correction=ERROR_CORRECT_M, box_size=10,
                    image_factory=SvgPathImage)
        qr.add_data(data, optimize=False)
        qr.make_image().save('out/qrcode_path_%s.svg' % data)

    def svg_qrcode_rects(data='QR Code Symbol'):
        """qrcode SVG rects"""
        qr = QRCode(error_correction=ERROR_CORRECT_M, box_size=10,
                    image_factory=SvgImage)
        qr.add_data(data, optimize=False)
        qr.make_image().save('out/qrcode_rects_%s.svg' % data)

    def png_qrcode(data='QR Code Symbol'):
        """qrcode PNG"""
        qr = QRCode(error_correction=ERROR_CORRECT_M, box_size=10,
                    image_factory=SvgImage)
        qr.add_data(data, optimize=False)
        qr.make_image().save('out/qrcode_%s.png' % data)

if pyqrcode:
    def create_pyqrcode(data='QR Code Symbol'):
        """PyQRCode create"""
        pyqrcode.create(data, error='m')

    def svg_pyqrcode(data='QR Code Symbol'):
        """PyQRCode SVG"""
        pyqrcode.create(data, error='m').svg('out/pyqrcode_%s.svg' % data, scale=10)

    def png_pyqrcode(data='QR Code Symbol'):
        """PyQRCode PNG"""
        pyqrcode.create(data, error='m').png('out/pyqrcode_%s.png' % data, scale=10)

if QrCode:
    def create_qrcodegen(data='QR Code Symbol'):
        """qrcodegen create"""
        QrCode.encode_text(data, QrCode.Ecc.MEDIUM)

    def svg_qrcodegen(data='QR Code Symbol'):
        """qrcodegen SVG"""
        with open('out/qrcodegen_%s.svg' % data, 'wt') as f:
            f.write(QrCode.encode_text(data, QrCode.Ecc.MEDIUM).to_svg_str(border=4))

def create_segno(data='QR Code Symbol'):
    """Segno create"""
    segno.make_qr(data, error='m')

def svg_segno(data='QR Code Symbol'):
    """Segno SVG"""
    segno.make_qr(data, error='m').save('out/segno_%s.svg' % data, scale=10)

def png_segno(data='QR Code Symbol'):
    """Segno PNG"""
    segno.make_qr(data, error='m').save('out/segno_%s.png' % data, scale=10)

def run_create_tests(which=None, number=200, table=None):
    tests = ('create_segno', 'create_pyqrcode', 'create_qrcodegen',
             'create_qrcode')
    if which:
        tests = filter(lambda n: n[len('create_'):] in which, tests)
    _run_tests(tests, number, table)

def run_svg_tests(which=None, number=200, table=None):
    tests = ('svg_segno', 'svg_pyqrcode', 'svg_qrcodegen',
             'svg_qrcode_path', 'svg_qrcode_rects')

    if which:
        tests = filter(lambda n: n[len('svg_'):] in which, tests)
    _run_tests(tests, number, table)

def run_png_tests(which=None, number=200, table=None):
    tests = ('png_segno', 'png_pyqrcode', 'png_qrcode')
    if which:
        tests = filter(lambda n: n[len('png_'):] in which, tests)
    _run_tests(tests, number, table)

def _run_tests(tests, number, table=None):
    # Code taken from <https://genshi.edgewall.org/browser/trunk/examples/bench/bigtable.py>
    # Author: Jonas Borgström <jonas@edgewall.com>
    # License: BSD (I'd think since Genshi uses BSD as well)
    for test in [t for t in tests if hasattr(sys.modules[__name__], t)]:
        t = timeit.Timer(setup='from __main__ import %s;' % test, stmt='%s()' % test)
        time = t.timeit(number=number) / number
        if time < 0.00001:
            result = '   (not installed?)'
        else:
            result = '%16.2f ms' % (1000 * time)
        name = getattr(sys.modules[__name__], test).__doc__
        print('%-35s %s' % (name, result))
        if table is not None:
            table.append((name, '%.2f' % (1000 * time)))

if __name__ == '__main__':
    import csv
    table = []
    run_create_tests(table=table)
    run_svg_tests(table=table)
    run_png_tests(table=table)
    with open('out/results.csv', 'wb') as f:
        writer = csv.writer(f)
        writer.writerows(table)

<|endoftext|>


<commit_before># coding: utf-8

""" Description... """

from __future__ import division, print_function

__author__ = "adrn <adrn@astro.columbia.edu>"

# Standard library
import os, sys

# Third-party
import numpy as np
import astropy.units as u
from astropy.constants import G

from ..potential import *

__all__ = ["minimum_distance_matrix",
           "relative_normalized_coordinates",
           "generalized_variance"]

def relative_normalized_coordinates(potential, particle_orbits, satellite_orbit):
    """ Compute the coordinates of particles relative to the satellite,
        with positions normalized by the tidal radius and velocities
        normalized by the escape velocity.

        Note::
            Assumes the particle and satellite position/velocity
            units are the same!

        Parameters
        ----------
        potential : streams.CartesianPotential
        particle_orbits : OrbitCollection
        satellite_orbit : OrbitCollection
    """

    # need to add a new axis to normalize each coordinate component
    r_tide = potential._tidal_radius(m=satellite_orbit._m,
                                     r=satellite_orbit._r)[:,:,np.newaxis]
    v_esc = potential._escape_velocity(m=satellite_orbit._m,
                                       r_tide=r_tide)

    return (particle_orbits._r - satellite_orbit._r) / r_tide, \
           (particle_orbits._v - satellite_orbit._v) / v_esc

def minimum_distance_matrix(potential, particle_orbits, satellite_orbit):
    """ Compute the Nx6 matrix of minimum phase-space distance vectors.

        Parameters
        ----------
        potential : Potential
        particle_orbits : OrbitCollection
        satellite_orbit : OrbitCollection
    """

    Nparticles = particle_orbits._r.shape[1]

    R,V = relative_normalized_coordinates(potential, particle_orbits, satellite_orbit)
    D_ps = np.sqrt(np.sum(R**2, axis=-1) + np.sum(V**2, axis=-1))

    # Find the index of the time of the minimum D_ps for each particle
    min_time_idx = D_ps.argmin(axis=0)

    min_ps = np.zeros((Nparticles,6))
    xx = zip(min_time_idx, range(Nparticles))
    for kk in range(Nparticles):
        jj,ii = xx[kk]
        min_ps[ii] = np.append(R[jj,ii], V[jj,ii])

    return min_ps

def generalized_variance(potential, particle_orbits, satellite_orbit):
    """ Compute the variance scalar that we will minimize.

        Parameters
        ----------
        potential : Potential
            The full Milky Way potential object.
        particle_orbits : OrbitCollection
            An object containing orbit information for a collection of
            particles.
        satellite_orbit : OrbitCollection
            Data for the Sgr satellite center, interpolated onto the
            time grid for our particles.
    """

    min_ps = minimum_distance_matrix(potential, particle_orbits, satellite_orbit)

    cov_matrix = np.cov(min_ps.T)

    # cov_matrix -> (6 x 6) covariance matrix for particles
    #w,v = np.linalg.eig(cov_matrix)
    #return np.sum(w)
    #return np.prod(w)

    sign,logdet = np.linalg.slogdet(cov_matrix)
    return logdet<commit_msg>5 times tidal radius...<commit_after># coding: utf-8

""" Description... """

from __future__ import division, print_function

__author__ = "adrn <adrn@astro.columbia.edu>"

# Standard library
import os, sys

# Third-party
import numpy as np
import astropy.units as u
from astropy.constants import G

from ..potential import *

__all__ = ["minimum_distance_matrix",
           "relative_normalized_coordinates",
           "generalized_variance"]

def relative_normalized_coordinates(potential, particle_orbits, satellite_orbit):
    """ Compute the coordinates of particles relative to the satellite,
        with positions normalized by the tidal radius and velocities
        normalized by the escape velocity.

        Note::
            Assumes the particle and satellite position/velocity
            units are the same!

        Parameters
        ----------
        potential : streams.CartesianPotential
        particle_orbits : OrbitCollection
        satellite_orbit : OrbitCollection
    """

    # need to add a new axis to normalize each coordinate component
    r_tide = potential._tidal_radius(m=satellite_orbit._m,
                                     r=satellite_orbit._r)[:,np.newaxis,:]
    v_esc = potential._escape_velocity(m=satellite_orbit._m,
                                       r_tide=r_tide)

    return (particle_orbits._r - satellite_orbit._r) / (5.*r_tide), \
           (particle_orbits._v - satellite_orbit._v) / v_esc

def minimum_distance_matrix(potential, particle_orbits, satellite_orbit):
    """ Compute the Nx6 matrix of minimum phase-space distance vectors.

        Parameters
        ----------
        potential : Potential
        particle_orbits : OrbitCollection
        satellite_orbit : OrbitCollection
    """

    Nparticles = particle_orbits._r.shape[1]

    R,V = relative_normalized_coordinates(potential, particle_orbits, satellite_orbit)
    D_ps = np.sqrt(np.sum(R**2, axis=-1) + np.sum(V**2, axis=-1))

    # Find the index of the time of the minimum D_ps for each particle
    min_time_idx = D_ps.argmin(axis=0)

    min_ps = np.zeros((Nparticles,6))
    xx = zip(min_time_idx, range(Nparticles))
    for kk in range(Nparticles):
        jj,ii = xx[kk]
        min_ps[ii] = np.append(R[jj,ii], V[jj,ii])

    return min_ps

def generalized_variance(potential, particle_orbits, satellite_orbit):
    """ Compute the variance scalar that we will minimize.

        Parameters
        ----------
        potential : Potential
            The full Milky Way potential object.
        particle_orbits : OrbitCollection
            An object containing orbit information for a collection of
            particles.
        satellite_orbit : OrbitCollection
            Data for the Sgr satellite center, interpolated onto the
            time grid for our particles.
    """

    min_ps = minimum_distance_matrix(potential, particle_orbits, satellite_orbit)

    cov_matrix = np.cov(min_ps.T)

    # cov_matrix -> (6 x 6) covariance matrix for particles
    #w,v = np.linalg.eig(cov_matrix)
    #return np.sum(w)
    #return np.prod(w)

    sign,logdet = np.linalg.slogdet(cov_matrix)
    return logdet<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*-encoding:utf-8-*-

import sys
reload(sys)
sys.setdefaultencoding('UTF8')

import os, re, shutil, time, collections, json
from HTMLParser import HTMLParser

import itchat
from itchat.content import *

sending_type = {'Picture': 'img', 'Video': 'vid'}
data_path = 'data'
from_group_names = {u'酒井 9#'}
to_group_names = [u'酒井民间自救群', u'酒井 9# 二号']
senders = {u'酒井9#'}

def get_sender_receiver(msg):
    sender = None
    receiver = None
    if msg['FromUserName'][0:2] == '@@': # group chat
        sender = msg['ActualNickName']
        m = itchat.search_chatrooms(userName=msg['FromUserName'])
        if m is not None:
            receiver = m['NickName']
    elif msg['ToUserName'][0:2] == '@@': # group chat by myself
        if 'ActualNickName' in msg:
            sender = msg['ActualNickName']
        else:
            m = itchat.search_friends(userName=msg['FromUserName'])
            if m is not None:
                sender = m['NickName']
        m = itchat.search_chatrooms(userName=msg['ToUserName'])
        if m is not None:
            receiver = m['NickName']
    else: # personal chat
        m = itchat.search_friends(userName=msg['FromUserName'])
        if m is not None:
            sender = m['NickName']
        m = itchat.search_friends(userName=msg['ToUserName'])
        if m is not None:
            receiver = m['NickName']
    return sender.strip(), receiver.strip()

def print_msg(msg):
    if len(msg) == 0:
        return
    print json.dumps(msg).decode('unicode-escape').encode('utf8')

def get_whole_msg(msg, download=False, senders={}, receivers={}):
    if msg['FileName'][-4:] == 'gif': # can't handle gif pictures
        return []
    sender, receiver = get_sender_receiver(msg)
    if (sender in senders) or (receiver not in receivers):
        return []
    if len(msg['FileName']) > 0 and len(msg['Url']) == 0:
        if download: # download the file into data_path directory
            fn = os.path.join(data_path, msg['FileName'])
            msg['Text'](fn)
            c = '@%s@%s' % (sending_type.get(msg['Type'], 'fil'), fn)
        else:
            c = '@%s@%s' % (sending_type.get(msg['Type'], 'fil'), msg['FileName'])
        return ['[%s]:' % (sender), c]
    c = msg['Text']
    if len(msg['Url']) > 0:
        url = HTMLParser().unescape(msg['Url'])
        c += ' ' + url
    return ['[%s]: %s' % (sender, c)]

@itchat.msg_register([TEXT, PICTURE, MAP, CARD, SHARING, RECORDING,
    ATTACHMENT, VIDEO, FRIENDS], isFriendChat=True, isGroupChat=True)
def normal_msg(msg):
    msg_send = get_whole_msg(msg, download=True, senders=senders, receivers=from_group_names)
    if len(msg_send) == 0:
        return
    print_msg(msg_send)
    for m in msg_send:
        for name in to_group_names:
            room = itchat.search_chatrooms(name=name)
            if room is not None and len(room) > 0:
                username = room[0]['UserName']
                itchat.send(m, toUserName=username)

if __name__ == '__main__':
    if not os.path.exists(data_path):
        os.mkdir(data_path)
    # if the QR code doesn't show correctly, you can try to change the value
    # of enableCdmQR to 1 or -1 or -2. It nothing works, you can change it to
    # enableCmdQR=True and a picture will show up.
    itchat.auto_login(hotReload=True, enableCmdQR=2)
    itchat.run()
<commit_msg>dont need strip<commit_after>#!/usr/bin/env python
# -*-encoding:utf-8-*-

import sys
reload(sys)
sys.setdefaultencoding('UTF8')

import os, re, shutil, time, collections, json
from HTMLParser import HTMLParser

import itchat
from itchat.content import *

sending_type = {'Picture': 'img', 'Video': 'vid'}
data_path = 'data'
from_group_names = {u'酒井 9#'}
to_group_names = [u'酒井民间自救群', u'酒井 9# 二号']
senders = {u'酒井9#'}

def get_sender_receiver(msg):
    sender = None
    receiver = None
    if msg['FromUserName'][0:2] == '@@': # group chat
        sender = msg['ActualNickName']
        m = itchat.search_chatrooms(userName=msg['FromUserName'])
        if m is not None:
            receiver = m['NickName']
    elif msg['ToUserName'][0:2] == '@@': # group chat by myself
        if 'ActualNickName' in msg:
            sender = msg['ActualNickName']
        else:
            m = itchat.search_friends(userName=msg['FromUserName'])
            if m is not None:
                sender = m['NickName']
        m = itchat.search_chatrooms(userName=msg['ToUserName'])
        if m is not None:
            receiver = m['NickName']
    else: # personal chat
        m = itchat.search_friends(userName=msg['FromUserName'])
        if m is not None:
            sender = m['NickName']
        m = itchat.search_friends(userName=msg['ToUserName'])
        if m is not None:
            receiver = m['NickName']
    return sender, receiver

def print_msg(msg):
    if len(msg) == 0:
        return
    print json.dumps(msg).decode('unicode-escape').encode('utf8')

def get_whole_msg(msg, download=False, senders={}, receivers={}):
    if msg['FileName'][-4:] == 'gif': # can't handle gif pictures
        return []
    sender, receiver = get_sender_receiver(msg)
    if (sender in senders) or (receiver not in receivers):
        return []
    if len(msg['FileName']) > 0 and len(msg['Url']) == 0:
        if download: # download the file into data_path directory
            fn = os.path.join(data_path, msg['FileName'])
            msg['Text'](fn)
            c = '@%s@%s' % (sending_type.get(msg['Type'], 'fil'), fn)
        else:
            c = '@%s@%s' % (sending_type.get(msg['Type'], 'fil'), msg['FileName'])
        return ['[%s]:' % (sender), c]
    c = msg['Text']
    if len(msg['Url']) > 0:
        url = HTMLParser().unescape(msg['Url'])
        c += ' ' + url
    return ['[%s]: %s' % (sender, c)]

@itchat.msg_register([TEXT, PICTURE, MAP, CARD, SHARING, RECORDING,
    ATTACHMENT, VIDEO, FRIENDS], isFriendChat=True, isGroupChat=True)
def normal_msg(msg):
    msg_send = get_whole_msg(msg, download=True, senders=senders, receivers=from_group_names)
    if len(msg_send) == 0:
        return
    print_msg(msg_send)
    for m in msg_send:
        for name in to_group_names:
            room = itchat.search_chatrooms(name=name)
            if room is not None and len(room) > 0:
                username = room[0]['UserName']
                itchat.send(m, toUserName=username)

if __name__ == '__main__':
    if not os.path.exists(data_path):
        os.mkdir(data_path)
    # if the QR code doesn't show correctly, you can try to change the value
    # of enableCdmQR to 1 or -1 or -2. It nothing works, you can change it to
    # enableCmdQR=True and a picture will show up.
    itchat.auto_login(hotReload=True, enableCmdQR=2)
    itchat.run()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Prefix tree / Digital tree / Radix tree / Trie implementation
See definition: https://en.wikipedia.org/wiki/Trie
"""
import random
import collections

SEPARATOR = '\n'

class PrefixTree(collections.MutableMapping):
    """Prefix tree. In fact this is just a trie node holder"""

    def __init__(self):
        self.tree = TrieNode()

    def __contains__(self, value):
        return bool(list(self.tree.lookup(value=value, fuzzy=False)))

    def __setitem__(self, _, value):
        """Set tree value

        :param _:
        :param value:
        """
        self.tree.insert(value=value)

    def __getitem__(self, value):
        return self.tree.lookup(value=value)

    def __delitem__(self, key):
        return self.tree.remove(value=key)

    def __iter__(self):
        return iter(self.tree)

    def __len__(self):
        return self.tree.total_tags()

class TrieNode(collections.Iterable):
    """Main tree node class"""

    def __init__(self, tag='', parent=None):
        self.tag = tag
        self.value = None
        self.words = 0
        self.tags_num = 0
        self.nodes = {}
        self.parent = parent

    def _get_or_create_node(self, value, create=True):
        """Get or create trie node.
        Reusable in the lookup, insert and delete methods

        :param value: str
        :return: TrieNode
        """
        node = self
        idx = 1
        while idx <= len(value):
            tag = value[:idx]
            # Create new node with tag if doesn't exist
            if tag not in node.nodes:
                if create:
                    node.nodes[tag] = TrieNode(tag=tag, parent=node)
                    node.tags_num += 1
                else:
                    # Return None if node is not found
                    return None
            node = node.nodes[tag]

            # Increment index and continue traversing
            idx += 1
        return node

    def insert(self, value):
        """Inserting value to a tree

        :param value: str: value to insert
        """
        if not isinstance(value, str):
            raise ValueError("Wrong value to insert '%r', must be str" % value)

        # Get or create node
        node = self._get_or_create_node(value=value)
        if node.value is None:
            node.value = value
            node.parent.words += 1

    def remove(self, value):
        """Remove value from the tree
        Find the node and remove the value or the node itself if no children

        :param value: str:
        :rtype : int: -1, 0 or 1, where
         -1 - nothing to delete
         0  - whole node has been removed
         1  - only node value has been removed
        """

        node = self._get_or_create_node(value=value, create=False)
        if node is None or node.value is None:
            # Nothing to delete, no such value in the tree
            return -1

        # If there are children, just reset the value
        parent = node.parent
        parent.words -= 1
        if node.nodes:
            node.value = None
            return 1
        else:
            # Remove link from parent and remove the node object itself
            parent.tags_num -= 1
            del parent.nodes[node.tag]
            del node
            return 0

    def lookup(self, value, fuzzy=True):
        """Values lookup method.

        :param fuzzy: bool: continue search and return values from child nodes
        :param value: str: string value
        :return:
        """

        node = self
        idx = 1
        while idx <= len(value):
            tag = value[:idx]
            # Create new node with tag if doesn't exist
            if tag not in node.nodes:
                node.nodes[tag] = TrieNode(tag=tag)
                node.tags_num += 1
            node = node.nodes[tag]

            # Increment index and continue searching
            idx += 1

        # Handle if this is a final value and check if anything further
        if fuzzy:
            for val in node.get_values():
                yield val
        else:
            if node.value:
                yield node.value

    def total_words(self, prefix=None):
        """Get total words in general and particularly

        :param prefix: if prefix is set filter sum by branch
        :return: int
        """

        if prefix:
            node = self._get_or_create_node(create=False, value=prefix)
        else:
            node = self
        cnt = node.words

        # Traverse tree and gather all counters
        for node in self.traverse_tree():
            cnt += node.words

        return cnt

    def total_tags(self, tag=None):
        """Get total tags

        :param tag:
        :return:
        """
        if tag:
            node = self._get_or_create_node(create=False, value=tag)
        else:
            node = self
        cnt = node.words

        # Traverse tree and gather all counters
        for node in self.traverse_tree():
            cnt += node.tags_num

        return cnt

    def traverse_tree(self):
        """Traverse nodes and get end values with depth search
        """
        nodes = [self]
        while nodes:
            node = nodes.pop()
            yield node

            if node.nodes:
                nodes.extend(node.nodes.values())

    def get_values(self):
        """Handy helper around traverse to get tree values
        :rtype : str: node value
        """
        for node in self.traverse_tree():
            if node.value:
                yield node.value

    def __repr__(self):
        return "%s(tag='%s', words=%d, tags_num=%d)" % (
            self.__class__.__name__, self.tag, self.words, self.tags_num)

    def __iter__(self):
        return iter(self.nodes)
<commit_msg>Trie update: cleanup<commit_after># -*- coding: utf-8 -*-
"""Prefix tree / Digital tree / Radix tree / Trie implementation
See definition: https://en.wikipedia.org/wiki/Trie
"""
import collections

class PrefixTree(collections.MutableMapping):
    """Prefix tree. In fact this is just a trie node holder"""

    def __init__(self):
        self.tree = TrieNode()

    def __contains__(self, value):
        return bool(list(self.tree.lookup(value=value, fuzzy=False)))

    def __setitem__(self, _, value):
        """Set tree value

        :param _:
        :param value:
        """
        self.tree.insert(value=value)

    def __getitem__(self, value):
        return self.tree.lookup(value=value)

    def __delitem__(self, key):
        return self.tree.remove(value=key)

    def __iter__(self):
        return iter(self.tree)

    def __len__(self):
        return self.tree.total_tags()

class TrieNode(collections.Iterable):
    """Main tree node class"""

    def __init__(self, tag='', parent=None):
        self.tag = tag
        self.value = None
        self.words = 0
        self.tags_num = 0
        self.nodes = {}
        self.parent = parent

    def _get_or_create_node(self, value, create=True):
        """Get or create trie node.
        Reusable in the lookup, insert and delete methods

        :param value: str
        :return: TrieNode
        """
        node = self
        idx = 1
        while idx <= len(value):
            tag = value[:idx]
            # Create new node with tag if doesn't exist
            if tag not in node.nodes:
                if create:
                    node.nodes[tag] = TrieNode(tag=tag, parent=node)
                    node.tags_num += 1
                else:
                    # Return None if node is not found
                    return None
            node = node.nodes[tag]

            # Increment index and continue traversing
            idx += 1
        return node

    def insert(self, value):
        """Inserting value to a tree

        :param value: str: value to insert
        """
        if not isinstance(value, str):
            raise ValueError("Wrong value to insert '%r', must be str" % value)

        # Get or create node
        node = self._get_or_create_node(value=value)
        if node.value is None:
            node.value = value
            node.parent.words += 1

    def remove(self, value):
        """Remove value from the tree
        Find the node and remove the value or the node itself if no children

        :param value: str:
        :rtype : int: -1, 0 or 1, where
         -1 - nothing to delete
         0  - whole node has been removed
         1  - only node value has been removed
        """

        node = self._get_or_create_node(value=value, create=False)
        if node is None or node.value is None:
            # Nothing to delete, no such value in the tree
            return -1

        # If there are children, just reset the value
        parent = node.parent
        parent.words -= 1
        if node.nodes:
            node.value = None
            return 1
        else:
            # Remove link from parent and remove the node object itself
            parent.tags_num -= 1
            del parent.nodes[node.tag]
            del node
            return 0

    def lookup(self, value, fuzzy=True):
        """Values lookup method.

        :param fuzzy: bool: continue search and return values from child nodes
        :param value: str: string value
        :return:
        """

        node = self
        idx = 1
        while idx <= len(value):
            tag = value[:idx]
            # Create new node with tag if doesn't exist
            if tag not in node.nodes:
                node.nodes[tag] = TrieNode(tag=tag)
                node.tags_num += 1
            node = node.nodes[tag]

            # Increment index and continue searching
            idx += 1

        # Handle if this is a final value and check if anything further
        if fuzzy:
            for val in node.get_values():
                yield val
        else:
            if node.value:
                yield node.value

    def total_words(self, prefix=None):
        """Get total words in general and particularly

        :param prefix: if prefix is set filter sum by branch
        :return: int
        """

        if prefix:
            node = self._get_or_create_node(create=False, value=prefix)
        else:
            node = self
        cnt = node.words

        # Traverse tree and gather all counters
        for node in self.traverse_tree():
            cnt += node.words

        return cnt

    def total_tags(self, tag=None):
        """Get total tags

        :param tag:
        :return:
        """
        if tag:
            node = self._get_or_create_node(create=False, value=tag)
        else:
            node = self
        cnt = node.words

        # Traverse tree and gather all counters
        for node in self.traverse_tree():
            cnt += node.tags_num

        return cnt

    def traverse_tree(self):
        """Traverse nodes and get end values with depth search
        """
        nodes = [self]
        while nodes:
            node = nodes.pop()
            yield node

            if node.nodes:
                nodes.extend(node.nodes.values())

    def get_values(self):
        """Handy helper around traverse to get tree values
        :rtype : str: node value
        """
        for node in self.traverse_tree():
            if node.value:
                yield node.value

    def __repr__(self):
        return "%s(tag='%s', words=%d, tags_num=%d)" % (
            self.__class__.__name__, self.tag, self.words, self.tags_num)

    def __iter__(self):
        return iter(self.nodes)
<|endoftext|>


<commit_before>import threading
from scadasim.utils import parse_yml, build_simulation
import sys
import logging

logging.basicConfig()
log = logging.getLogger()
log.setLevel(logging.DEBUG)

class Simulator(object):

	def __init__(self, dbus=False):
		self.dbus = dbus
		if self.dbus:
			from dbusservice import DBusService
        	self.dbusservice = DBusService()

	def load_yml(self, path_to_yaml_config):
		"""Read and parse YAML configuration file into simulator devices
		"""
		self.path_to_yaml_config = path_to_yaml_config
		self.config = parse_yml(path_to_yaml_config)
		simulation = build_simulation(self.config)
		self.settings = simulation['settings']
		self.devices = simulation['devices']
		self.sensors = simulation['sensors']
		self.plcs = simulation['plcs']

		self.set_speed(self.settings['speed'])

	def start(self):
		"""Start the simulation"""
		for device in self.devices.values():
			device.activate()

		for sensor in self.sensors.values():
			sensor.activate()

		if self.dbus:
        	self.dbusservice.load_plcs(self.plcs)
        	self.dbusservice.set_speed(self.settings['speed'])
        	self.dbusservice.start()

	def pause(self):
		"""Pause the simulation"""
		for device in self.devices.values():
			device.deactivate()

		for sensor in self.sensors.values():
			sensor.deactivate()

	def stop(self):
		"""Stop and destroy the simulation"""
		self.pause()
		sys.exit(0)

	def set_speed(self, speed):
		"""Increase/Decrease the speed of the simulation
			default: 1/second
		"""
		for device in self.devices.values():
			device.speed = speed

		for sensor in self.sensors.values():
			sensor.speed = speed

	def restart(self):
		"""Stop and reload the simulation from the original config"""
		self.pause()
		self.load_yml(self.path_to_yaml_config)
		self.start()<commit_msg>fixed typo<commit_after>import threading
from scadasim.utils import parse_yml, build_simulation
import sys
import logging

logging.basicConfig()
log = logging.getLogger()
log.setLevel(logging.DEBUG)

class Simulator(object):

    def __init__(self, dbus=False):
        self.dbus = dbus
        if self.dbus:
            from dbusservice import DBusService
            self.dbusservice = DBusService()

    def load_yml(self, path_to_yaml_config):
        """Read and parse YAML configuration file into simulator devices
        """
        self.path_to_yaml_config = path_to_yaml_config
        self.config = parse_yml(path_to_yaml_config)
        simulation = build_simulation(self.config)
        self.settings = simulation['settings']
        self.devices = simulation['devices']
        self.sensors = simulation['sensors']
        self.plcs = simulation['plcs']

        self.set_speed(self.settings['speed'])

    def start(self):
        """Start the simulation"""
        for device in self.devices.values():
            device.activate()

        for sensor in self.sensors.values():
            sensor.activate()

        if self.dbus:
            self.dbusservice.load_plcs(self.plcs)
            self.dbusservice.set_speed(self.settings['speed'])
            self.dbusservice.start()

    def pause(self):
        """Pause the simulation"""
        for device in self.devices.values():
            device.deactivate()

        for sensor in self.sensors.values():
            sensor.deactivate()

    def stop(self):
        """Stop and destroy the simulation"""
        self.pause()
        sys.exit(0)

    def set_speed(self, speed):
        """Increase/Decrease the speed of the simulation
            default: 1/second
        """
        for device in self.devices.values():
            device.speed = speed

        for sensor in self.sensors.values():
            sensor.speed = speed

    def restart(self):
        """Stop and reload the simulation from the original config"""
        self.pause()
        self.load_yml(self.path_to_yaml_config)
        self.start()<|endoftext|>


<commit_before>import threading
from scadasim.utils import parse_yml, build_simulation
import sys
import logging

logging.basicConfig()
log = logging.getLogger()
log.setLevel(logging.DEBUG)

class Simulator(object):

    def __init__(self, dbus=False):
        self.dbus = dbus
        if self.dbus:
            from dbusservice import DBusService
            self.dbusservice = DBusService()

    def load_yml(self, path_to_yaml_config):
        """Read and parse YAML configuration file into simulator devices
        """
        self.path_to_yaml_config = path_to_yaml_config
        self.config = parse_yml(path_to_yaml_config)
        simulation = build_simulation(self.config)
        self.settings = simulation['settings']
        self.devices = simulation['devices']
        self.sensors = simulation['sensors']
        self.plcs = simulation['plcs']

        self.set_speed(self.settings['speed'])

    def start(self):
        """Start the simulation"""
        for device in self.devices.values():
            device.activate()

        for sensor in self.sensors.values():
            sensor.activate()

        log.debug(self.sensors)
        log.debug(self.plcs)

        if self.dbus:
            for plc in self.plcs:
                for sensor in self.plcs[plc]['sensors']:
                    self.plcs[plc][sensor]['read_sensor'] = self.sensors[sensor].read_sensor
            self.dbusservice.load_plcs(self.plcs)
            self.dbusservice.set_speed(self.settings['speed'])
            self.dbusservice.activate()

    def pause(self):
        """Pause the simulation"""
        for device in self.devices.values():
            device.deactivate()

        for sensor in self.sensors.values():
            sensor.deactivate()

        if self.dbus:
            self.dbusservice.deactivate()

    def stop(self):
        """Stop and destroy the simulation"""
        self.pause()
        sys.exit(0)

    def set_speed(self, speed):
        """Increase/Decrease the speed of the simulation
            default: 1/second
        """
        for device in self.devices.values():
            device.speed = speed

        for sensor in self.sensors.values():
            sensor.speed = speed

        if self.dbus:
            self.dbusservice.set_speed(speed)

    def restart(self):
        """Stop and reload the simulation from the original config"""
        self.pause()
        self.load_yml(self.path_to_yaml_config)
        self.start()<commit_msg>updated dbus handlers<commit_after>import threading
from scadasim.utils import parse_yml, build_simulation
import sys
import logging

logging.basicConfig()
log = logging.getLogger()
log.setLevel(logging.DEBUG)

class Simulator(object):

    def __init__(self, dbus=False):
        self.dbus = dbus
        if self.dbus:
            from dbusservice import DBusService
            self.dbusservice = DBusService()

    def load_yml(self, path_to_yaml_config):
        """Read and parse YAML configuration file into simulator devices
        """
        self.path_to_yaml_config = path_to_yaml_config
        self.config = parse_yml(path_to_yaml_config)
        simulation = build_simulation(self.config)
        self.settings = simulation['settings']
        self.devices = simulation['devices']
        self.sensors = simulation['sensors']
        self.plcs = simulation['plcs']

        self.set_speed(self.settings['speed'])

    def start(self):
        """Start the simulation"""
        for device in self.devices.values():
            device.activate()

        for sensor in self.sensors.values():
            sensor.activate()

        if self.dbus:
            for plc in self.plcs:
                for sensor in self.plcs[plc]['sensors']:
                    self.plcs[plc]['sensors'][sensor]['read_sensor'] = self.sensors[sensor].read_sensor
            self.dbusservice.load_plcs(self.plcs)
            self.dbusservice.set_speed(self.settings['speed'])
            self.dbusservice.activate()

    def pause(self):
        """Pause the simulation"""
        for device in self.devices.values():
            device.deactivate()

        for sensor in self.sensors.values():
            sensor.deactivate()

        if self.dbus:
            self.dbusservice.deactivate()

    def stop(self):
        """Stop and destroy the simulation"""
        self.pause()
        sys.exit(0)

    def set_speed(self, speed):
        """Increase/Decrease the speed of the simulation
            default: 1/second
        """
        for device in self.devices.values():
            device.speed = speed

        for sensor in self.sensors.values():
            sensor.speed = speed

        if self.dbus:
            self.dbusservice.set_speed(speed)

    def restart(self):
        """Stop and reload the simulation from the original config"""
        self.pause()
        self.load_yml(self.path_to_yaml_config)
        self.start()<|endoftext|>


<commit_before>import seaborn as sns
from matplotlib import pyplot as plt
import pandas as pd
from . import utils
from ..preprocessing.simple import normalize_per_cell

def highest_expr_genes(adata, n_top=30, save=None, show=None, ax=None, **kwargs):
    """
    Computes, for each gene, the fraction of counts assigned
    to that gene within a cell. The n_top genes with the highest
    mean fraction over all cells are plotted as boxplots.

    This plot is similar to the `scater` package function
    `plotQC(type = "highest-expression")`.
    See (https://bioconductor.org/packages/devel/bioc/vignettes/scater/inst/doc/vignette-qc.html)

    Parameters
    ----------
    adata : :class:`~anndata.AnnData`
        Annotated data matrix.
    n_top : int, optional (default:30)
        Number of top
    save : `bool` or `str`, optional (default: `None`)
        If `True` or a `str`, save the figure. A string is appended to the
        default filename. Infer the filetype if ending on {{'.pdf', '.png', '.svg'}}.
    show : bool, optional (default: None)
        Show the plot, do not return axis.
    ax : `matplotlib.Axes`
         A `matplotlib.Axes` object.
    **kwargs : keyword arguments
        Are passed to `seaborn.boxplot`.

    Returns
    -------
    A `matplotlib.Axes` object

    """
    from scipy.sparse import issparse

    # compute the percentage of each gene per cell
    dat = normalize_per_cell(adata, counts_per_cell_after=100, copy=True)

    # identify the genes with the highest mean
    if issparse(dat.X):
        dat.var['mean_percent'] = dat.X.mean(axis=0).A1
        top = dat.var.sort_values('mean_percent', ascending=False).index[:n_top]
        dat = dat[:, top]
        dat = pd.DataFrame(dat.X.toarray(), index=dat.obs_names, columns=dat.var_names)

    else:
        dat.var['mean_percent'] = dat.X.mean(axis=0)
        top = dat.var.sort_values('mean_percent', ascending=False).index[:n_top]
        dat = dat[:, top]
        dat = pd.DataFrame(dat.X, index=dat.obs_names, columns=dat.var_names)

    if not ax:
        # figsize is hardcoded to produce a tall image. To change the fig size,
        # a matplotlib.Axes object needs to be passed.
        height = (n_top * 0.2) + 1.5
        fig, ax = plt.subplots(figsize=(5, height))
    sns.boxplot(data=dat, orient='h', ax=ax, fliersize=1, **kwargs)
    ax.set_xlabel("% of total counts")
    utils.savefig_or_show('highest_expression', show=show, save=save)
    return ax
<commit_msg>cleaned up docs and axes return in high_expr_genes<commit_after>import seaborn as sns
from matplotlib import pyplot as plt
import pandas as pd
from . import utils
from ..preprocessing.simple import normalize_per_cell

def highest_expr_genes(adata, n_top=30, save=None, show=None, ax=None, **kwargs):
    """Fraction of counts assigned to each gene over all cells.

    Computes, for each gene, the fraction of counts assigned to that gene within
    a cell. The `n_top` genes with the highest mean fraction over all cells are
    plotted as boxplots.

    This plot is similar to the `scater` package function `plotQC(type =
    "highest-expression")`.
    See `here <https://bioconductor.org/packages/devel/bioc/vignettes/scater/inst/doc/vignette-qc.html>`__.

    Parameters
    ----------
    adata : :class:`~anndata.AnnData`
        Annotated data matrix.
    n_top : int, optional (default:30)
        Number of top
    save : `bool` or `str`, optional (default: `None`)
        If `True` or a `str`, save the figure. A string is appended to the
        default filename. Infer the filetype if ending on {'.pdf', '.png', '.svg'}.
    show : bool, optional (default: None)
        Show the plot, do not return axis.
    ax : `matplotlib.Axes`
         A `matplotlib.Axes` object.
    **kwargs : keyword arguments
        Are passed to `seaborn.boxplot`.

    Returns
    -------
    A `matplotlib.Axes` object

    """
    from scipy.sparse import issparse

    # compute the percentage of each gene per cell
    dat = normalize_per_cell(adata, counts_per_cell_after=100, copy=True)

    # identify the genes with the highest mean
    if issparse(dat.X):
        dat.var['mean_percent'] = dat.X.mean(axis=0).A1
        top = dat.var.sort_values('mean_percent', ascending=False).index[:n_top]
        dat = dat[:, top]
        dat = pd.DataFrame(dat.X.toarray(), index=dat.obs_names, columns=dat.var_names)

    else:
        dat.var['mean_percent'] = dat.X.mean(axis=0)
        top = dat.var.sort_values('mean_percent', ascending=False).index[:n_top]
        dat = dat[:, top]
        dat = pd.DataFrame(dat.X, index=dat.obs_names, columns=dat.var_names)

    if not ax:
        # figsize is hardcoded to produce a tall image. To change the fig size,
        # a matplotlib.Axes object needs to be passed.
        height = (n_top * 0.2) + 1.5
        fig, ax = plt.subplots(figsize=(5, height))
    sns.boxplot(data=dat, orient='h', ax=ax, fliersize=1, **kwargs)
    ax.set_xlabel("% of total counts")
    utils.savefig_or_show('highest_expression', show=show, save=save)
    return ax if show == False else None
<|endoftext|>


<commit_before>import logging
from google.appengine.ext import webapp
from google.appengine.ext.webapp import util
from django.utils import simplejson
from model import get_current_youtify_user_model
from model import get_display_name_for_youtify_user_model
from model import get_playlist_struct_from_playlist_model
from model import get_playlist_structs_by_id
from model import Playlist

class FollowPlaylist(webapp.RequestHandler):

    def post(self):
        """Follows a playlist"""
        youtify_user_model = get_current_youtify_user_model()
        if youtify_user_model == None:
            self.error(403)
            return

        playlist_id = self.request.path.split('/')[-1]
        playlist_model = Playlist.get_by_id(int(playlist_id))
        if playlist_model is None:
            self.error(404)
            return

        youtify_user_model.playlist_subscriptions.append(playlist_model.key())
        youtify_user_model.save()

        playlist_model.followers.append(youtify_user_model.key())
        playlist_model.save()

        self.response.headers['Content-Type'] = 'text/plain'
        self.response.out.write('ok')

    def delete(self):
        """Unfollows a playlist"""
        youtify_user_mode = get_current_youtify_user_model()
        if youtify_user_model == None:
            self.error(403)
            return

        playlist_id = self.request.path.split('/')[-1]
        playlist_model = Playlist.get_by_id(int(playlist_id))

        youtify_user_model.playlist_subscriptions.remove(m.key())
        youtify_user_model.save()

        playlist_model.followers.remove(youtify_user_model.key())
        playlist_model.save()

        self.response.headers['Content-Type'] = 'text/plain'
        self.response.out.write('ok')

class SpecificPlaylistHandler(webapp.RequestHandler):

    def get(self):
        """Get playlist"""
        playlist_id = self.request.path.split('/')[-1]
        playlist_struct = get_playlist_structs_by_id(playlist_id)

        if playlist_struct:
            self.response.headers['Content-Type'] = 'application/json'
            self.response.out.write(simplejson.dumps(playlist_struct))
        else:
            self.error(404)

    def post(self):
        """Update playlist"""
        youtify_user_model = get_current_youtify_user_model()
        if youtify_user_model == None:
            self.error(403)
            return

        playlist_id = self.request.path.split('/')[-1]
        playlist_model = Playlist.get_by_id(int(playlist_id))
        json = self.request.get('json', None)
        device = self.request.get('device')

        if json is None:
            self.error(400)
            return

        if playlist_model.owner.key() == youtify_user_model.key():
            if youtify_user_model.device != device:
                self.error(409)
                self.response.out.write('wrong_device')
                return
            else:
                old_playlist = simplejson.loads(json)
                playlist_model.private = old_playlist.get('isPrivate', False)
                playlist_model.tracks_json = simplejson.dumps(old_playlist['videos'])
                playlist_model.owner = youtify_user_model
                playlist_model.title = old_playlist['title']
                playlist_model.remote_id = old_playlist['remoteId']
                playlist_model.json = None
                playlist_model.save()

                self.response.out.write(str(playlist_model.key().id()))
        else:
            self.error(403)

    def delete(self):
        """Delete playlist"""
        youtify_user_mode = get_current_youtify_user_model()
        if youtify_user_model == None:
            self.error(403)
            return

        playlist_id = self.request.path.split('/')[-1]
        playlist_model = Playlist.get_by_id(int(playlist_id))

        if playlist_model.owner.key() == youtify_user_model.key():
            youtify_user_model.playlists.remove(playlist_model.key())
            youtify_user_model.save()

            playlist_model.delete()
        else:
            self.error(403)

class PlaylistsHandler(webapp.RequestHandler):

    def get(self):
        """Get playlists for logged in user"""
        youtify_user_model = get_current_youtify_user_model()
        youtify_user_struct = get_youtify_user_struct(youtify_user_model, False, True)

        self.response.headers['Content-Type'] = 'application/json'
        self.response.out.write(simplejson.dumps(playlists = youtify_user_struct['playlists']))

    def post(self):
        """Create new playlist"""
        youtify_user_model = get_current_youtify_user_model()
        json_playlist = simplejson.loads(self.request.get('json'))

        if json_playlist is None:
            self.error(500)

        playlist_model = Playlist(owner=youtify_user_model, json=None)
        playlist_model.private = json_playlist.get('isPrivate', False)
        playlist_model.tracks_json = None
        playlist_model.title = json_playlist['title']
        playlist_model.put()

        youtify_user_model.playlists.append(playlist_model.key())
        youtify_user_model.save()

        self.response.headers['Content-Type'] = 'application/json'
        self.response.out.write(simplejson.dumps(get_playlist_struct_from_playlist_model(playlist_model)))

def main():
    application = webapp.WSGIApplication([
        ('/api/playlists/follow/.*', FollowPlaylist),
        ('/api/playlists/.*', SpecificPlaylistHandler),
        ('/api/playlists', PlaylistsHandler),
    ], debug=True)
    util.run_wsgi_app(application)

if __name__ == '__main__':
    main()
<commit_msg>Fix bugs in playlist subscribe backend<commit_after>import logging
from google.appengine.ext import webapp
from google.appengine.ext.webapp import util
from django.utils import simplejson
from model import get_current_youtify_user_model
from model import get_display_name_for_youtify_user_model
from model import get_playlist_struct_from_playlist_model
from model import get_playlist_structs_by_id
from model import Playlist

class FollowPlaylist(webapp.RequestHandler):

    def post(self):
        """Follows a playlist"""
        youtify_user_model = get_current_youtify_user_model()
        if youtify_user_model == None:
            self.error(403)
            return

        playlist_id = self.request.path.split('/')[-1]
        playlist_model = Playlist.get_by_id(int(playlist_id))
        if playlist_model is None:
            self.error(404)
            return

        youtify_user_model.playlist_subscriptions.append(playlist_model.key())
        youtify_user_model.save()

        playlist_model.followers.append(youtify_user_model.key())
        playlist_model.save()

        self.response.headers['Content-Type'] = 'text/plain'
        self.response.out.write('ok')

    def delete(self):
        """Unfollows a playlist"""
        youtify_user_model = get_current_youtify_user_model()
        if youtify_user_model == None:
            self.error(403)
            return

        playlist_id = self.request.path.split('/')[-1]
        playlist_model = Playlist.get_by_id(int(playlist_id))

        youtify_user_model.playlist_subscriptions.remove(playlist_model.key())
        youtify_user_model.save()

        playlist_model.followers.remove(youtify_user_model.key())
        playlist_model.save()

        self.response.headers['Content-Type'] = 'text/plain'
        self.response.out.write('ok')

class SpecificPlaylistHandler(webapp.RequestHandler):

    def get(self):
        """Get playlist"""
        playlist_id = self.request.path.split('/')[-1]
        playlist_struct = get_playlist_structs_by_id(playlist_id)

        if playlist_struct:
            self.response.headers['Content-Type'] = 'application/json'
            self.response.out.write(simplejson.dumps(playlist_struct))
        else:
            self.error(404)

    def post(self):
        """Update playlist"""
        youtify_user_model = get_current_youtify_user_model()
        if youtify_user_model == None:
            self.error(403)
            return

        playlist_id = self.request.path.split('/')[-1]
        playlist_model = Playlist.get_by_id(int(playlist_id))
        json = self.request.get('json', None)
        device = self.request.get('device')

        if json is None:
            self.error(400)
            return

        if playlist_model.owner.key() == youtify_user_model.key():
            if youtify_user_model.device != device:
                self.error(409)
                self.response.out.write('wrong_device')
                return
            else:
                old_playlist = simplejson.loads(json)
                playlist_model.private = old_playlist.get('isPrivate', False)
                playlist_model.tracks_json = simplejson.dumps(old_playlist['videos'])
                playlist_model.owner = youtify_user_model
                playlist_model.title = old_playlist['title']
                playlist_model.remote_id = old_playlist['remoteId']
                playlist_model.json = None
                playlist_model.save()

                self.response.out.write(str(playlist_model.key().id()))
        else:
            self.error(403)

    def delete(self):
        """Delete playlist"""
        youtify_user_mode = get_current_youtify_user_model()
        if youtify_user_model == None:
            self.error(403)
            return

        playlist_id = self.request.path.split('/')[-1]
        playlist_model = Playlist.get_by_id(int(playlist_id))

        if playlist_model.owner.key() == youtify_user_model.key():
            youtify_user_model.playlists.remove(playlist_model.key())
            youtify_user_model.save()

            playlist_model.delete()
        else:
            self.error(403)

class PlaylistsHandler(webapp.RequestHandler):

    def get(self):
        """Get playlists for logged in user"""
        youtify_user_model = get_current_youtify_user_model()
        youtify_user_struct = get_youtify_user_struct(youtify_user_model, False, True)

        self.response.headers['Content-Type'] = 'application/json'
        self.response.out.write(simplejson.dumps(playlists = youtify_user_struct['playlists']))

    def post(self):
        """Create new playlist"""
        youtify_user_model = get_current_youtify_user_model()
        json_playlist = simplejson.loads(self.request.get('json'))

        if json_playlist is None:
            self.error(500)

        playlist_model = Playlist(owner=youtify_user_model, json=None)
        playlist_model.private = json_playlist.get('isPrivate', False)
        playlist_model.tracks_json = None
        playlist_model.title = json_playlist['title']
        playlist_model.put()

        youtify_user_model.playlists.append(playlist_model.key())
        youtify_user_model.save()

        self.response.headers['Content-Type'] = 'application/json'
        self.response.out.write(simplejson.dumps(get_playlist_struct_from_playlist_model(playlist_model)))

def main():
    application = webapp.WSGIApplication([
        ('/api/playlists/follow/.*', FollowPlaylist),
        ('/api/playlists/.*', SpecificPlaylistHandler),
        ('/api/playlists', PlaylistsHandler),
    ], debug=True)
    util.run_wsgi_app(application)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

import numpy as np
import matplotlib.pyplot as plt
from sqlalchemy import func

from model import (
    Contact,
    Message,
    create_session,
)

def main():
    session = create_session()
    date_format = '%Y-%m-%W'
    results = session.query(
        func.strftime(date_format, Message.sent),
        func.sum(Message.polarity),
        func.count(Message.id),
    ).join(Contact).filter(
        Contact.name == "Me",
    ).group_by(
        func.strftime(date_format, Message.sent),
    ).all()

    # width = .35
    x_axis_labels = [x[1][0] for x in enumerate(results) if x[0] % 10 == 0]
    x_axis = [x[0] for x in enumerate(results)]
    y_axis = [x[1]/x[2] for x in results]

    print "Samples: {}".format(len(x_axis))

    plt.bar(x_axis, y_axis)
    plt.xticks(
        np.arange(min(x_axis), max(x_axis), 10),
        x_axis_labels,
        rotation=45,
        horizontalalignment='center',
    )

    plt.savefig('output.png', pad_inches=0.1)

if __name__ == '__main__':
    main()
<commit_msg>Update plotting script to plot better graphs<commit_after>#!/usr/bin/env python
# encoding: utf-8

import numpy as np
import matplotlib.pyplot as plt
from sqlalchemy import func, or_, and_

from model import (
    Contact,
    Message,
    create_session,
)

def main():
    session = create_session()
    date_format = '%Y-%m-%W'
    results = session.query(
        func.strftime(date_format, Message.sent),
        func.sum(Message.polarity),
        func.count(Message.id),
        func.strftime('%m-%d-%Y', Message.sent),
    ).join(Contact).filter(
        and_(
            Contact.name == "Me",
            Message.polarity != 0,
            Message.subjectivity >= .9,
        )
    ).group_by(
        func.strftime(date_format, Message.sent),
    ).order_by(
        Message.sent.asc()
    ).all()

    # width = .35
    axis_every = 10 * 1

    x_axis_labels = [
        x[1][3] for x in enumerate(results) if x[0] % axis_every == 0]
    x_axis = [x[0] for x in enumerate(results)]
    y_axis = [x[1]/x[2] for x in results]
    colors = map(lambda v: {True: 'g', False: 'r'}[v > 0], y_axis)
    #y_axis = map(abs, y_axis)

    print "Samples: {}".format(len(x_axis))

    plt.rcParams["figure.figsize"][0] = 21.0
    plt.rcParams["figure.figsize"][1] = 9.0
    plt.rcParams["figure.autolayout"] = True

    plt.bar(x_axis, y_axis, linewidth=0, color=colors)
    plt.grid(b=True, which='major', linestyle='-')
    plt.title('Positive or Negative calculated sentiment of sent text messages per week')
    plt.ylabel("Calculated percentage of sentiment per message")
    plt.xlabel("Date")
    plt.xticks(
        np.arange(min(x_axis), max(x_axis), axis_every),
        x_axis_labels,
        rotation=35,
        horizontalalignment='center',
    )
    plt.xlim([min(x_axis) - 0.5, max(x_axis) + 0.5])
    plt.ylim([min(y_axis) - .1, max(y_axis) + .1])

    print plt.rcParams["figure.figsize"]

    plt.savefig('output.png')

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import os
import sys
import re
import codecs
from io import StringIO
import xml.sax
import xml.sax.saxutils
import asyncio
import importlib
from zipfile import ZipFile
from datetime import datetime

from .toolbox.config import get_description

class Element:
    """
    Simple XML element that can have either a text content or child
    elements. This is common for config files, Freevo fxd files, XMPP
    stanzas, and many other use cases. It is possible to access the
    attribute or the (first) child by accessing a member variable with
    that name. If the name exists as attribute and child or if there
    are several children by that name, additional helper functions are
    provided. The element's name can be accessed using the tagname
    member variable.

    A special variable is 'content'. If provided to __init__, it can
    contain text element's children (either a list of Elements or one
    Element) or the text of the element. The same is true when
    accessing Element.content. If the element has children, the
    attribute or child with that name is returned (or None if it does
    not exist). If it is a node without children, Element.content will
    refer to the text content.

    SAX parser have to use the internal variables _children, _attr,
    and _content.
    """
    def __init__(self, tagname, xmlns=None, content=None, **attr):
        self.tagname = tagname
        self.xmlns = xmlns
        self._content = ''
        self._children = []
        self._attr = attr
        if content:
            if isinstance(content, (list, tuple)):
                self._children = content
            elif isinstance(content, Element):
                self._children = [ content ]
            else:
                self._content = content

    @property
    def attributes(self):
        return self._attr.keys()

    def append(self, element):
        """
        Append an element to the list of children.
        """
        self._children.append(element)

    def add_child(self, tagname, xmlns=None, content=None, **attr):
        """
        Append an element to the list of children.
        """
        element = Element(tagname, xmlns, content, **attr)
        self._children.append(element)
        return element

    def has_child(self, name):
        """
        Return if the element has at least one child with the given element name.
        """
        return self.get_child(name) is not None

    def get_child(self, name):
        """
        Return the first child with the given name or None.
        """
        for child in self._children:
            if child.tagname == name:
                return child
        return None

    def get_children(self, name=None):
        """
        Return a list of children with the given name.
        """
        if name is None:
            return self._children[:]
        children = []
        for child in self._children:
            if child.tagname == name:
                children.append(child)
        return children

    def __iter__(self):
        """
        Iterate through the children.
        """
        return self._children.__iter__()

    def get(self, item, default=None):
        """
        Get the given attribute value or None if not set.
        """
        return self._attr.get(item, default)

    def __getitem__(self, item):
        """
        Get the given attribute value or raise a KeyError if not set.
        """
        return self._attr[item]

    def __setitem__(self, item, value):
        """
        Set the given attribute to a new value.
        """
        self._attr[item] = value

    def __getattr__(self, attr):
        """
        Magic function to return the attribute or child with the given name.
        """
        if attr == 'content' and not self._children:
            return self._content
        result = self._attr.get(attr)
        if result is not None:
            return result
        result = self.get_child(attr)
        if result is not None:
            return result
        if '_' in attr:
            return getattr(self, attr.replace('_', '-'))

    def __cmp__(self, other):
        if isinstance(other, (str, unicode)):
            return cmp(self.tagname, other)
        return object.__cmp__(self, other)

    def __repr__(self):
        """
        Python representation string
        """
        return '<Element %s>' % self.tagname

    def __str__(self):
        """
        Convert the element into an XML unicode string.
        """
        result = '<%s' % self.tagname
        if self.xmlns:
            result += ' xmlns="%s"' % self.xmlns
        for key, value in self._attr.items():
            if value is None:
                continue
            value = tostr(value)
            result += ' %s=%s' % (key, xml.sax.saxutils.quoteattr(value))
        if not self._children and not self._content:
            return result + '/>'
        result += '>'
        for child in self._children:
            if not isinstance(child, Element):
                child = child.__xml__()
            result += tostr(child)
        return result + xml.sax.saxutils.escape(self._content.strip()) + '</%s>' % self.tagname

    def __bytes__(self):
        return tobytes(self.__str__())

class ElementParser(xml.sax.ContentHandler):
    """
    Handler for the SAX parser. The member function 'handle' will be
    called everytime an element given on init is closed. The parameter
    is the tree with this element as root. An element can either have
    children or text content. The ElementParser is usefull for simple
    xml files found in config files and information like epg data.
    """
    def __init__(self, *names):
        """
        Create handler with a list of element names.
        """
        self._names = names
        self._elements = []
        self.attr = {}

    def startElement(self, name, attr):
        """
        SAX callback
        """
        element = Element(name)
        element._attr = dict(attr)
        if len(self._elements):
            self._elements[-1].append(element)
        else:
            self.attr = dict(attr)
        self._elements.append(element)

    def endElement(self, name):
        """
        SAX callback
        """
        element = self._elements.pop()
        element._content = element._content.strip()
        if name in self._names or (not self._names and len(self._elements) == 1):
            self.handle(element)
        if not self._elements:
            self.finalize()

    def characters(self, c):
        """
        SAX callback
        """
        if len(self._elements):
            self._elements[-1]._content += c

    def handle(self, element):
        """
        ElementParser callback for a complete element.
        """
        pass

    def finalize(self):
        """
        ElementParser callback at the end of parsing.
        """
        pass

def load_plugins(type, names):
    #filter = lambda name: name != 'base' and '_config' not in name
    #plugins = get_plugins(group='stagehand.'+type, location=scope['__file__'], filter=filter, scope=scope)
    from .config import config

    valid, invalid = {}, {}
    for name in names:
        try:
            module = importlib.import_module('.' + name, 'stagehand.' + type)
        except Exception as e:
            invalid[name] = e
            continue

        #name = module.__name__.split('.')[-1]
        if hasattr(module, 'modconfig'):
            getattr(config, type).add_variable(name, module.modconfig)
        valid[name] = module
        if hasattr(module, 'load'):
            module.load()
    return valid, invalid

def invoke_plugins(plugins, func, *args):
    """
    Invokes an async function in parallel on all supplied plugins.

    :param plugins: a dict of plugins (name -> plugin)
    :param func: the name of the function to invoke if it exists
    :type func: str
    :param *args: the arguments to pass to the plugin function
    :returns: Future instance
    """
    asyncfuncs = [getattr(p, func)(*args) for p in plugins.values() if hasattr(p, func)]
    return asyncio.gather(*asyncfuncs)

def fixsep(s, path=True):
    """
    Applies the configured separator policy to the given string.

    :param path: if True, return a result that can be used as a path name on the
                 filesystem (replace path separators and colons)
    """
    from .config import config
    if path:
        # TODO: fix / : " ? * < > |
        s = s.replace(os.path.sep, ' ')
        # Colons are problematic on CIFS mounts.  Even on filesystems where
        # they're allowed, they'll cause problems if the user wants to copy a
        # file to a filesystem where they're not.  e.g. Linux with an ntfs-3g
        # mount will happily let you copy a file with a colon in its name, and
        # then the directory won't be readable by an actual Windows machine.
        # So, just replace them with spaces.
        s = s.replace(':', ' ')
        # For the same reason, strip question marks.
        s = s.replace('?', '')
        # Now condense multiple spaces.
        s = re.sub(r'\s+', ' ', s)
    s = s.replace(' ', config.naming.separator)
    return s

def fixquotes(u):
    """
    Given a unicode string, replaces "smart" quotes, ellipses, etc.
    with ASCII equivalents.
    """
    if not u:
        return u
    # Double quotes
    u = u.replace('\u201c', '"').replace('\u201d', '"')
    # Single quotes
    u = u.replace('\u2018', "'").replace('\u2019', "'")
    # Endash
    u = u.replace('\u2014', '--')
    # Ellipses
    u = u.replace('\u2026', '...')
    return u

def remove_stop_words(s):
    """
    Removes (English) stop words from the given string.
    """
    stopwords = 'a', 'the'
    words = [word for word in s.split() if word not in stopwords]
    # Join remaining words and remove whitespace.
    return ' '.join(words)

def cfgdesc2html(item):
    """
    Given a config item, returns the description with paragraph breaks
    (double newline) converted to
 for use in HTML.
    """
    desc = get_description(item)
    return re.sub(r'\n\s*\n', '

', desc)

def name_to_url_segment(name):
    """
    Given some kind of name, return a lower case string without any punctuation
    or spaces, sutiable for use as a URL path segment.
    """
    name = name.lower().replace('&', 'and').replace(' ', '_')
    if name.startswith('the_'):
        name = name[4:]
    return re.sub(r'\W', '', name)

def episode_status_icon_info(ep):
    """
    Given an Episode object, returns the icon details for the episode's status.

    :params ep: Episode object
    :returns: (status, title)

    status is one of 'forced', 'have', 'ignore', 'need', or 'future' and title
    is a printable string for the state.
    """
    if ep.status == ep.STATUS_NEED_FORCED:
        return 'need-forced', 'Needed (Forced by User)'
    elif ep.status == ep.STATUS_HAVE:
        return 'have', 'Downloaded'
    elif ep.status == ep.STATUS_IGNORE or ep.obsolete or ep.season.number == 0:
        return 'ignore', 'Ignored'
    elif ep.aired:
        return 'need', 'Needed'
    else:
        return 'future', 'Not Aired'

def abspath_to_zippath(path):
    try:
        zippath = abspath_to_zippath.zippath
    except AttributeError:
        try:
            zippath = abspath_to_zippath.zippath = os.path.abspath(__loader__.archive)
        except AttributeError:
            zippath = abspath_to_zippath.zippath = None

    if zippath and path.startswith(zippath + os.path.sep):
        relpath = path[len(zippath) + 1:]
        # Zip path separator is always /, even on Windows.
        return relpath.replace(os.path.sep, '/')

def get_file_from_zip(root, filename, ims=None):
    try:
        zipfile, files = get_file_from_zip.zipfile, get_file_from_zip.files
    except AttributeError:
        get_file_from_zip.zipfile = zipfile = ZipFile(__loader__.archive)
        get_file_from_zip.files = files = dict((i.filename, i) for i in get_file_from_zip.zipfile.infolist())

    # Don't use os.path.join here because zipfile uses / as a path sep even on
    # Windows.
    filename = (root.rstrip(os.path.sep) + '/' + filename).replace(os.path.sep, '/')
    if filename not in files:
        raise FileNotFoundError

    info = files[filename]
    mtime = datetime(*info.date_time)
    if ims and datetime.utcfromtimestamp(ims) >= mtime:
        return info, mtime, None
    else:
        return info, mtime, zipfile.open(filename)
<commit_msg>Fix bug in stop words removal<commit_after>import os
import sys
import re
import codecs
from io import StringIO
import xml.sax
import xml.sax.saxutils
import asyncio
import importlib
from zipfile import ZipFile
from datetime import datetime

from .toolbox.config import get_description

class Element:
    """
    Simple XML element that can have either a text content or child
    elements. This is common for config files, Freevo fxd files, XMPP
    stanzas, and many other use cases. It is possible to access the
    attribute or the (first) child by accessing a member variable with
    that name. If the name exists as attribute and child or if there
    are several children by that name, additional helper functions are
    provided. The element's name can be accessed using the tagname
    member variable.

    A special variable is 'content'. If provided to __init__, it can
    contain text element's children (either a list of Elements or one
    Element) or the text of the element. The same is true when
    accessing Element.content. If the element has children, the
    attribute or child with that name is returned (or None if it does
    not exist). If it is a node without children, Element.content will
    refer to the text content.

    SAX parser have to use the internal variables _children, _attr,
    and _content.
    """
    def __init__(self, tagname, xmlns=None, content=None, **attr):
        self.tagname = tagname
        self.xmlns = xmlns
        self._content = ''
        self._children = []
        self._attr = attr
        if content:
            if isinstance(content, (list, tuple)):
                self._children = content
            elif isinstance(content, Element):
                self._children = [ content ]
            else:
                self._content = content

    @property
    def attributes(self):
        return self._attr.keys()

    def append(self, element):
        """
        Append an element to the list of children.
        """
        self._children.append(element)

    def add_child(self, tagname, xmlns=None, content=None, **attr):
        """
        Append an element to the list of children.
        """
        element = Element(tagname, xmlns, content, **attr)
        self._children.append(element)
        return element

    def has_child(self, name):
        """
        Return if the element has at least one child with the given element name.
        """
        return self.get_child(name) is not None

    def get_child(self, name):
        """
        Return the first child with the given name or None.
        """
        for child in self._children:
            if child.tagname == name:
                return child
        return None

    def get_children(self, name=None):
        """
        Return a list of children with the given name.
        """
        if name is None:
            return self._children[:]
        children = []
        for child in self._children:
            if child.tagname == name:
                children.append(child)
        return children

    def __iter__(self):
        """
        Iterate through the children.
        """
        return self._children.__iter__()

    def get(self, item, default=None):
        """
        Get the given attribute value or None if not set.
        """
        return self._attr.get(item, default)

    def __getitem__(self, item):
        """
        Get the given attribute value or raise a KeyError if not set.
        """
        return self._attr[item]

    def __setitem__(self, item, value):
        """
        Set the given attribute to a new value.
        """
        self._attr[item] = value

    def __getattr__(self, attr):
        """
        Magic function to return the attribute or child with the given name.
        """
        if attr == 'content' and not self._children:
            return self._content
        result = self._attr.get(attr)
        if result is not None:
            return result
        result = self.get_child(attr)
        if result is not None:
            return result
        if '_' in attr:
            return getattr(self, attr.replace('_', '-'))

    def __cmp__(self, other):
        if isinstance(other, (str, unicode)):
            return cmp(self.tagname, other)
        return object.__cmp__(self, other)

    def __repr__(self):
        """
        Python representation string
        """
        return '<Element %s>' % self.tagname

    def __str__(self):
        """
        Convert the element into an XML unicode string.
        """
        result = '<%s' % self.tagname
        if self.xmlns:
            result += ' xmlns="%s"' % self.xmlns
        for key, value in self._attr.items():
            if value is None:
                continue
            value = tostr(value)
            result += ' %s=%s' % (key, xml.sax.saxutils.quoteattr(value))
        if not self._children and not self._content:
            return result + '/>'
        result += '>'
        for child in self._children:
            if not isinstance(child, Element):
                child = child.__xml__()
            result += tostr(child)
        return result + xml.sax.saxutils.escape(self._content.strip()) + '</%s>' % self.tagname

    def __bytes__(self):
        return tobytes(self.__str__())

class ElementParser(xml.sax.ContentHandler):
    """
    Handler for the SAX parser. The member function 'handle' will be
    called everytime an element given on init is closed. The parameter
    is the tree with this element as root. An element can either have
    children or text content. The ElementParser is usefull for simple
    xml files found in config files and information like epg data.
    """
    def __init__(self, *names):
        """
        Create handler with a list of element names.
        """
        self._names = names
        self._elements = []
        self.attr = {}

    def startElement(self, name, attr):
        """
        SAX callback
        """
        element = Element(name)
        element._attr = dict(attr)
        if len(self._elements):
            self._elements[-1].append(element)
        else:
            self.attr = dict(attr)
        self._elements.append(element)

    def endElement(self, name):
        """
        SAX callback
        """
        element = self._elements.pop()
        element._content = element._content.strip()
        if name in self._names or (not self._names and len(self._elements) == 1):
            self.handle(element)
        if not self._elements:
            self.finalize()

    def characters(self, c):
        """
        SAX callback
        """
        if len(self._elements):
            self._elements[-1]._content += c

    def handle(self, element):
        """
        ElementParser callback for a complete element.
        """
        pass

    def finalize(self):
        """
        ElementParser callback at the end of parsing.
        """
        pass

def load_plugins(type, names):
    #filter = lambda name: name != 'base' and '_config' not in name
    #plugins = get_plugins(group='stagehand.'+type, location=scope['__file__'], filter=filter, scope=scope)
    from .config import config

    valid, invalid = {}, {}
    for name in names:
        try:
            module = importlib.import_module('.' + name, 'stagehand.' + type)
        except Exception as e:
            invalid[name] = e
            continue

        #name = module.__name__.split('.')[-1]
        if hasattr(module, 'modconfig'):
            getattr(config, type).add_variable(name, module.modconfig)
        valid[name] = module
        if hasattr(module, 'load'):
            module.load()
    return valid, invalid

def invoke_plugins(plugins, func, *args):
    """
    Invokes an async function in parallel on all supplied plugins.

    :param plugins: a dict of plugins (name -> plugin)
    :param func: the name of the function to invoke if it exists
    :type func: str
    :param *args: the arguments to pass to the plugin function
    :returns: Future instance
    """
    asyncfuncs = [getattr(p, func)(*args) for p in plugins.values() if hasattr(p, func)]
    return asyncio.gather(*asyncfuncs)

def fixsep(s, path=True):
    """
    Applies the configured separator policy to the given string.

    :param path: if True, return a result that can be used as a path name on the
                 filesystem (replace path separators and colons)
    """
    from .config import config
    if path:
        # TODO: fix / : " ? * < > |
        s = s.replace(os.path.sep, ' ')
        # Colons are problematic on CIFS mounts.  Even on filesystems where
        # they're allowed, they'll cause problems if the user wants to copy a
        # file to a filesystem where they're not.  e.g. Linux with an ntfs-3g
        # mount will happily let you copy a file with a colon in its name, and
        # then the directory won't be readable by an actual Windows machine.
        # So, just replace them with spaces.
        s = s.replace(':', ' ')
        # For the same reason, strip question marks.
        s = s.replace('?', '')
        # Now condense multiple spaces.
        s = re.sub(r'\s+', ' ', s)
    s = s.replace(' ', config.naming.separator)
    return s

def fixquotes(u):
    """
    Given a unicode string, replaces "smart" quotes, ellipses, etc.
    with ASCII equivalents.
    """
    if not u:
        return u
    # Double quotes
    u = u.replace('\u201c', '"').replace('\u201d', '"')
    # Single quotes
    u = u.replace('\u2018', "'").replace('\u2019', "'")
    # Endash
    u = u.replace('\u2014', '--')
    # Ellipses
    u = u.replace('\u2026', '...')
    return u

def remove_stop_words(s):
    """
    Removes (English) stop words from the given string.
    """
    stopwords = 'a', 'the'
    words = [word for word in s.split() if word.lower() not in stopwords]
    # Join remaining words and remove whitespace.
    return ' '.join(words)

def cfgdesc2html(item):
    """
    Given a config item, returns the description with paragraph breaks
    (double newline) converted to
 for use in HTML.
    """
    desc = get_description(item)
    return re.sub(r'\n\s*\n', '

', desc)

def name_to_url_segment(name):
    """
    Given some kind of name, return a lower case string without any punctuation
    or spaces, sutiable for use as a URL path segment.
    """
    name = name.lower().replace('&', 'and').replace(' ', '_')
    if name.startswith('the_'):
        name = name[4:]
    return re.sub(r'\W', '', name)

def episode_status_icon_info(ep):
    """
    Given an Episode object, returns the icon details for the episode's status.

    :params ep: Episode object
    :returns: (status, title)

    status is one of 'forced', 'have', 'ignore', 'need', or 'future' and title
    is a printable string for the state.
    """
    if ep.status == ep.STATUS_NEED_FORCED:
        return 'need-forced', 'Needed (Forced by User)'
    elif ep.status == ep.STATUS_HAVE:
        return 'have', 'Downloaded'
    elif ep.status == ep.STATUS_IGNORE or ep.obsolete or ep.season.number == 0:
        return 'ignore', 'Ignored'
    elif ep.aired:
        return 'need', 'Needed'
    else:
        return 'future', 'Not Aired'

def abspath_to_zippath(path):
    try:
        zippath = abspath_to_zippath.zippath
    except AttributeError:
        try:
            zippath = abspath_to_zippath.zippath = os.path.abspath(__loader__.archive)
        except AttributeError:
            zippath = abspath_to_zippath.zippath = None

    if zippath and path.startswith(zippath + os.path.sep):
        relpath = path[len(zippath) + 1:]
        # Zip path separator is always /, even on Windows.
        return relpath.replace(os.path.sep, '/')

def get_file_from_zip(root, filename, ims=None):
    try:
        zipfile, files = get_file_from_zip.zipfile, get_file_from_zip.files
    except AttributeError:
        get_file_from_zip.zipfile = zipfile = ZipFile(__loader__.archive)
        get_file_from_zip.files = files = dict((i.filename, i) for i in get_file_from_zip.zipfile.infolist())

    # Don't use os.path.join here because zipfile uses / as a path sep even on
    # Windows.
    filename = (root.rstrip(os.path.sep) + '/' + filename).replace(os.path.sep, '/')
    if filename not in files:
        raise FileNotFoundError

    info = files[filename]
    mtime = datetime(*info.date_time)
    if ims and datetime.utcfromtimestamp(ims) >= mtime:
        return info, mtime, None
    else:
        return info, mtime, zipfile.open(filename)
<|endoftext|>


<commit_before>from public_records_portal import models, scribd_helpers
from models import Record, Note

class ResponsePresenter:
	def __init__(self, record = None, note = None):
		if record:
			self.response = record
			self.update_url = "update_a_record_delete"
			if self.response.access:
				self.type = "offline"
			elif self.response.doc_id:
				self.type = "document"
			else:
				self.type = "link"
		if note:
			self.response = note
			self.update_url = "update_a_note_delete"
			self.type = "note"
			if "Request extended" in self.response.text:
				self.type = "extension"
		if self.type=="offline":
			self.icon = "icon-file-alt icon-large"
		elif self.type=="note":
			self.icon = "icon-edit icon-large"
		elif self.type=="link":
			self.icon = "icon-link icon-large"
		elif self.type =="document":
			self.icon = "icon-file-alt icon-large"
		elif self.type=="extension":
			self.icon = "icon-calendar icon-large"

	def get_update_url(self):
		return self.update_url

	def get_id(self):
		return self.response.id

	def uid(self):
		return self.response.user_id

	def staff_name(self):
		return get_attribute(attribute = "alias", obj_id = self.response.uid, obj_type = "User")

	def staff_dept(self):
		return get_attribute(attribute = "department", obj_id = self.response.uid, obj_type = "User")

	def staff_phone(self):
		return get_attribute(attribute = "phone", obj_id = self.response.uid, obj_type = "User")

	def staff_email(self):
		return get_attribute(attribute = "email", obj_id = self.response.uid, obj_type = "User")

	def display_text(self):
		if self.type == "offline":
			return "Name of Record: %s
 How to Access Record: %s" %(self.response.description, self.response.access)
		elif self.type == "document":
			download_url = self.response.download_url
			if not download_url:
				download_url = scribd_helpers.get_scribd_download_url(doc_id = self.response.doc_id, record_id = self.response.id)
			return """
			%s

			""" % (self.response.url, self.response.url, self.response.description, download_url, download_url)
		elif self.type == "note":
			return self.response.text
		elif self.type == "link":
			return "%s " % (self.response.url, self.response.url, self.response.description)
		elif self.type == "extension":
			junk, text = self.response.text.split(":")
			return text

	def get_icon(self):
		return self.icon

	def set_icon(self, icon):
		self.icon = icon

	def date(self):
		return self.response.date_created

<commit_msg>moved link hover text from right to top<commit_after>from public_records_portal import models, scribd_helpers
from models import Record, Note

class ResponsePresenter:
	def __init__(self, record = None, note = None):
		if record:
			self.response = record
			self.update_url = "update_a_record_delete"
			if self.response.access:
				self.type = "offline"
			elif self.response.doc_id:
				self.type = "document"
			else:
				self.type = "link"
		if note:
			self.response = note
			self.update_url = "update_a_note_delete"
			self.type = "note"
			if "Request extended" in self.response.text:
				self.type = "extension"
		if self.type=="offline":
			self.icon = "icon-file-alt icon-large"
		elif self.type=="note":
			self.icon = "icon-edit icon-large"
		elif self.type=="link":
			self.icon = "icon-link icon-large"
		elif self.type =="document":
			self.icon = "icon-file-alt icon-large"
		elif self.type=="extension":
			self.icon = "icon-calendar icon-large"

	def get_update_url(self):
		return self.update_url

	def get_id(self):
		return self.response.id

	def uid(self):
		return self.response.user_id

	def staff_name(self):
		return get_attribute(attribute = "alias", obj_id = self.response.uid, obj_type = "User")

	def staff_dept(self):
		return get_attribute(attribute = "department", obj_id = self.response.uid, obj_type = "User")

	def staff_phone(self):
		return get_attribute(attribute = "phone", obj_id = self.response.uid, obj_type = "User")

	def staff_email(self):
		return get_attribute(attribute = "email", obj_id = self.response.uid, obj_type = "User")

	def display_text(self):
		if self.type == "offline":
			return "Name of Record: %s
 How to Access Record: %s" %(self.response.description, self.response.access)
		elif self.type == "document":
			download_url = self.response.download_url
			if not download_url:
				download_url = scribd_helpers.get_scribd_download_url(doc_id = self.response.doc_id, record_id = self.response.id)
			return """
			%s

			""" % (self.response.url, self.response.url, self.response.description, download_url, download_url)
		elif self.type == "note":
			return self.response.text
		elif self.type == "link":
			return "%s " % (self.response.url, self.response.url, self.response.description)
		elif self.type == "extension":
			junk, text = self.response.text.split(":")
			return text

	def get_icon(self):
		return self.icon

	def set_icon(self, icon):
		self.icon = icon

	def date(self):
		return self.response.date_created

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import pyjobs_crawlers.run

import crawlers
import pyjobsweb.commands

class DryRunCrawlCommand(pyjobsweb.commands.AppContextCommand):
    def get_parser(self, prog_name):

        parser = super(DryRunCrawlCommand, self).get_parser(prog_name)

        parser.add_argument(
                "-s", "--spider",
                help='Spider name.',
                dest='spider_name'
        )

        parser.add_argument(
                "-t", "--type",
                help='Mandatory value : (list|offer).',
                dest='page_type'
        )

        parser.add_argument(
                "-u", "--url",
                help='Page url.',
                dest='url'
        )

        return parser

    def take_action(self, parsed_args):
        super(DryRunCrawlCommand, self).take_action(parsed_args)
        pyjobs_crawlers.run.start_crawler(
                connector_class=crawlers.PyJobsWebDryRunCrawlConnector(),
                spider_name=parsed_args.spider_name
                #type=parsed_args.type,
                #url=parsed_args.url
        )
<commit_msg>Code style<commit_after># -*- coding: utf-8 -*-
import pyjobs_crawlers.run

import crawlers
import pyjobsweb.commands

class DryRunCrawlCommand(pyjobsweb.commands.AppContextCommand):
    def get_parser(self, prog_name):

        parser = super(DryRunCrawlCommand, self).get_parser(prog_name)

        parser.add_argument("-s", "--spider",
                            help='Spider name.', dest='spider_name')

        parser.add_argument("-t", "--type",
                            help='Mandatory value : (list|offer).',
                            dest='page_type')

        parser.add_argument("-u", "--url", help='Page url.', dest='url')

        return parser

    def take_action(self, parsed_args):
        super(DryRunCrawlCommand, self).take_action(parsed_args)
        pyjobs_crawlers.run.start_crawler(
                connector_class=crawlers.PyJobsWebDryRunCrawlConnector(),
                spider_name=parsed_args.spider_name
                #type=parsed_args.type,
                #url=parsed_args.url
        )
<|endoftext|>