<commit_before>## Python control script for 4x4x4 LED Cube
## John Caswell, 2014

import serial
import time
import struct
import random
import threading
from bottle import route, get, post, request, run, redirect, static_file

## Configuration:  set these to match your setup
interface  = "0.0.0.0"       # Interface to listen on, 0.0.0.0 is all interfaces
port       = 8080            # Port to listen on
serialPort = "/dev/ttyUSB0"  # Serial port where LED cube is connected
baud       = 9600            # Baud rate of cube, 9600 is default
timeout    = 30              # How long to run animations in sequence
serverHome = "/home/pi/Projects/LEDCube" # Path to installation
## End config

## Globals
matrix = 0x0000000000000000
animation = "beamdown"
status = "stopped"
starting = True
modes = ["beamdown", "bounce", "crazy", "helix", "innerouter",\
         "raindrops", "spin", "twinkle", "sequence", "random"]

## Attempt to open connection to cube
try:
    arduino = serial.Serial(serialPort, baud)
    time.sleep(2)
except:
    print("Error opening serial port, quitting.")
    quit()

## Cube control thread, this handles interfacing with the cube
## It is controlled by the status, starting, animation, and matrix globals
## If status is running, it will draw a frame of animation and yield
def cube():
    global matrix
    global starting
    mask = [0x8001, 0x810, 0x180, 0x1008, 0x2004, 0x4002]
    i = 0
    while True:
        if status == "running":
            ## Light each plane in order, faster and faster until we fill the cube
            if animation == "beamdown":
                if starting:
                    plane = 0xffff
                    sleeptime = 1
                    starting = False
                    done = False
                if plane < 0x10000000000000000 and not done:
                    arduino.write(struct.pack("Q", plane))
                    if sleeptime > 0.016:
                        time.sleep(sleeptime)
                        plane = plane * 0x10000
                    else:
                        arduino.write(struct.pack("<Q", 0xffffffffffffffff))
                        done = True
                elif not done:
                    plane = 0xffff
                    sleeptime = sleeptime / 2.0
                else:
                    time.sleep(0.001)
            ## Just randomly set the states of LEDs
            elif animation == "crazy":
                arduino.write(struct.pack("<Q", random.randint(0, 0xffffffffffffffff)))
                time.sleep(0.5)
            ## Raindrops trickle from the top layer down
            elif animation == "raindrops":
                drops = random.randint(0, 2)
                for x in range(0, drops):
                    matrix += 1 << random.randint(0, 15)
                arduino.write(struct.pack("<Q", matrix))
                matrix *= 0x10000
                matrix &= 0xffffffffffffffff
                time.sleep(0.15)
            ## Rotating double helix
            elif animation == "helix":
                if starting:
                    i = 0
                    starting = False
                arduino.write(struct.pack("<Q", matrix))
                matrix = matrix >> 16
                matrix |= (mask[i] << 48)
                i += 1
                if i > 5:
                    i = 0
                time.sleep(0.125)
            ## Alternate inner and outer cube wireframe
            elif animation == "innerouter":
                arduino.write(struct.pack("<Q", 0x66006600000))
                time.sleep(1)
                arduino.write(struct.pack("<Q", 0xf99f90099009f99f))
                time.sleep(1)
        else:
            time.sleep(0.001)

## Create the thread to control the cube and start it
t = threading.Thread(target = cube)
t.setDaemon(True)
t.start()

## Handle main control buttons
@route('/start')
def start():
    global status
    status = "running"
    redirect("/")

@route('/stop')
def stop():
    global matrix
    global status
    global starting
    starting = True
    status = "stopped"
    matrix = 0x0000000000000000
    arduino.write(struct.pack("<Q", matrix))
    redirect("/")

@route('/pause')
def pause():
    global status
    status = "paused"
    redirect("/")

## Add path for images
@route('/images/<filename>')
def images(filename):
    return static_file(filename, root=serverHome + "/images/")

## Define main page
@get('/')
def root():
    header = '''<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">

<meta http-equiv="Content-type" content="text/html;charset=UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>LED Cube Control</title>

'''
    body = '''
<map name="buttonmap">
<area shape="circle" coords="108,112,34" href="/pause" alt="Pause">
<area shape="circle" coords="195,112,34" href="/start" alt="Play">
<area shape="circle" coords="277,112,34" href="/stop" alt="Stop">
</map>
<form action="" method="post">
<select name="animation">
'''
    formlist= ""
    for mode in modes:
        formlist += "<option"
        if mode == animation:
            formlist += " selected=\"selected\""
        formlist += ">" + mode + "</option>\n"
    footer='''</select>
<input type="image" src="/images/go.png">
</form>

'''
    response = header
    response += "current animation:  "
    response += animation + " "
    response += status + "
\n"
    response += body
    response += formlist
    response += footer
    return response

## Handle animation selection form
@post('/')
def form():
    global animation
    global matrix
    global starting
    starting = True
    animation = request.forms.get('animation')
    matrix = 0x0000000000000000
    return start()

## Start HTTP Server
run(host=interface, port=port)

<commit_msg>Refactored Python script to clean it up visually, and simplify adding new animations.<commit_after>## Python control script for 4x4x4 LED Cube
## John Caswell, 2014

import serial
import time
import struct
import random
import threading
from bottle import route, get, post, request, run, redirect, static_file

## Configuration:  set these to match your setup
interface  = "0.0.0.0"       # Interface to listen on, 0.0.0.0 is all interfaces
port       = 8080            # Port to listen on
serialPort = "/dev/ttyUSB0"  # Serial port where LED cube is connected
baud       = 9600            # Baud rate of cube, 9600 is default
timeout    = 30              # How long to run animations in sequence
serverHome = "/home/pi/Projects/LEDCube" # Path to installation
## End config

## Globals
matrix = 0x0000000000000000
status = "stopped"
starting = True
modes = ["beamdown", "bounce", "crazy", "helix", "innerouter",\
         "raindrops", "spin", "twinkle", "_sequence", "_random"]
animation = modes[0]

## Attempt to open connection to cube
try:
    arduino = serial.Serial(serialPort, baud)
    time.sleep(2)
except:
    print("Error opening serial port, quitting.")
    quit()

## Animation functions.  These should perform a single frame of animation
## and then return.  They must be named the same as the strings in modes[].
def beamdown():
    ## Light each plane in order, faster and faster until we fill the cube
    global starting
    if not hasattr(beamdown, "plane"):
        beamdown.plane = 0xffff
    if not hasattr(beamdown, "sleeptime"):
        beamdown.sleeptime = 1
    if not hasattr(beamdown, "done"):
        beamdown.done = False
    if starting:
        beamdown.plane = 0xffff
        beamdown.sleeptime = 1
        starting = False
        beamdown.done = False
    if beamdown.plane < 0x10000000000000000 and not beamdown.done:
        arduino.write(struct.pack("Q", beamdown.plane))
        if beamdown.sleeptime > 0.016:
            time.sleep(beamdown.sleeptime)
            beamdown.plane = beamdown.plane * 0x10000
        else:
            arduino.write(struct.pack("<Q", 0xffffffffffffffff))
            beamdown.done = True
    elif not beamdown.done:
        beamdown.plane = 0xffff
        beamdown.sleeptime = beamdown.sleeptime / 2.0
    else:
        time.sleep(0.001)

def bounce():
    pass

def crazy():
    ## Just randomly set the states of LEDs
    arduino.write(struct.pack("<Q", random.randint(0, 0xffffffffffffffff)))
    time.sleep(0.5)

def helix():
    ## Rotating double helix
    global starting
    global matrix
    if not hasattr(helix, "i"):
        helix.i = 0
    if not hasattr(helix, "mask"):
        helix.mask = [0x8001, 0x810, 0x180, 0x1008, 0x2004, 0x4002]
    if starting:
        helix.i = 0
        starting = False
    arduino.write(struct.pack("<Q", matrix))
    matrix = matrix >> 16
    matrix |= (helix.mask[helix.i] << 48)
    helix.i += 1
    if helix.i > 5:
        helix.i = 0
    time.sleep(0.125)

def innerouter():
    ## Alternate inner and outer cube wireframe
    arduino.write(struct.pack("<Q", 0x66006600000))
    time.sleep(1)
    arduino.write(struct.pack("<Q", 0xf99f90099009f99f))
    time.sleep(1)

def raindrops():
    ## Raindrops trickle from the top layer down
    global matrix
    drops = random.randint(0, 2)
    for x in range(0, drops):
        matrix += 1 << random.randint(0, 15)
    arduino.write(struct.pack("<Q", matrix))
    matrix *= 0x10000
    matrix &= 0xffffffffffffffff
    time.sleep(0.15)

def spin():
    pass

def twinkle():
    ## Twinkle on and off randomly, switching when cube is full / empty
    global starting
    global matrix
    if not hasattr(twinkle, "toggle"):
        twinkle.toggle = True
    if starting:
        twinkle.toggle = True
        starting = False
    if twinkle.toggle:
        if matrix < 0xffffffffffffffff:
            twink = 1 << random.randint(0, 63)
            if not twink & matrix:
                matrix |= twink
                arduino.write(struct.pack("<Q", matrix))
                time.sleep(0.15)
        else:
            twinkle.toggle = False
            time.sleep(0.5)
    else:
        if matrix > 0:
            twink = 1 << random.randint(0, 63)
            if matrix & twink:
                matrix &= matrix ^ twink
                arduino.write(struct.pack("Q", matrix))
                time.sleep(0.15)
        else:
            twinkle.toggle = True
            time.sleep(0.5)

def _sequence():
    pass

def _random():
    pass

## Cube control thread, this handles interfacing with the cube
## It is controlled by the status, starting, animation, and matrix globals
## If status is running, it will call the named animation and yield
def cube():
    while True:
        if status == "running":
            globals()[animation]()  # Call the function named in animation
        else:
            time.sleep(0.001)

## Create the thread to control the cube and start it
t = threading.Thread(target = cube)
t.setDaemon(True)
t.start()

## Handle main control buttons
@route('/start')
def start():
    global status
    status = "running"
    redirect("/")

@route('/stop')
def stop():
    global matrix
    global status
    global starting
    starting = True
    status = "stopped"
    matrix = 0x0000000000000000
    arduino.write(struct.pack("<Q", matrix))
    redirect("/")

@route('/pause')
def pause():
    global status
    status = "paused"
    redirect("/")

## Add path for images
@route('/images/<filename>')
def images(filename):
    return static_file(filename, root=serverHome + "/images/")

## Define main page
@get('/')
def root():
    header = '''<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">

<meta http-equiv="Content-type" content="text/html;charset=UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>LED Cube Control</title>

'''
    body = '''
<map name="buttonmap">
<area shape="circle" coords="108,112,34" href="/pause" alt="Pause">
<area shape="circle" coords="195,112,34" href="/start" alt="Play">
<area shape="circle" coords="277,112,34" href="/stop" alt="Stop">
</map>
<form action="" method="post">
<select name="animation">
'''
    formlist= ""
    for mode in modes:
        formlist += "<option"
        if mode == animation:
            formlist += " selected=\"selected\""
        formlist += ">" + mode + "</option>\n"
    footer='''</select>
<input type="image" src="/images/go.png">
</form>

'''
    response = header
    response += "current animation:  "
    response += animation + " "
    response += status + "
\n"
    response += body
    response += formlist
    response += footer
    return response

## Handle animation selection form
@post('/')
def form():
    global animation
    global matrix
    global starting
    starting = True
    animation = request.forms.get('animation')
    matrix = 0x0000000000000000
    return start()

## Start HTTP Server
run(host=interface, port=port)

<|endoftext|>


<commit_before>import os
from setuptools import setup

package_data = {}
start_dir = os.getcwd()

def add_file_for_package(package, subdir, f):
    full_path = os.path.join(subdir, f)
    #print "%s: %s" % (package, full_path)
    return full_path

INSTALL_ONLY_DEPENDENCIES = 'INSTALL_ONLY_DEPENDENCIES' in os.environ

if 'INSTALL_ONLY_DEPENDENCIES' in os.environ:
    packages = []
    package_data = {}
    package_scripts = []
else:
    package_scripts = ["rdrf-manage.py"]

    packages = [ 'rdrf',
                 'registry',
                 'registry.common',
                 'registry.patients',
                 'registry.groups',
                 'registry.genetic',
                 'explorer',
               ]

    for package in ['rdrf', 'registry.common', 'registry.genetic',
                'registry.groups', 'registry.patients', 'registry.humangenome', 'explorer']:
        package_data[package] = []
        if "." in package:
            base_dir,package_dir = package.split(".")
            os.chdir(os.path.join(start_dir,base_dir, package_dir))
        else:
            base_dir = package
            os.chdir(os.path.join(start_dir, base_dir))

        for data_dir in ('templates', 'static', 'migrations', 'fixtures', 'features', 'hooks', 'templatetags', 'management'):
            package_data[package].extend(
                [ add_file_for_package(package, subdir, f) for (subdir, dirs, files) in os.walk(data_dir) for f in files])

        os.chdir(start_dir)

setup(name='django-rdrf',
    version="0.8.34",
    packages=packages,
    description='RDRF',
    long_description='Rare Disease Registry Framework',
    author='Centre for Comparative Genomics',
    author_email='web@ccg.murdoch.edu.au',
    package_data= package_data,
    zip_safe=False,
    scripts=package_scripts,
    install_requires=[
        'Django==1.6.11',
        'pymongo==2.8',
        'pyyaml',
        'South==0.8.2',
        'django-extensions>=0.7.1',
        'django-picklefield==0.1.9',
        'django-templatetag-sugar==0.1',
        'pyparsing==1.5.6',
        'wsgiref==0.1.2',
        'python-memcached==1.54',
        'django-extensions>=0.7.1',
        'django-messages-ui==0.2.6',
        'ccg-django-utils==0.3.1',
        'django-userlog==0.2.1',
        'django-nose',
        'sure==1.2.1',
        'django-templatetag-handlebars==1.2.0',
        'django-iprestrict==0.1',
        'django-suit',
        'django-ajax-selects',
        'pysam==0.8.1',
        'hgvs==0.2.1',
        'django-positions',
        'django-tastypie==0.12.1',
        'pycountry==1.12',
        'django-countries',
        'hgtools',
        'nose',
        'nose-timer',
        'sphinx',
        'sphinxcontrib-fulltoc',
        'uwsgi==2.0.10',
        'uwsgitop',
        'pyinotify==0.9.4',
        'Werkzeug',
        'six==1.9.0',
        'python-gettext',
        'django-registration-redux==1.1.0',
        'psycopg2==2.6.1',
        'django-secure==1.0.1',
    ],
    dependency_links=[
        "https://bitbucket.org/ccgmurdoch/ccg-django-utils/downloads/ccg-django-utils-0.3.1.tar.gz",
        "https://bitbucket.org/ccgmurdoch/django-userlog/downloads/django_userlog-0.2.1.tar.gz",
        "https://bitbucket.org/ccgmurdoch/ccg-django-extras/downloads/django-iprestrict-0.1.tar.gz",
    ],
)
<commit_msg>hgvs==0.2.2<commit_after>import os
from setuptools import setup

package_data = {}
start_dir = os.getcwd()

def add_file_for_package(package, subdir, f):
    full_path = os.path.join(subdir, f)
    #print "%s: %s" % (package, full_path)
    return full_path

INSTALL_ONLY_DEPENDENCIES = 'INSTALL_ONLY_DEPENDENCIES' in os.environ

if 'INSTALL_ONLY_DEPENDENCIES' in os.environ:
    packages = []
    package_data = {}
    package_scripts = []
else:
    package_scripts = ["rdrf-manage.py"]

    packages = [ 'rdrf',
                 'registry',
                 'registry.common',
                 'registry.patients',
                 'registry.groups',
                 'registry.genetic',
                 'explorer',
               ]

    for package in ['rdrf', 'registry.common', 'registry.genetic',
                'registry.groups', 'registry.patients', 'registry.humangenome', 'explorer']:
        package_data[package] = []
        if "." in package:
            base_dir,package_dir = package.split(".")
            os.chdir(os.path.join(start_dir,base_dir, package_dir))
        else:
            base_dir = package
            os.chdir(os.path.join(start_dir, base_dir))

        for data_dir in ('templates', 'static', 'migrations', 'fixtures', 'features', 'hooks', 'templatetags', 'management'):
            package_data[package].extend(
                [ add_file_for_package(package, subdir, f) for (subdir, dirs, files) in os.walk(data_dir) for f in files])

        os.chdir(start_dir)

setup(name='django-rdrf',
    version="0.8.34",
    packages=packages,
    description='RDRF',
    long_description='Rare Disease Registry Framework',
    author='Centre for Comparative Genomics',
    author_email='web@ccg.murdoch.edu.au',
    package_data= package_data,
    zip_safe=False,
    scripts=package_scripts,
    install_requires=[
        'Django==1.6.11',
        'pymongo==2.8',
        'pyyaml',
        'South==0.8.2',
        'django-extensions>=0.7.1',
        'django-picklefield==0.1.9',
        'django-templatetag-sugar==0.1',
        'pyparsing==1.5.6',
        'wsgiref==0.1.2',
        'python-memcached==1.54',
        'django-extensions>=0.7.1',
        'django-messages-ui==0.2.6',
        'ccg-django-utils==0.3.1',
        'django-userlog==0.2.1',
        'django-nose',
        'sure==1.2.1',
        'django-templatetag-handlebars==1.2.0',
        'django-iprestrict==0.1',
        'django-suit',
        'django-ajax-selects',
        'pysam==0.8.1',
        'hgvs==0.2.2',
        'django-positions',
        'django-tastypie==0.12.1',
        'pycountry==1.12',
        'django-countries',
        'hgtools',
        'nose',
        'nose-timer',
        'sphinx',
        'sphinxcontrib-fulltoc',
        'uwsgi==2.0.10',
        'uwsgitop',
        'pyinotify==0.9.4',
        'Werkzeug',
        'six==1.9.0',
        'python-gettext',
        'django-registration-redux==1.1.0',
        'psycopg2==2.6.1',
        'django-secure==1.0.1',
    ],
    dependency_links=[
        "https://bitbucket.org/ccgmurdoch/ccg-django-utils/downloads/ccg-django-utils-0.3.1.tar.gz",
        "https://bitbucket.org/ccgmurdoch/django-userlog/downloads/django_userlog-0.2.1.tar.gz",
        "https://bitbucket.org/ccgmurdoch/ccg-django-extras/downloads/django-iprestrict-0.1.tar.gz",
    ],
)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import struct

import numpy

def step(index, format):
    return index + struct.calcsize(format)

def stepstring(file, index, length=None):
    if length is None:
        length = file[index]
        index += 1
        if length == 255:
            length = file[index : index + 4].view(numpy.uint32)
            index += 4
    return index + length

def string(file, index, length=None):
    if length is None:
        length = file[index]
        index += 1
        if length == 255:
            length = file[index : index + 4].view(numpy.uint32)
            index += 4
    return file[index : index + length].tostring()

def field(file, index, format):
    return struct.unpack(format, file[index : index + struct.calcsize(format)])[0]

def fields(file, index, format):
    return struct.unpack(format, file[index : index + struct.calcsize(format)])

def databytes(file, index, dtype, bytes):
    return file[index : index + bytes].view(dtype)

def dataitems(file, index, dtype, items):
    if not isinstance(dtype, numpy.dtype):
        dtype = numpy.dtype(dtype)
    return file[index : index + dtype.itemsize * items].view(dtype)

class TFile(object):
    def __init__(self, filepath):
        self.filepath = filepath
        self.file = numpy.memmap(self.filepath, mode="r")

        index = 0

        if field(self.file, index, "!4s") != "root":
            raise IOError("not a ROOT file (wrong magic bytes)")
        index = step(index, "!4s")

        self.version, self.begin = fields(self.file, index, "!ii")
        index = step(index, "!ii")

        if self.version < 1000000:  # small file
            self.end, self.seekfree, self.nbytesfree, self.nfree, self.nbytesname, self.units, self.compression, self.seekinfo, self.nbytesinfo = fields(self.file, index, "!iiiiiBiii")
            index = step(index, "!iiiiiBiii")
        else:
            self.end, self.seekfree, self.nbytesfree, self.nfree, self.nbytesname, self.units, self.compression, self.seekinfo, self.nbytesinfo = fields(self.file, index, "!qqiiiBiqi")
            index = step(index, "!qqiiiBiqi")
        self.version %= 1000000

        self.uuid = field(self.file, index, "!18s")
        index = step(index, "!18s")

        recordSize = 2 + 4 + 4 + 4 + 4   # fVersion, ctime, mtime, nbyteskeys, nbytesname
        if self.version >= 40000:
            recordSize += 8 + 8 + 8      # seekdir, seekparent, seekkeys
        else:
            recordSize += 4 + 4 + 4      # seekdir, seekparent, seekkeys

        nbytes = self.nbytesname + recordSize

        if nbytes + self.begin > self.end:
            raise IOError("TDirectory header length")

        self.dir = TDirectory(self.file, self.begin, self.nbytesname)

    def __repr__(self):
        return "<TFile {0} at 0x{1:012x}>".format(repr(self.filepath), id(self))

    def get(self, name):
        self.dir.get(name)

class TDirectory(object):
    def __init__(self, file, begin, nbytesname):
        self.file = file
        index = begin + nbytesname

        self.version, self.ctime, self.mtime = fields(self.file, index, "!hII")
        index = step(index, "!hII")

        self.nbyteskeys, self.nbytesname = fields(self.file, index, "!ii")
        index = step(index, "!ii")

        if self.version <= 1000:
            self.seekdir, self.seekparent, self.seekkeys = fields(self.file, index, "!iii")
        else:
            self.seekdir, self.seekparent, self.seekkeys = fields(self.file, index, "!qqq")

        nk = 4
        keyversion = field(self.file, begin + nk, "!h")
        if keyversion > 1000:
            nk += 2 + 2*4 + 2*2 + 2*8  # fVersion, fObjectSize*Date, fKeyLength*fCycle, fSeekKey*fSeekParentDirectory
        else:
            nk += 2 + 2*4 + 2*2 + 2*4  # fVersion, fObjectSize*Date, fKeyLength*fCycle, fSeekKey*fSeekParentDirectory

        index = begin + nk
        self.classname = string(self.file, index)
        index = stepstring(self.file, index)
        self.name = string(self.file, index)
        index = stepstring(self.file, index)
        self.title = string(self.file, index)
        index = stepstring(self.file, index)

        if not 10 <= self.nbytesname <= 1000:
            raise IOError("directory info")

        self.keys = TKeys(self.file, self.seekkeys)

    def __repr__(self):
        return "<TDirectory {0} at 0x{1:012x}>".format(repr(self.name), id(self))

    def get(self, name):
        # FIXME: parse slashes and get subdirectories
        self.keys.get(name)

class TKeys(object):
    def __init__(self, file, index):
        self.file = file
        self.header = TKey(self.file, index)

        index += self.header.keylen
        nkeys = field(self.file, index, "!i")
        index = step(index, "!i")

        self.keys = []
        for i in range(nkeys):
            self.keys.append(TKey(self.file, index))
            index = self.keys[-1]._index

    def __repr__(self):
        return "<TKeys len={0} at 0x{1:012x}>".format(len(self.keys), id(self))

    def __getitem__(self, i):
        return self.keys[i]

    def get(self, name):
        for key in self.keys:
            if key.name == name:
                return key.get()
        raise KeyError("not found: {0}".format(repr(name)))

class TKey(object):
    classes = {}

    def __init__(self, file, index):
        self.file = file

        self.bytes, self.version, self.objlen, self.datetime, self.keylen, self.cycle = fields(self.file, index, "!ihiIhh")
        index = step(index, "!ihiIhh")

        if self.version > 1000:
            self.seekkey, self.seekpdir = fields(self.file, index, "!qq")
            index = step(index, "!qq")
        else:
            self.seekkey, self.seekpdir = fields(self.file, index, "!ii")
            index = step(index, "!ii")

        self.classname = string(self.file, index)
        index = stepstring(self.file, index)
        self.name = string(self.file, index)
        index = stepstring(self.file, index)
        self.title = string(self.file, index)
        self._index = stepstring(self.file, index)

    def __repr__(self):
        return "<TKey {0} at 0x{1:012x}>".format(repr(self.name), id(self))

    @property
    def isCompressed(self):
        return self.objlen != self.bytes - self.keylen

    def get(self):
        if self.classname not in self.classes:
            raise NotImplementedError("class not recognized: {0}".format(self.classname))

        if self.isCompressed:
            raise NotImplementedError

        start = self.seekkey + self.keylen
        return self.classes[self.classname](self.file, start)

def readversion(file, index):
    bcnt, vers = fields(file, index, "!IH")
    index = step(index, "!IH")
    bcnt = int(numpy.int64(bcnt) & ~readversion.kByteCountMask)
    if bcnt == 0:
        raise IOError("readversion byte count is zero")
    return index, (vers, bcnt)
readversion.kByteCountMask = numpy.int64(0x40000000)

def skipversion(file, index):
    version = field(file, index, "!h")
    index = step(index, "!h")
    if numpy.int64(version) & skipversion.kByteCountVMask:
        index = step(index, "!hh")
    return index
skipversion.kByteCountVMask = numpy.int64(0x4000)

def skiptobject(file, index):
    id, bits = fields(file, index, "!II")
    index = step(index, "!II")
    bits = numpy.uint32(bits) | skiptobject.kIsOnHeap
    if bits & skiptobject.kIsReferenced:
        index = step(index, "!H")
    return index
skiptobject.kIsOnHeap = numpy.uint32(0x01000000)
skiptobject.kIsReferenced = numpy.uint32(1 << 4)

class TTree(object):
    def __init__(self, file, index):
        self.file = file

        ttree_start = index
        index, (ttree_vers, ttree_bcnt) = readversion(self.file, index)

        if ttree_vers < 16:
            raise NotImplementedError("TTree too old")

        # START TNamed
        tnamed_start = index
        index, (tnamed_vers, tnamed_bcnt) = readversion(self.file, index)

        index = skipversion(self.file, index)
        index = skiptobject(self.file, index)

        self.name = repr(string(self.file, index))
        index = stepstring(self.file, index)
        self.title = repr(string(self.file, index))
        index = stepstring(self.file, index)

        if index - tnamed_start != tnamed_bcnt + 4:
            raise IOError("TNamed byte count")
        # END TNamed

        # START TAttLine
        tattline_start = index
        index, (tattline_vers, tattline_bcnt) = readversion(self.file, index)
        index = step(index, "!hhh")  # color, style, width
        if index - tattline_start != tattline_bcnt + 4:
            raise IOError("TAttLine byte count")
        # END TAttLine

        # START TAttFill
        tattfill_start = index
        index, (tattfill_vers, tattfill_bcnt) = readversion(self.file, index)
        index = step(index, "!hh")  # color, style
        if index - tattfill_start != tattfill_bcnt + 4:
            raise IOError("TAttFill byte count")
        # END TAttFill

        # START TAttMarker
        tattmarker_start = index
        index, (tattmarker_vers, tattmarker_bcnt) = readversion(self.file, index)
        index = step(index, "!hhf")  # color, style, width
        if index - tattmarker_start != tattmarker_bcnt + 4:
            raise IOError("TAttMarker byte count")
        # END TAttMarker

TKey.classes["TTree"] = TTree

file = TFile("/home/pivarski/storage/data/TrackResonanceNtuple_uncompressed.root")
print file.get("twoMuon")
<commit_msg>checkpoint: at TBranch TObjArray<commit_after>#!/usr/bin/env python

import struct

import numpy

def step(index, format):
    return index + struct.calcsize(format)

def stepstring(file, index, length=None):
    if length is None:
        length = file[index]
        index += 1
        if length == 255:
            length = file[index : index + 4].view(numpy.uint32)
            index += 4
    return index + length

def string(file, index, length=None):
    if length is None:
        length = file[index]
        index += 1
        if length == 255:
            length = file[index : index + 4].view(numpy.uint32)
            index += 4
    return file[index : index + length].tostring()

def field(file, index, format):
    return struct.unpack(format, file[index : index + struct.calcsize(format)])[0]

def fields(file, index, format):
    return struct.unpack(format, file[index : index + struct.calcsize(format)])

def databytes(file, index, dtype, bytes):
    return file[index : index + bytes].view(dtype)

def dataitems(file, index, dtype, items):
    if not isinstance(dtype, numpy.dtype):
        dtype = numpy.dtype(dtype)
    return file[index : index + dtype.itemsize * items].view(dtype)

class TFile(object):
    def __init__(self, filepath):
        self.filepath = filepath
        self.file = numpy.memmap(self.filepath, mode="r")

        index = 0

        if field(self.file, index, "!4s") != "root":
            raise IOError("not a ROOT file (wrong magic bytes)")
        index = step(index, "!4s")

        self.version, self.begin = fields(self.file, index, "!ii")
        index = step(index, "!ii")

        if self.version < 1000000:  # small file
            self.end, self.seekfree, self.nbytesfree, self.nfree, self.nbytesname, self.units, self.compression, self.seekinfo, self.nbytesinfo = fields(self.file, index, "!iiiiiBiii")
            index = step(index, "!iiiiiBiii")
        else:
            self.end, self.seekfree, self.nbytesfree, self.nfree, self.nbytesname, self.units, self.compression, self.seekinfo, self.nbytesinfo = fields(self.file, index, "!qqiiiBiqi")
            index = step(index, "!qqiiiBiqi")
        self.version %= 1000000

        self.uuid = field(self.file, index, "!18s")
        index = step(index, "!18s")

        recordSize = 2 + 4 + 4 + 4 + 4   # fVersion, ctime, mtime, nbyteskeys, nbytesname
        if self.version >= 40000:
            recordSize += 8 + 8 + 8      # seekdir, seekparent, seekkeys
        else:
            recordSize += 4 + 4 + 4      # seekdir, seekparent, seekkeys

        nbytes = self.nbytesname + recordSize

        if nbytes + self.begin > self.end:
            raise IOError("TDirectory header length")

        self.dir = TDirectory(self.file, self.begin, self.nbytesname)

    def __repr__(self):
        return "<TFile {0} at 0x{1:012x}>".format(repr(self.filepath), id(self))

    def get(self, name):
        self.dir.get(name)

class TDirectory(object):
    def __init__(self, file, begin, nbytesname):
        self.file = file
        index = begin + nbytesname

        self.version, self.ctime, self.mtime = fields(self.file, index, "!hII")
        index = step(index, "!hII")

        self.nbyteskeys, self.nbytesname = fields(self.file, index, "!ii")
        index = step(index, "!ii")

        if self.version <= 1000:
            self.seekdir, self.seekparent, self.seekkeys = fields(self.file, index, "!iii")
        else:
            self.seekdir, self.seekparent, self.seekkeys = fields(self.file, index, "!qqq")

        nk = 4
        keyversion = field(self.file, begin + nk, "!h")
        if keyversion > 1000:
            nk += 2 + 2*4 + 2*2 + 2*8  # fVersion, fObjectSize*Date, fKeyLength*fCycle, fSeekKey*fSeekParentDirectory
        else:
            nk += 2 + 2*4 + 2*2 + 2*4  # fVersion, fObjectSize*Date, fKeyLength*fCycle, fSeekKey*fSeekParentDirectory

        index = begin + nk
        self.classname = string(self.file, index)
        index = stepstring(self.file, index)
        self.name = string(self.file, index)
        index = stepstring(self.file, index)
        self.title = string(self.file, index)
        index = stepstring(self.file, index)

        if not 10 <= self.nbytesname <= 1000:
            raise IOError("directory info")

        self.keys = TKeys(self.file, self.seekkeys)

    def __repr__(self):
        return "<TDirectory {0} at 0x{1:012x}>".format(repr(self.name), id(self))

    def get(self, name):
        # FIXME: parse slashes and get subdirectories
        self.keys.get(name)

class TKeys(object):
    def __init__(self, file, index):
        self.file = file
        self.header = TKey(self.file, index)

        index += self.header.keylen
        nkeys = field(self.file, index, "!i")
        index = step(index, "!i")

        self.keys = []
        for i in range(nkeys):
            self.keys.append(TKey(self.file, index))
            index = self.keys[-1]._index

    def __repr__(self):
        return "<TKeys len={0} at 0x{1:012x}>".format(len(self.keys), id(self))

    def __getitem__(self, i):
        return self.keys[i]

    def get(self, name):
        for key in self.keys:
            if key.name == name:
                return key.get()
        raise KeyError("not found: {0}".format(repr(name)))

class TKey(object):
    classes = {}

    def __init__(self, file, index):
        self.file = file

        self.bytes, self.version, self.objlen, self.datetime, self.keylen, self.cycle = fields(self.file, index, "!ihiIhh")
        index = step(index, "!ihiIhh")

        if self.version > 1000:
            self.seekkey, self.seekpdir = fields(self.file, index, "!qq")
            index = step(index, "!qq")
        else:
            self.seekkey, self.seekpdir = fields(self.file, index, "!ii")
            index = step(index, "!ii")

        self.classname = string(self.file, index)
        index = stepstring(self.file, index)
        self.name = string(self.file, index)
        index = stepstring(self.file, index)
        self.title = string(self.file, index)
        self._index = stepstring(self.file, index)

    def __repr__(self):
        return "<TKey {0} at 0x{1:012x}>".format(repr(self.name), id(self))

    @property
    def isCompressed(self):
        return self.objlen != self.bytes - self.keylen

    def get(self):
        if self.classname not in self.classes:
            raise NotImplementedError("class not recognized: {0}".format(self.classname))

        if self.isCompressed:
            raise NotImplementedError

        start = self.seekkey + self.keylen
        return self.classes[self.classname](self.file, start)

def readversion(file, index):
    bcnt, vers = fields(file, index, "!IH")
    index = step(index, "!IH")
    bcnt = int(numpy.int64(bcnt) & ~readversion.kByteCountMask)
    if bcnt == 0:
        raise IOError("readversion byte count is zero")
    return index, (vers, bcnt)
readversion.kByteCountMask = numpy.int64(0x40000000)

def skipversion(file, index):
    version = field(file, index, "!h")
    index = step(index, "!h")
    if numpy.int64(version) & skipversion.kByteCountVMask:
        index = step(index, "!hh")
    return index
skipversion.kByteCountVMask = numpy.int64(0x4000)

def skiptobject(file, index):
    id, bits = fields(file, index, "!II")
    index = step(index, "!II")
    bits = numpy.uint32(bits) | skiptobject.kIsOnHeap
    if bits & skiptobject.kIsReferenced:
        index = step(index, "!H")
    return index
skiptobject.kIsOnHeap = numpy.uint32(0x01000000)
skiptobject.kIsReferenced = numpy.uint32(1 << 4)

class TTree(object):
    def __init__(self, file, index):
        self.file = file

        ttree_start = index
        index, (ttree_vers, ttree_bcnt) = readversion(self.file, index)

        # START TNamed
        tnamed_start = index
        index, (tnamed_vers, tnamed_bcnt) = readversion(self.file, index)

        index = skipversion(self.file, index)
        index = skiptobject(self.file, index)

        self.name = repr(string(self.file, index))
        index = stepstring(self.file, index)
        self.title = repr(string(self.file, index))
        index = stepstring(self.file, index)

        if index - tnamed_start != tnamed_bcnt + 4:
            raise IOError("TNamed byte count")
        # END TNamed

        # START TAttLine
        tattline_start = index
        index, (tattline_vers, tattline_bcnt) = readversion(self.file, index)
        index = step(index, "!hhh")  # color, style, width
        if index - tattline_start != tattline_bcnt + 4:
            raise IOError("TAttLine byte count")
        # END TAttLine

        # START TAttFill
        tattfill_start = index
        index, (tattfill_vers, tattfill_bcnt) = readversion(self.file, index)
        index = step(index, "!hh")  # color, style
        if index - tattfill_start != tattfill_bcnt + 4:
            raise IOError("TAttFill byte count")
        # END TAttFill

        # START TAttMarker
        tattmarker_start = index
        index, (tattmarker_vers, tattmarker_bcnt) = readversion(self.file, index)
        index = step(index, "!hhf")  # color, style, width
        if index - tattmarker_start != tattmarker_bcnt + 4:
            raise IOError("TAttMarker byte count")
        # END TAttMarker

        entries, self.totbytes, self.zipbytes = fields(self.file, index, "!qqq")
        index = step(index, "!qqq")

        if ttree_vers < 16:
            raise NotImplementedError("TTree too old")

        if ttree_vers >= 19:
            index = step(index, "!q")  # fSavedBytes

        if ttree_vers >= 18:
            index = step(index, "!q")  # flushed bytes

        index = step(index, "!diii")   # fWeight, fTimerInterval, fScanField, fUpdate

        if ttree_vers >= 18:
            index = step(index, "!i")  # fDefaultEntryOffsetLen

        nclus = 0
        if ttree_vers >= 19:
            nclus = field(self.file, index, "!i")
            index = step(index, "!i")  # fNClusterRange

        index = step(index, "!qqqq")   # fMaxEntries, fMaxEntryLoop, fMaxVirtualSize, fAutoSave

        if ttree_vers >= 18:
            index = step(index, "!q")  # fAutoFlush

        index = step(index, "!q")      # fEstimate

        if ttree_vers >= 19:  # "FIXME" in go-hep
            index = step(index, "!b{0}qb{0}b".format(nclus))  # ?, fClusterRangeEnd, ?, fClusterSize

        print "TObjArray", readversion(self.file, index)

TKey.classes["TTree"] = TTree

file = TFile("/home/pivarski/storage/data/TrackResonanceNtuple_uncompressed.root")
print file.get("twoMuon")
<|endoftext|>


<commit_before>#!/bin/python3
import numpy as np

def PIP_identification(P, Q_length=7):
    """
    Input:
            P: input sequence
            Q_length: length of query sequence; Defaults : 7
    Output:
            returns a sequence of Perceptually important point (PIP) identification of size Q_length
    """
    SP = [0] * Q_length
    SP[0] = P[0]
    SP[Q_length - 1] = P[-1]
    counter = 1
    index_mid = int((Q_length - 1) / 2)
    SP[index_mid], index_end = maximize_PIP_distance(P)
    index_start = index_end

    while counter < index_mid:

        SP[index_mid -
            counter], index_end = maximize_PIP_distance(P[0:index_end + 1])
        SP[index_mid +
            counter], index_start_sub = maximize_PIP_distance(P[index_start:])
        index_start += index_start_sub
        counter += 1

    return SP

def maximize_PIP_distance(P):
    """
    Input:
            P: input sequence
    Output:
            returns a point with maximum distance to P[1] and P[-1]
    """
    P1 = [1, P[0]]
    P2 = [len(P), P[-1]]

    distance_P1_P2 = ((P2[1] - P1[0]) ** 2 + (P2[0] - P1[0]) ** 2) ** 0.5

    np_perpendicular_dist = np.fromiter((perpendicular_distance(
        P1, P2, [xi + 1, P[xi]], distance_P1_P2) for xi in range(1, len(P) - 1)), np.float64)

    print(P, np_perpendicular_dist)

    index_max = 1 + np.argmax(np_perpendicular_dist)

    return P[index_max], index_max

def perpendicular_distance(P1, P2, P3, distance):
    """
    Input:
            P1,P2,P3: 3 points in [x,y] format
            S: slope between P1 and P2; this can pre-calculated
    Output:
            returns (perpendicular distance) between these 3 points
    """

    PD = abs((P2[1] - P1[1]) * P3[0] - (P2[0] - P1[0]) *
             P3[1] + P2[0] * P1[1] - P2[1] * P1[0]) / distance

    return PD
<commit_msg>added exception for valueerror caused due to empty numpy array for argmax<commit_after>#!/bin/python3
import numpy as np

def PIP_identification(P, Q_length=7):
    """
    Input:
            P: input sequence
            Q_length: length of query sequence; Defaults : 7
    Output:
            returns a sequence of Perceptually important point (PIP) identification of size Q_length
    """
    try:
        SP = [-1] * Q_length
        SP[0] = P[0]
        SP[Q_length - 1] = P[-1]
        counter = 1
        index_mid = int((Q_length - 1) / 2)
        SP[index_mid], index_lower_end = maximize_PIP_distance(P)
        index_upper_start = index_lower_end
        index_lower_start = 0
        index_upper_end = len(P) - 1

        while counter < index_mid:
            SP[index_mid - counter], index_lower_end = maximize_PIP_distance(
                P[index_lower_start:index_lower_end + 1])

            SP[index_mid + counter], index_temp = maximize_PIP_distance(
                P[index_upper_start:index_upper_end + 1])

            index_upper_start += index_temp
            counter += 1

        return SP
    except ValueError:
        return []

def maximize_PIP_distance(P):
    """
    Input:
            P: input sequence
    Output:
            returns a point with maximum distance to P[1] and P[-1]
    """
    P1 = [1, P[0]]
    P2 = [len(P), P[-1]]

    distance_P1_P2 = ((P2[1] - P1[0]) ** 2 + (P2[0] - P1[0]) ** 2) ** 0.5

    np_perpendicular_dist = np.fromiter((perpendicular_distance(
        P1, P2, [xi + 1, P[xi]], distance_P1_P2) for xi in range(1, len(P) - 1)), np.float64)

    index_max = 1 + np.argmax(np_perpendicular_dist)

    return P[index_max], index_max

def perpendicular_distance(P1, P2, P3, distance):
    """
    Input:
            P1,P2,P3: 3 points in [x,y] format
            S: slope between P1 and P2; this can pre-calculated
    Output:
            returns (perpendicular distance) between these 3 points
    """

    PD = abs((P2[1] - P1[1]) * P3[0] - (P2[0] - P1[0]) *
             P3[1] + P2[0] * P1[1] - P2[1] * P1[0]) / distance

    return PD
<|endoftext|>


<commit_before>#!/usr/bin/env python

from datetime import datetime
import json
import logging
from math import floor
from uuid import uuid4

from celery.decorators import task
from celery.task import Task
from django.conf import settings
from sunburnt import SolrInterface

from csvkit import CSVKitReader

SOLR_ADD_BUFFER_SIZE = 500

class DatasetImportTask(Task):
    """
    Import a dataset into Solr.
    """
    def __call__(self, dataset_id, *args, **kwargs):
        """
        Setup task tracking.
        """
        from redd.models import Dataset, TaskStatus

        task_status = TaskStatus.objects.create(
            task_id=self.request.id,
            task_name=self.name)

        dataset = Dataset.objects.get(id=dataset_id)
        dataset.current_task = task_status
        dataset.save()

        return self.run(dataset_id, *args, **kwargs)

    def _count_lines(self, filename):
        """
        Efficiently count the number of lines in a file.
        """
        with open(filename) as f:
            for i, l in enumerate(f):
                pass
        return i + 1

    def run(self, dataset_id):
        """
        Execute import.
        """
        from redd.models import Dataset

        log = logging.getLogger('redd.tasks.DatasetImportTask')
        log.info('Beginning import, dataset_id: %i' % dataset_id)

        dataset = Dataset.objects.get(id=dataset_id)

        task_status = dataset.current_task
        task_status.start = datetime.now()
        task_status.message = 'Preparing to import'
        task_status.save()

        line_count = self._count_lines(dataset.data_upload.get_path())

        solr = SolrInterface(settings.SOLR_ENDPOINT)

        reader = CSVKitReader(open(dataset.data_upload.get_path(), 'r'))
        reader.next()

        add_buffer = []

        for i, row in enumerate(reader, start=1):
            data = {
                'id': uuid4(),
                'dataset_id': dataset.id,
                'row': i,
                'full_text': '\n'.join(row),
                'data': json.dumps(row)
            }

            add_buffer.append(data)

            if i % SOLR_ADD_BUFFER_SIZE == 0:
                solr.add(add_buffer)
                add_buffer = []

                task_status.message = '%i%% complete (estimated)' % floor(float(i) / line_count * 100)
                task_status.save()

        if add_buffer:
            solr.add(add_buffer)
            add_buffer = []

        solr.commit()

        log.info('Finished import, dataset_id: %i' % dataset_id)

    def after_return(self, status, retval, task_id, args, kwargs, einfo):
        """
        Save final status, results, etc.
        """
        from redd.models import TaskStatus

        task_status = TaskStatus.objects.get(task_id=self.request.id)

        task_status.status = status
        task_status.message = 'Import complete'
        task_status.end = datetime.now()

        if einfo:
            task_status.traceback = einfo.traceback

        task_status.save()

#tasks.register(DatasetImportTask)

@task(name='Purge data')
def dataset_purge_data(dataset_id):
    """
    Purge a dataset from Solr.
    """
    log = logging.getLogger('redd.tasks.dataset_purge_data')
    log.info('Beginning purge, dataset_id: %i' % dataset_id)

    solr = SolrInterface(settings.SOLR_ENDPOINT)
    solr.delete(queries='dataset_id: %i' % dataset_id, commit=True)

    log.info('Finished purge, dataset_id: %i' % dataset_id)

<commit_msg>Fix float formatting.<commit_after>#!/usr/bin/env python

from datetime import datetime
import json
import logging
from math import floor
from uuid import uuid4

from celery.decorators import task
from celery.task import Task
from django.conf import settings
from sunburnt import SolrInterface

from csvkit import CSVKitReader

SOLR_ADD_BUFFER_SIZE = 500

class DatasetImportTask(Task):
    """
    Import a dataset into Solr.
    """
    def __call__(self, dataset_id, *args, **kwargs):
        """
        Setup task tracking.
        """
        from redd.models import Dataset, TaskStatus

        task_status = TaskStatus.objects.create(
            task_id=self.request.id,
            task_name=self.name)

        dataset = Dataset.objects.get(id=dataset_id)
        dataset.current_task = task_status
        dataset.save()

        return self.run(dataset_id, *args, **kwargs)

    def _count_lines(self, filename):
        """
        Efficiently count the number of lines in a file.
        """
        with open(filename) as f:
            for i, l in enumerate(f):
                pass
        return i + 1

    def run(self, dataset_id):
        """
        Execute import.
        """
        from redd.models import Dataset

        log = logging.getLogger('redd.tasks.DatasetImportTask')
        log.info('Beginning import, dataset_id: %i' % dataset_id)

        dataset = Dataset.objects.get(id=dataset_id)

        task_status = dataset.current_task
        task_status.status = 'STARTED'
        task_status.start = datetime.now()
        task_status.message = 'Preparing to import'
        task_status.save()

        line_count = self._count_lines(dataset.data_upload.get_path())

        solr = SolrInterface(settings.SOLR_ENDPOINT)

        reader = CSVKitReader(open(dataset.data_upload.get_path(), 'r'))
        reader.next()

        add_buffer = []

        for i, row in enumerate(reader, start=1):
            data = {
                'id': uuid4(),
                'dataset_id': dataset.id,
                'row': i,
                'full_text': '\n'.join(row),
                'data': json.dumps(row)
            }

            add_buffer.append(data)

            if i % SOLR_ADD_BUFFER_SIZE == 0:
                solr.add(add_buffer)
                add_buffer = []

                task_status.message = '%.0f%% complete (estimated)' % floor(float(i) / float(line_count) * 100)
                task_status.save()

        if add_buffer:
            solr.add(add_buffer)
            add_buffer = []

        solr.commit()

        log.info('Finished import, dataset_id: %i' % dataset_id)

    def after_return(self, status, retval, task_id, args, kwargs, einfo):
        """
        Save final status, results, etc.
        """
        from redd.models import TaskStatus

        task_status = TaskStatus.objects.get(task_id=self.request.id)

        task_status.status = status
        task_status.message = 'Import complete'
        task_status.end = datetime.now()

        if einfo:
            task_status.traceback = einfo.traceback

        task_status.save()

#tasks.register(DatasetImportTask)

@task(name='Purge data')
def dataset_purge_data(dataset_id):
    """
    Purge a dataset from Solr.
    """
    log = logging.getLogger('redd.tasks.dataset_purge_data')
    log.info('Beginning purge, dataset_id: %i' % dataset_id)

    solr = SolrInterface(settings.SOLR_ENDPOINT)
    solr.delete(queries='dataset_id: %i' % dataset_id, commit=True)

    log.info('Finished purge, dataset_id: %i' % dataset_id)

<|endoftext|>


<commit_before>import os
import time
from slackclient import SlackClient
from redmine import Redmine

# redminebot's ID and external host from environment variables
BOT_ID = os.environ.get('BOT_ID')
EXT_HOST = os.environ.get('REDMINE_EXT_HOST')

# constants
AT_BOT = "<@" + BOT_ID + ">"

# instantiate Slack & Redmine clients
sc = SlackClient(os.environ.get('BOT_TOKEN'))
rc = Redmine(os.environ.get('REDMINE_HOST'), version=os.environ.get('REDMINE_VERSION'), \
     key=os.environ.get('REDMINE_TOKEN'))

def handle_command(command, channel, username):
    """
        Receives commands directed at the bot and determines if they
        are valid commands. If so, then acts on the commands. If not,
        returns back what it needs for clarification.
    """
    response = ":question: Unknown/invalid command - Try `help` for a list of supported commands"
    commands = command.split()
    if commands:
        operator = commands[0]
        s = " "
        msg = s.join(commands[1:])
        if operator == "issueto" and len(commands) > 2:
            assigneduser = commands[1]
            newmsg = s.join(commands[2:])
            response = create_issue(newmsg, username, assigneduser)
        elif operator == "issue" and len(commands) > 1:
            response = create_issue(msg, username, username)
        elif operator == "help":
            response = show_commands()
    message = "@" + username + " " + response
    sc.api_call("chat.postMessage", channel=channel, \
                          text=message, as_user=True)

def parse_slack_output(slack_rtm_output):
    """
        The Slack Real Time Messaging API is an events firehose.
        this parsing function returns None unless a message is
        directed at the Bot, based on its ID.
    """
    output_list = slack_rtm_output
    if output_list and len(output_list) > 0:
        for output in output_list:
            if output and 'text' in output and AT_BOT in output['text']:
                profile = sc.api_call("users.info", user=output['user'])
                username = profile['user']['name']
                # return text after the @ mention, whitespace removed
                return output['text'].split(AT_BOT)[1].strip().lower(), \
                       output['channel'], username
    return None, None, None

def show_commands():
    """
        Return ist of commands that bot can handle
    """
    return ":wrench: List of supported commands:\n" \
           "`issue <subject>` - creates new issue and assigns it to yourself\n" \
	   "`issueto <name> <subject>` - creates new issue and assigns it to `<user>`"

"""
    Redmine commands
"""
def create_issue(text, username, assigneduser):
        user = rc.user.filter(name=username)[0]
    assigned = rc.user.filter(name=assigneduser)[0]
    if user.id == None or assigned.id == None:
            return ":x: Failed to find your user name in Redmine"
    # impersonate user so it looks like the update is from them
    rcn = Redmine(os.environ.get('REDMINE_HOST'), version=os.environ.get('REDMINE_VERSION'), \
     key=os.environ.get('REDMINE_TOKEN'), impersonate=user.login)
    issue = rcn.issue.create(project_id='general', subject=text, tracker_id=2, assigned_to_id=assigned.id)
        if issue.id:
            return ":white_check_mark: Created Issue <"+EXT_HOST+"/issues/"+str(issue.id)+ \
        "|#"+str(issue.id)+" "+issue.subject+"> assigned to "+assigned.firstname+" "+assigned.lastname
        else:
            return ":x: Issue creation failed"

if __name__ == "__main__":
    READ_WEBSOCKET_DELAY = 1 # 1 second delay between reading from firehose
    if sc.rtm_connect():
        print("RedmineBot connected and running!")
        while True:
            command, channel, username = parse_slack_output(sc.rtm_read())
            if command and channel and username:
                handle_command(command, channel, username)
            elif channel and username:
                handle_command("help", channel, username)
            time.sleep(READ_WEBSOCKET_DELAY)
    else:
        print("Connection failed. Invalid Slack token or bot ID?")
<commit_msg>added close, list, and listfrom commands<commit_after>import os
import time
from slackclient import SlackClient
from redmine import Redmine

# redminebot's ID and external host from environment variables
BOT_ID = os.environ.get('BOT_ID')
EXT_HOST = os.environ.get('REDMINE_EXT_HOST')

# constants
AT_BOT = "<@" + BOT_ID + ">"

# instantiate Slack & Redmine clients
sc = SlackClient(os.environ.get('BOT_TOKEN'))
rc = Redmine(os.environ.get('REDMINE_HOST'), version=os.environ.get('REDMINE_VERSION'), \
     key=os.environ.get('REDMINE_TOKEN'))

def handle_command(command, channel, username):
    """
        Receives commands directed at the bot and determines if they
        are valid commands. If so, then acts on the commands. If not,
        returns back what it needs for clarification.
    """
    response = ":question: Unknown/invalid command - Try `help` for a list of supported commands"
    commands = command.split()
    if commands:
        operator = commands[0]
        s = " "
        msg = s.join(commands[1:])
        if operator == "issueto" and len(commands) > 2:
            assigneduser = commands[1]
            newmsg = s.join(commands[2:])
            response = create_issue(newmsg, username, assigneduser)
        elif operator == "issue" and len(commands) > 1:
            response = create_issue(msg, username, username)
        elif operator == "close" and len(commands) > 2:
            issue = commands[1]
            newmsg = s.join(commands[2:])
            response = close_issue(newmsg, issue, username)
        elif operator == "list":
            response = list_issues(username)
        elif operator == "listfrom" and len(commands) > 1:
            listuser = commands[1]
            response = list_issues(listuser)
        elif operator == "help":
            response = show_commands()
    message = "@" + username + " " + response
    sc.api_call("chat.postMessage", channel=channel, \
                          text=message, as_user=True)

def parse_slack_output(slack_rtm_output):
    """
        The Slack Real Time Messaging API is an events firehose.
        this parsing function returns None unless a message is
        directed at the Bot, based on its ID.
    """
    output_list = slack_rtm_output
    if output_list and len(output_list) > 0:
        for output in output_list:
            if output and 'text' in output and AT_BOT in output['text']:
                profile = sc.api_call("users.info", user=output['user'])
                username = profile['user']['name']
                # return text after the @ mention, whitespace removed
                return output['text'].split(AT_BOT)[1].strip().lower(), \
                       output['channel'], username
    return None, None, None

def show_commands():
    """
        Return ist of commands that bot can handle
    """
    return ":wrench: List of supported commands:\n" \
           "`issue <subject>` - creates new issue and assigns it to you\n" \
	   "`issueto <name> <subject>` - creates new issue and assigns it to `<name>`\n" \
           "`close <issue #> <comment>` - closes an issue with the following comment\n" \
           "`list` - list all open issues assigned to you\n" \
           "`listfrom <name>` - list all open issues assigned to `<name>`"

"""
    Redmine commands
"""
def create_issue(text, username, assigneduser):
    user = rc.user.filter(name=username)[0]
    assigned = rc.user.filter(name=assigneduser)[0]
    if user.id == None or assigned.id == None:
        return ":x: Failed to find your user name in Redmine"
    # impersonate user so it looks like the update is from them
    rcn = Redmine(os.environ.get('REDMINE_HOST'), version=os.environ.get('REDMINE_VERSION'), \
     key=os.environ.get('REDMINE_TOKEN'), impersonate=user.login)
    issue = rcn.issue.create(project_id='general', subject=text, tracker_id=2, assigned_to_id=assigned.id)
    if issue.id:
        return ":white_check_mark: Created Issue <"+EXT_HOST+"/issues/"+str(issue.id)+ \
        "|#"+str(issue.id)+" "+issue.subject+"> assigned to "+assigned.firstname+" "+assigned.lastname
    else:
        return ":x: Issue creation failed"

def list_issues(username):
    user = rc.user.filter(name=username)[0]
    if user.id == None:
        return ":x: Failed to find your user name in Redmine"
    result = rc.issue.filter(sort='project', assigned_to_id=user.id, status_id='open')
    response = ""
    if len(result) > 0:
        response = ":book: Open issues assigned to "+user.firstname+" "+user.lastname+":\n"
        for issue in result:
            response += ""+issue.project.name+" <"+EXT_HOST+"/issues/"+str(issue.id)+"|#"+str(issue.id)+ \
                        " "+issue.subject+">\n"
    else:
        response = ":thumbsup_all: No open issues assigned to "+user.firstname+" "+user.lastname
    return response

def close_issue(text, issue, username):
    user = rc.user.filter(name=username)[0]
    if user.id == None:
        return ":x: Failed to find your user name in Redmine"
    # impersonate user so it looks like the update is from them
    rcn = Redmine(os.environ.get('REDMINE_HOST'), version=os.environ.get('REDMINE_VERSION'), \
     key=os.environ.get('REDMINE_TOKEN'), impersonate=user.login)
    result = rcn.issue.update(issue, status_id=5, notes=text, done_ratio=100)
    if result:
        return ":white_check_mark: Closed Issue <"+EXT_HOST+"/issues/"+str(issue)+ \
        "|#"+str(issue)+">"
    else:
        return ":x: Issue closing failed"

if __name__ == "__main__":
    READ_WEBSOCKET_DELAY = 1 # 1 second delay between reading from firehose
    if sc.rtm_connect():
        print("RedmineBot connected and running!")
        while True:
            command, channel, username = parse_slack_output(sc.rtm_read())
            if command and channel and username:
                handle_command(command, channel, username)
            elif channel and username:
                handle_command("help", channel, username)
            time.sleep(READ_WEBSOCKET_DELAY)
    else:
        print("Connection failed. Invalid Slack token or bot ID?")
<|endoftext|>


<commit_before>from UM.Operations.Operation import Operation
from UM.Operations.AddSceneNodeOperation import AddSceneNodeOperation
from UM.Operations.TranslateOperation import TranslateOperation
from UM.Operations.GroupedOperation import GroupedOperation

##  A specialised operation designed specifically to modify the previous operation.
class PlatformPhysicsOperation(Operation):
    def __init__(self, node, translation):
        super().__init__()
        self._node = node
        self._transform = node.getLocalTransformation()
        self._position = node.getPosition() + translation
        self._always_merge = True

    def undo(self):
        self._node.setLocalTransformation(self._transform)

    def redo(self):
        self._node.setPosition(self._position)

    def mergeWith(self, other):
        group = GroupedOperation()

        group.addOperation(self)
        group.addOperation(other)

        return group

    def __repr__(self):
        return 'PlatformPhysicsOperation(t = {0})'.format(self._position)
<commit_msg>Fix undo of PlatformPhysicsOperation after the SceneNode changes<commit_after>from UM.Operations.Operation import Operation
from UM.Operations.AddSceneNodeOperation import AddSceneNodeOperation
from UM.Operations.TranslateOperation import TranslateOperation
from UM.Operations.GroupedOperation import GroupedOperation

##  A specialised operation designed specifically to modify the previous operation.
class PlatformPhysicsOperation(Operation):
    def __init__(self, node, translation):
        super().__init__()
        self._node = node
        self._old_position = node.getPosition()
        self._new_position = node.getPosition() + translation
        self._always_merge = True

    def undo(self):
        self._node.setPosition(self._old_position)

    def redo(self):
        self._node.setPosition(self._new_position)

    def mergeWith(self, other):
        group = GroupedOperation()

        group.addOperation(self)
        group.addOperation(other)

        return group

    def __repr__(self):
        return 'PlatformPhysicsOperation(t = {0})'.format(self._position)
<|endoftext|>


<commit_before>import time
import datetime
from collections import deque

import lxml.html
import pytz
import icalendar
import requests
from pupa.scrape import Scraper
import scrapelib

from .base import LegistarScraper, LegistarAPIScraper

class LegistarEventsScraper(LegistarScraper):
    def eventPages(self, since) :
        # Directly use the requests library here, so that we do not
        # use a cached page, which may have expired .NET state values,
        # even in fastmode (which uses the cache).
        response = requests.get(self.EVENTSPAGE, verify=False)
        entry = response.text
        page = lxml.html.fromstring(entry)
        page.make_links_absolute(self.EVENTSPAGE)

        if since is None :
            for page in self.eventSearch(page, 'All'):
                yield page
        else :
            for year in range(since, self.now().year + 1) :
                yield from self.eventSearch(page, str(year))

    def eventSearch(self, page, value) :
        payload = self.sessionSecrets(page)

        payload['ctl00_ContentPlaceHolder1_lstYears_ClientState'] = '{"value":"%s"}' % value

        payload['__EVENTTARGET'] = 'ctl00$ContentPlaceHolder1$lstYears'

        return self.pages(self.EVENTSPAGE, payload)

    def events(self, follow_links=True, since=None) :
        # If an event is added to the the legistar system while we
        # are scraping, it will shift the list of events down and
        # we might revisit the same event. So, we keep track of
        # the last few events we've visited in order to
        # make sure we are not revisiting
        scraped_events = deque([], maxlen=10)

        if since:
            try:
                since_year = since - 1
            except TypeError:
                raise TypeError('Expected integer for keyword arg :since, got "{since}" of type {type}'.format(since=since,
                                                                                type=type(since)))
        else:
            since_year = 0

        for year in range(self.now().year, since_year, -1):
            for page in self.eventPages(year) :

                events_table = page.xpath("//table[@class='rgMasterTable']")[0]
                for event, _, _ in self.parseDataTable(events_table) :
                    if follow_links and type(event["Meeting Details"]) == dict :
                        detail_url = event["Meeting Details"]['url']
                        if detail_url in scraped_events :
                            continue
                        else :
                            scraped_events.append(detail_url)

                        meeting_details = self.lxmlize(detail_url)

                        agenda = self.agenda(detail_url)

                    else :
                        agenda = None

                    yield event, agenda

    def agenda(self, detail_url) :
        page = self.lxmlize(detail_url)

        payload = self.sessionSecrets(page)

        payload.update({"__EVENTARGUMENT": "3:1",
                        "__EVENTTARGET":"ctl00$ContentPlaceHolder1$menuMain"})

        for page in self.pages(detail_url, payload) :
            agenda_table = page.xpath(
                "//table[@id='ctl00_ContentPlaceHolder1_gridMain_ctl00']")[0]
            agenda = self.parseDataTable(agenda_table)
            yield from agenda

    def addDocs(self, e, events, doc_type) :
        try :
            if events[doc_type] != 'Not\xa0available' :
                e.add_document(note= events[doc_type]['label'],
                               url = events[doc_type]['url'],
                               media_type="application/pdf")
        except ValueError :
            pass

    def extractRollCall(self, action_detail_url) :
        action_detail_page = self.lxmlize(action_detail_url)
        try:
            rollcall_table = action_detail_page.xpath("//table[@id='ctl00_ContentPlaceHolder1_gridRollCall_ctl00']")[0]
        except IndexError:
            self.warning("No rollcall found in table")
            return []
        roll_call = list(self.parseDataTable(rollcall_table))
        call_list = []
        for call, _, _ in roll_call :
            option = call['Attendance']
            call_list.append((option,
                              call['Person Name']['label']))

        return call_list

    def ical(self, ical_text):
        value = icalendar.Calendar.from_ical(ical_text)
        return value

class LegistarAPIEventScraper(LegistarAPIScraper):

    def events(self, since_datetime=None):
        self._events = self._scrapeWebCalendar()
        self._scraped_events = {}

        if since_datetime:
            # Minutes are often published after an event occurs – without a
            # corresponding event modification. Query all update fields so later
            # changes are always caught by our scraper, particularly when
            # scraping narrower windows of time.
            update_fields = ('EventLastModifiedUtc',
                             'EventAgendaLastPublishedUTC',
                             'EventMinutesLastPublishedUTC')

            since_fmt = " gt datetime'{}'".format(since_datetime.isoformat())
            since_filter = ' or '.join(field + since_fmt for field in update_fields)

            params = {'$filter' : since_filter}

        else:
            params = {}

        events_url = self.BASE_URL + '/events/'

        for api_event in self.pages(events_url,
                                    params=params,
                                    item_key="EventId"):

            # EventTime may be 'None': this try-except block catches those instances.
            try:
                start_time = time.strptime(api_event['EventTime'], '%I:%M %p')

            except TypeError:
                continue

            else:
                start = self.toTime(api_event['EventDate'])
                api_event['start'] = start.replace(hour=start_time.tm_hour,
                                                   minute=start_time.tm_min)

                api_event['status'] = self._event_status(api_event)

                # Skip events that do not appear in the web interface.
                if api_event['EventAgendaStatusId'] == 1:
                    continue

                else:
                    web_event = self.web_results(api_event)
                    yield api_event, web_event

    def agenda(self, event):
        agenda_url = self.BASE_URL + '/events/{}/eventitems'.format(event['EventId'])

        response = self.get(agenda_url)

        try:
            # Order the event items according to the EventItemMinutesSequence. If an
            # event item does not have a value for EventItemMinutesSequence, the script
            #will throw a TypeError. In that case, try to order by EventItemAgendaSequence.
            filtered_response = sorted((item for item in response.json()
                                        if item['EventItemTitle']),
                                       key = lambda item : item['EventItemMinutesSequence'])
        except TypeError:
            try:
                filtered_response = sorted((item for item in response.json()
                                            if item['EventItemTitle']),
                                           key = lambda item : item['EventItemAgendaSequence'])
            except TypeError:
                filtered_response = (item for item in response.json()
                                     if item['EventItemTitle'])

        for item in filtered_response:
            yield item

    def rollcalls(self, event):
        for item in self.agenda(event):
            if item['EventItemRollCallFlag']:
                rollcall_url = self.BASE_URL + '/eventitems/{}/rollcalls'.format(item['EventItemId'])

                response = self.get(rollcall_url)

                for item in response.json():
                    yield item

    def web_results(self, event):
        api_key = (event['EventBodyName'].strip(),
                   event['start'])

        if api_key in self._scraped_events:
            return self._scraped_events[api_key]

        else:
            for web_key, event in self._events:
                self._scraped_events[web_key] = event
                if web_key == api_key:
                    return event

    def _scrapeWebCalendar(self):
        '''Generator yielding events from Legistar in roughly reverse
        chronological order.
        '''
        web_scraper = LegistarEventsScraper(self.jurisdiction,
                                            self.datadir,
                                            strict_validation=self.strict_validation,
                                            fastmode=(self.requests_per_minute == 0))

        web_scraper.EVENTSPAGE = self.EVENTSPAGE
        web_scraper.BASE_URL = self.WEB_URL
        web_scraper.TIMEZONE = self.TIMEZONE
        web_scraper.date_format = '%m/%d/%Y'

        for event, _ in web_scraper.events(follow_links=False):
            event_key = self._event_key(event, web_scraper)
            yield event_key, event

    def _event_key(self, event, web_scraper):
        '''Since Legistar InSite contains more information about events than
        are available in the API, we need to scrape both. Then, we have
        to line them up. This method makes a key that should be
        uniquely identify every event and will allow us to link
        events from the two data sources.
        '''
        response = web_scraper.get(event['iCalendar']['url'], verify=False)
        event_time = web_scraper.ical(response.text).subcomponents[0]['DTSTART'].dt
        event_time = pytz.timezone(self.TIMEZONE).localize(event_time)

        key = (event['Name']['label'],
               event_time)

        return key

    def addDocs(self, e, events, doc_type):
        try :
            if events[doc_type] != 'Not\xa0available':
                e.add_document(note= events[doc_type]['label'],
                               url = events[doc_type]['url'],
                               media_type="application/pdf")
        except ValueError :
            pass

    def _event_status(self, event):
        '''Events can have a status of tentative, confirmed, cancelled, or
        passed (http://docs.opencivicdata.org/en/latest/data/event.html). By
        default, set status to passed if the current date and time exceeds the
        event date and time, or confirmed otherwise. Available for override in
        jurisdictional scrapers.
        '''
        if datetime.datetime.utcnow().replace(tzinfo=pytz.utc) > event['start']:
            status = 'passed'
        else:
            status = 'confirmed'

        return status

<commit_msg>Remove year iteration from eventPages<commit_after>import time
import datetime
from collections import deque

import lxml.html
import pytz
import icalendar
import requests
from pupa.scrape import Scraper
import scrapelib

from .base import LegistarScraper, LegistarAPIScraper

class LegistarEventsScraper(LegistarScraper):
    def eventPages(self, since) :
        # Directly use the requests library here, so that we do not
        # use a cached page, which may have expired .NET state values,
        # even in fastmode (which uses the cache).
        response = requests.get(self.EVENTSPAGE, verify=False)
        entry = response.text
        page = lxml.html.fromstring(entry)
        page.make_links_absolute(self.EVENTSPAGE)

        for page in self.eventSearch(page, since):
            yield page

    def eventSearch(self, page, value) :
        payload = self.sessionSecrets(page)

        payload['ctl00_ContentPlaceHolder1_lstYears_ClientState'] = '{"value":"%s"}' % value

        payload['__EVENTTARGET'] = 'ctl00$ContentPlaceHolder1$lstYears'

        return self.pages(self.EVENTSPAGE, payload)

    def events(self, follow_links=True, since=None) :
        # If an event is added to the the legistar system while we
        # are scraping, it will shift the list of events down and
        # we might revisit the same event. So, we keep track of
        # the last few events we've visited in order to
        # make sure we are not revisiting
        scraped_events = deque([], maxlen=10)

        if since:
            try:
                since_year = since - 1
            except TypeError:
                raise TypeError('Expected integer for keyword arg :since, got "{since}" of type {type}'.format(since=since,
                                                                                type=type(since)))
        else:
            since_year = 0

        # Anticipate events will be scheduled for the following year to avoid
        # missing upcoming events during scrapes near the end of the current
        # year.
        for year in range(self.now().year + 1, since_year, -1):
            for page in self.eventPages(year) :

                events_table = page.xpath("//table[@class='rgMasterTable']")[0]
                for event, _, _ in self.parseDataTable(events_table) :
                    if follow_links and type(event["Meeting Details"]) == dict :
                        detail_url = event["Meeting Details"]['url']
                        if detail_url in scraped_events :
                            continue
                        else :
                            scraped_events.append(detail_url)

                        meeting_details = self.lxmlize(detail_url)

                        agenda = self.agenda(detail_url)

                    else :
                        agenda = None

                    yield event, agenda

    def agenda(self, detail_url) :
        page = self.lxmlize(detail_url)

        payload = self.sessionSecrets(page)

        payload.update({"__EVENTARGUMENT": "3:1",
                        "__EVENTTARGET":"ctl00$ContentPlaceHolder1$menuMain"})

        for page in self.pages(detail_url, payload) :
            agenda_table = page.xpath(
                "//table[@id='ctl00_ContentPlaceHolder1_gridMain_ctl00']")[0]
            agenda = self.parseDataTable(agenda_table)
            yield from agenda

    def addDocs(self, e, events, doc_type) :
        try :
            if events[doc_type] != 'Not\xa0available' :
                e.add_document(note= events[doc_type]['label'],
                               url = events[doc_type]['url'],
                               media_type="application/pdf")
        except ValueError :
            pass

    def extractRollCall(self, action_detail_url) :
        action_detail_page = self.lxmlize(action_detail_url)
        try:
            rollcall_table = action_detail_page.xpath("//table[@id='ctl00_ContentPlaceHolder1_gridRollCall_ctl00']")[0]
        except IndexError:
            self.warning("No rollcall found in table")
            return []
        roll_call = list(self.parseDataTable(rollcall_table))
        call_list = []
        for call, _, _ in roll_call :
            option = call['Attendance']
            call_list.append((option,
                              call['Person Name']['label']))

        return call_list

    def ical(self, ical_text):
        value = icalendar.Calendar.from_ical(ical_text)
        return value

class LegistarAPIEventScraper(LegistarAPIScraper):

    def events(self, since_datetime=None):
        self._events = self._scrapeWebCalendar()
        self._scraped_events = {}

        if since_datetime:
            # Minutes are often published after an event occurs – without a
            # corresponding event modification. Query all update fields so later
            # changes are always caught by our scraper, particularly when
            # scraping narrower windows of time.
            update_fields = ('EventLastModifiedUtc',
                             'EventAgendaLastPublishedUTC',
                             'EventMinutesLastPublishedUTC')

            since_fmt = " gt datetime'{}'".format(since_datetime.isoformat())
            since_filter = ' or '.join(field + since_fmt for field in update_fields)

            params = {'$filter' : since_filter}

        else:
            params = {}

        events_url = self.BASE_URL + '/events/'

        for api_event in self.pages(events_url,
                                    params=params,
                                    item_key="EventId"):

            # EventTime may be 'None': this try-except block catches those instances.
            try:
                start_time = time.strptime(api_event['EventTime'], '%I:%M %p')

            except TypeError:
                continue

            else:
                start = self.toTime(api_event['EventDate'])
                api_event['start'] = start.replace(hour=start_time.tm_hour,
                                                   minute=start_time.tm_min)

                api_event['status'] = self._event_status(api_event)

                # Skip events that do not appear in the web interface.
                if api_event['EventAgendaStatusId'] == 1:
                    continue

                else:
                    web_event = self.web_results(api_event)
                    yield api_event, web_event

    def agenda(self, event):
        agenda_url = self.BASE_URL + '/events/{}/eventitems'.format(event['EventId'])

        response = self.get(agenda_url)

        try:
            # Order the event items according to the EventItemMinutesSequence. If an
            # event item does not have a value for EventItemMinutesSequence, the script
            #will throw a TypeError. In that case, try to order by EventItemAgendaSequence.
            filtered_response = sorted((item for item in response.json()
                                        if item['EventItemTitle']),
                                       key = lambda item : item['EventItemMinutesSequence'])
        except TypeError:
            try:
                filtered_response = sorted((item for item in response.json()
                                            if item['EventItemTitle']),
                                           key = lambda item : item['EventItemAgendaSequence'])
            except TypeError:
                filtered_response = (item for item in response.json()
                                     if item['EventItemTitle'])

        for item in filtered_response:
            yield item

    def rollcalls(self, event):
        for item in self.agenda(event):
            if item['EventItemRollCallFlag']:
                rollcall_url = self.BASE_URL + '/eventitems/{}/rollcalls'.format(item['EventItemId'])

                response = self.get(rollcall_url)

                for item in response.json():
                    yield item

    def web_results(self, event):
        api_key = (event['EventBodyName'].strip(),
                   event['start'])

        if api_key in self._scraped_events:
            return self._scraped_events[api_key]

        else:
            for web_key, event in self._events:
                self._scraped_events[web_key] = event
                if web_key == api_key:
                    return event

    def _scrapeWebCalendar(self):
        '''Generator yielding events from Legistar in roughly reverse
        chronological order.
        '''
        web_scraper = LegistarEventsScraper(self.jurisdiction,
                                            self.datadir,
                                            strict_validation=self.strict_validation,
                                            fastmode=(self.requests_per_minute == 0))

        web_scraper.EVENTSPAGE = self.EVENTSPAGE
        web_scraper.BASE_URL = self.WEB_URL
        web_scraper.TIMEZONE = self.TIMEZONE
        web_scraper.date_format = '%m/%d/%Y'

        for event, _ in web_scraper.events(follow_links=False):
            event_key = self._event_key(event, web_scraper)
            yield event_key, event

    def _event_key(self, event, web_scraper):
        '''Since Legistar InSite contains more information about events than
        are available in the API, we need to scrape both. Then, we have
        to line them up. This method makes a key that should be
        uniquely identify every event and will allow us to link
        events from the two data sources.
        '''
        response = web_scraper.get(event['iCalendar']['url'], verify=False)
        event_time = web_scraper.ical(response.text).subcomponents[0]['DTSTART'].dt
        event_time = pytz.timezone(self.TIMEZONE).localize(event_time)

        key = (event['Name']['label'],
               event_time)

        return key

    def addDocs(self, e, events, doc_type):
        try :
            if events[doc_type] != 'Not\xa0available':
                e.add_document(note= events[doc_type]['label'],
                               url = events[doc_type]['url'],
                               media_type="application/pdf")
        except ValueError :
            pass

    def _event_status(self, event):
        '''Events can have a status of tentative, confirmed, cancelled, or
        passed (http://docs.opencivicdata.org/en/latest/data/event.html). By
        default, set status to passed if the current date and time exceeds the
        event date and time, or confirmed otherwise. Available for override in
        jurisdictional scrapers.
        '''
        if datetime.datetime.utcnow().replace(tzinfo=pytz.utc) > event['start']:
            status = 'passed'
        else:
            status = 'confirmed'

        return status

<|endoftext|>


<commit_before># Copyright (C) 2019  Renato Lima - Akretion
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html

{
    "name": "NF-e",
    "summary": "Brazilian Eletronic Invoice NF-e .",
    "category": "Localisation",
    "license": "AGPL-3",
    "author": "Akretion," "KMEE," "Odoo Community Association (OCA)",
    "website": "https://github.com/OCA/l10n-brazil",
    "development_status": "Beta",
    "version": "12.0.2.5.1",
    "depends": [
        "l10n_br_fiscal",
        "l10n_br_nfe_spec",
        "spec_driven_model",
    ],
    "data": [
        # Security
        "security/nfe_security.xml",
        # Views
        "views/res_company_view.xml",
        "views/nfe_document_view.xml",
        "views/res_config_settings_view.xml",
    ],
    "demo": [
        "demo/res_users_demo.xml",
        "demo/fiscal_document_demo.xml",
        "demo/company_demo.xml",
    ],
    "post_init_hook": "post_init_hook",
    "installable": True,
    "auto_install": False,
    "external_dependencies": {
        "python": [
            "nfelib",
            "erpbrasil.base",
            "erpbrasil.assinatura",
            "erpbrasil.transmissao",
            "erpbrasil.edoc",
            "xmldiff",
        ],
    },
}
<commit_msg>l10n_br_nfe 12.0.3.0.0<commit_after># Copyright (C) 2019  Renato Lima - Akretion
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html

{
    "name": "NF-e",
    "summary": "Brazilian Eletronic Invoice NF-e .",
    "category": "Localisation",
    "license": "AGPL-3",
    "author": "Akretion," "KMEE," "Odoo Community Association (OCA)",
    "website": "https://github.com/OCA/l10n-brazil",
    "development_status": "Beta",
    "version": "12.0.3.0.0",
    "depends": [
        "l10n_br_fiscal",
        "l10n_br_nfe_spec",
        "spec_driven_model",
    ],
    "data": [
        # Security
        "security/nfe_security.xml",
        # Views
        "views/res_company_view.xml",
        "views/nfe_document_view.xml",
        "views/res_config_settings_view.xml",
    ],
    "demo": [
        "demo/res_users_demo.xml",
        "demo/fiscal_document_demo.xml",
        "demo/company_demo.xml",
    ],
    "post_init_hook": "post_init_hook",
    "installable": True,
    "auto_install": False,
    "external_dependencies": {
        "python": [
            "nfelib",
            "erpbrasil.base",
            "erpbrasil.assinatura",
            "erpbrasil.transmissao",
            "erpbrasil.edoc",
            "xmldiff",
        ],
    },
}
<|endoftext|>


<commit_before># Copyright (C) 2019  Renato Lima - Akretion
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html

{
    "name": "NF-e",
    "summary": "Brazilian Eletronic Invoice NF-e .",
    "category": "Localisation",
    "license": "AGPL-3",
    "author": "Akretion," "KMEE," "Odoo Community Association (OCA)",
    "website": "https://github.com/OCA/l10n-brazil",
    "development_status": "Beta",
    "version": "14.0.3.0.0",
    "depends": [
        "l10n_br_fiscal",
        "l10n_br_nfe_spec",
        "spec_driven_model",
    ],
    "data": [
        # Data
        "data/ir_config_parameter.xml",
        # Security
        "security/nfe_security.xml",
        # Views
        "views/res_company_view.xml",
        "views/nfe_document_view.xml",
        "views/res_config_settings_view.xml",
    ],
    "demo": [
        "demo/res_users_demo.xml",
        "demo/fiscal_document_demo.xml",
        "demo/company_demo.xml",
    ],
    "post_init_hook": "post_init_hook",
    "installable": True,
    "auto_install": False,
    "external_dependencies": {
        "python": [
            "nfelib",
            "erpbrasil.base",
            "erpbrasil.assinatura",
            "erpbrasil.transmissao",
            "erpbrasil.edoc",
            "erpbrasil.edoc.pdf",
            "xmldiff",
        ],
    },
}
<commit_msg>l10n_br_nfe 14.0.4.0.0<commit_after># Copyright (C) 2019  Renato Lima - Akretion
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html

{
    "name": "NF-e",
    "summary": "Brazilian Eletronic Invoice NF-e .",
    "category": "Localisation",
    "license": "AGPL-3",
    "author": "Akretion," "KMEE," "Odoo Community Association (OCA)",
    "website": "https://github.com/OCA/l10n-brazil",
    "development_status": "Beta",
    "version": "14.0.4.0.0",
    "depends": [
        "l10n_br_fiscal",
        "l10n_br_nfe_spec",
        "spec_driven_model",
    ],
    "data": [
        # Data
        "data/ir_config_parameter.xml",
        # Security
        "security/nfe_security.xml",
        # Views
        "views/res_company_view.xml",
        "views/nfe_document_view.xml",
        "views/res_config_settings_view.xml",
    ],
    "demo": [
        "demo/res_users_demo.xml",
        "demo/fiscal_document_demo.xml",
        "demo/company_demo.xml",
    ],
    "post_init_hook": "post_init_hook",
    "installable": True,
    "auto_install": False,
    "external_dependencies": {
        "python": [
            "nfelib",
            "erpbrasil.base",
            "erpbrasil.assinatura",
            "erpbrasil.transmissao",
            "erpbrasil.edoc",
            "erpbrasil.edoc.pdf",
            "xmldiff",
        ],
    },
}
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2011-2017 Camptocamp SA
# Copyright 2014 Olivier Jossen (brain-tec AG)
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
{
    'name': 'Switzerland - Postal codes (ZIP) list',
    'version': '10.0.1.0.0',
    'author': '''
        Camptocamp,
        brain-tec AG,
        copadoMEDIA UG,
        Odoo Community Association (OCA)
    ''',
    'category': 'Localisation',
    'website': 'http://www.camptocamp.com',
    'license': 'AGPL-3',
    'summary': 'Provides all Swiss postal codes for auto-completion',
    'depends': [
        'base',
        'base_location',  # in https://github.com/OCA/partner-contact/
        'l10n_ch_states',  # in https://github.com/OCA/l10n-switzerland/
    ],
    # We use csv file as xml is too slow
    # unfortunately it doesn't work with noupdate thus we use a post_init hook
    # 'init': ['data/res.better.zip.csv'],
    'post_init_hook': 'post_init',
    'images': [],
    'demo': [],
    'auto_install': False,
    'installable': True,
    'application': True,
}
<commit_msg>Bump version<commit_after># -*- coding: utf-8 -*-
# Copyright 2011-2017 Camptocamp SA
# Copyright 2014 Olivier Jossen (brain-tec AG)
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
{
    'name': 'Switzerland - Postal codes (ZIP) list',
    'version': '10.0.1.0.1',
    'author': '''
        Camptocamp,
        brain-tec AG,
        copadoMEDIA UG,
        Odoo Community Association (OCA)
    ''',
    'category': 'Localisation',
    'website': 'http://www.camptocamp.com',
    'license': 'AGPL-3',
    'summary': 'Provides all Swiss postal codes for auto-completion',
    'depends': [
        'base',
        'base_location',  # in https://github.com/OCA/partner-contact/
        'l10n_ch_states',  # in https://github.com/OCA/l10n-switzerland/
    ],
    # We use csv file as xml is too slow
    # unfortunately it doesn't work with noupdate thus we use a post_init hook
    # 'init': ['data/res.better.zip.csv'],
    'post_init_hook': 'post_init',
    'images': [],
    'demo': [],
    'auto_install': False,
    'installable': True,
    'application': True,
}
<|endoftext|>