<commit_before>""" Class for windowed template matching over a spatial grid """

import dem
from osgeo import osr, ogr, gdal

import math
import numpy as np
import numexpr

from scipy.signal import convolve2d
from scipy.special import erf, erfinv
import matplotlib.pyplot as plt

np.seterr(divide='ignore', invalid='ignore')

DEFAULT_EPSG = 32610 # UTM 10N

class WindowedTemplate(object):

    def __init__(self):

        self.d = None
        self.alpha = None
        self.nx = None
        self.ny = None
        self.de = None

    def parse_args(self, **kwargs):

        keys = tuple(kwargs.keys())
        values = tuple(kwargs.values())

        args = dict(zip(keys, values))

        return args

    def template(self):
        pass

    def get_window_limits(self):

        x4 = self.d*np.cos(self.alpha - np.pi/2)
        y4 = self.d*np.sin(self.alpha - np.pi/2)
        x1 = self.d*np.cos(self.alpha)
        y1 = self.d*np.sin(self.alpha)
        an_y = abs((x4 - x1) + 2 * self.c * np.cos(self.alpha - np.pi/2))
        an_x = abs((y1 - y4) + 2 * self.c * np.sin(self.alpha - np.pi/2))

        x = self.de*np.linspace(1, self.nx, num=self.nx)
        y = self.de*np.linspace(1, self.ny, num=self.ny)
        x = x - np.mean(x)
        y = y - np.mean(y)

        X, Y = np.meshgrid(x, y)
        mask = ((X < (min(x) + an_x)) | (X > (max(x) - an_x)) | (Y < (min(y) + an_y)) | (Y > (max(y) - an_y)))

        return mask

class Scarp(WindowedTemplate):

    def __init__(self, d, kt, alpha, nx, ny, de):

        self.d = d
        self.kt = kt
        self.alpha = -alpha
        self.nx = nx
        self.ny = ny
        self.de = de

        frac = 0.9
        self.c = abs(2 * np.sqrt(self.kt) * erfinv(frac))

    def template(self):

        x = self.de * np.linspace(1, self.nx, num=self.nx)
        y = self.de * np.linspace(1, self.ny, num=self.ny)
        x = x - np.mean(x)
        y = y - np.mean(y)

        x, y = np.meshgrid(x, y)
        xr = x * np.cos(self.alpha) + y * np.sin(self.alpha)
        yr = -x * np.sin(self.alpha) + y * np.cos(self.alpha)

        W = (-xr / (2. * self.kt ** (3 / 2.) * np.sqrt(np.pi))) \
            * np.exp(-xr ** 2. / (4. * self.kt))

        mask = (abs(xr) < self.c) & (abs(yr) < self.d)
        W = W * mask
        #W = W.T
        W = np.flipud(np.fliplr(W))

        return W

    def template_numexpr(self):

        alpha = self.alpha
        kt = self.kt
        c = self.c
        d = self.d

        x = self.de * np.linspace(1, self.nx, num=self.nx)
        y = self.de * np.linspace(1, self.ny, num=self.ny)
        x = x - np.mean(x)
        y = y - np.mean(y)

        x, y = np.meshgrid(x, y)
        xr = numexpr.evaluate("x * cos(alpha) + y * sin(alpha)")
        yr = numexpr.evaluate("-x * sin(alpha) + y * cos(alpha)")

        pi = np.pi
        W = numexpr.evaluate("(-xr / (2 * kt ** (3/2) * sqrt(pi))) * exp(-xr ** 2 / (4 * kt))")

        mask = numexpr.evaluate("(abs(xr) < c) & (abs(yr) < d)")
        W = numexpr.evaluate("W * mask")
        #W = W.T
        W = np.flipud(np.fliplr(W))

        return W

class Morlet(WindowedTemplate):

    def __init__(self, d, kt, alpha):

        self.d = d
        self.kt = kt
        self.alpha = alpha

    def template(self):
        pass

class Ricker(WindowedTemplate):

    def __init__(self, d, f, alpha, nx, ny, de):

        self.d = d
        self.f = f
        self.alpha = -alpha
        self.nx = nx
        self.ny = ny
        self.de = de

    def template(self):

        alpha = self.alpha
        #c = self.c
        d = self.d
        f = self.f

        x = self.de * np.linspace(1, self.nx, num=self.nx)
        y = self.de * np.linspace(1, self.ny, num=self.ny)
        x = x - np.mean(x)
        y = y - np.mean(y)

        x, y = np.meshgrid(x, y)
        xr = numexpr.evaluate("x * cos(alpha) + y * sin(alpha)")
        yr = numexpr.evaluate("-x * sin(alpha) + y * cos(alpha)")

        pi = np.pi
        W = numexpr.evaluate("(1. - 2. * (pi * f * xr) ** 2.) * exp(-(pi * f * xr) ** 2.)")

        #mask = numexpr.evaluate("(abs(xr) < c) & (abs(yr) < d)")
        #W = numexpr.evaluate("W*mask")
        #W = W.T
        #W = np.flipud(np.fliplr(W))

        return W

class Channel(Ricker):
    pass

<commit_msg>Clean up imports<commit_after>""" Class for windowed template matching over a spatial grid """

import math
import numpy as np
import numexpr

import matplotlib.pyplot as plt

from scipy.special import erf, erfinv

np.seterr(divide='ignore', invalid='ignore')

class WindowedTemplate(object):

    def __init__(self):

        self.d = None
        self.alpha = None
        self.nx = None
        self.ny = None
        self.de = None

    def parse_args(self, **kwargs):

        keys = tuple(kwargs.keys())
        values = tuple(kwargs.values())

        args = dict(zip(keys, values))

        return args

    def template(self):
        pass

    def get_window_limits(self):

        x4 = self.d*np.cos(self.alpha - np.pi/2)
        y4 = self.d*np.sin(self.alpha - np.pi/2)
        x1 = self.d*np.cos(self.alpha)
        y1 = self.d*np.sin(self.alpha)
        an_y = abs((x4 - x1) + 2 * self.c * np.cos(self.alpha - np.pi/2))
        an_x = abs((y1 - y4) + 2 * self.c * np.sin(self.alpha - np.pi/2))

        x = self.de*np.linspace(1, self.nx, num=self.nx)
        y = self.de*np.linspace(1, self.ny, num=self.ny)
        x = x - np.mean(x)
        y = y - np.mean(y)

        X, Y = np.meshgrid(x, y)
        mask = ((X < (min(x) + an_x)) | (X > (max(x) - an_x)) | (Y < (min(y) + an_y)) | (Y > (max(y) - an_y)))

        return mask

class Scarp(WindowedTemplate):

    def __init__(self, d, kt, alpha, nx, ny, de):

        self.d = d
        self.kt = kt
        self.alpha = -alpha
        self.nx = nx
        self.ny = ny
        self.de = de

        frac = 0.9
        self.c = abs(2 * np.sqrt(self.kt) * erfinv(frac))

    def template(self):

        x = self.de * np.linspace(1, self.nx, num=self.nx)
        y = self.de * np.linspace(1, self.ny, num=self.ny)
        x = x - np.mean(x)
        y = y - np.mean(y)

        x, y = np.meshgrid(x, y)
        xr = x * np.cos(self.alpha) + y * np.sin(self.alpha)
        yr = -x * np.sin(self.alpha) + y * np.cos(self.alpha)

        W = (-xr / (2. * self.kt ** (3 / 2.) * np.sqrt(np.pi))) \
            * np.exp(-xr ** 2. / (4. * self.kt))

        mask = (abs(xr) < self.c) & (abs(yr) < self.d)
        W = W * mask
        #W = W.T
        W = np.flipud(np.fliplr(W))

        return W

    def template_numexpr(self):

        alpha = self.alpha
        kt = self.kt
        c = self.c
        d = self.d

        x = self.de * np.linspace(1, self.nx, num=self.nx)
        y = self.de * np.linspace(1, self.ny, num=self.ny)
        x = x - np.mean(x)
        y = y - np.mean(y)

        x, y = np.meshgrid(x, y)
        xr = numexpr.evaluate("x * cos(alpha) + y * sin(alpha)")
        yr = numexpr.evaluate("-x * sin(alpha) + y * cos(alpha)")

        pi = np.pi
        W = numexpr.evaluate("(-xr / (2 * kt ** (3/2) * sqrt(pi))) * exp(-xr ** 2 / (4 * kt))")

        mask = numexpr.evaluate("(abs(xr) < c) & (abs(yr) < d)")
        W = numexpr.evaluate("W * mask")
        #W = W.T
        W = np.flipud(np.fliplr(W))

        return W

class Morlet(WindowedTemplate):

    def __init__(self, d, kt, alpha):

        self.d = d
        self.kt = kt
        self.alpha = alpha

    def template(self):
        pass

class Ricker(WindowedTemplate):

    def __init__(self, d, f, alpha, nx, ny, de):

        self.d = d
        self.f = f
        self.alpha = -alpha
        self.nx = nx
        self.ny = ny
        self.de = de

    def template(self):

        alpha = self.alpha
        #c = self.c
        d = self.d
        f = self.f

        x = self.de * np.linspace(1, self.nx, num=self.nx)
        y = self.de * np.linspace(1, self.ny, num=self.ny)
        x = x - np.mean(x)
        y = y - np.mean(y)

        x, y = np.meshgrid(x, y)
        xr = numexpr.evaluate("x * cos(alpha) + y * sin(alpha)")
        yr = numexpr.evaluate("-x * sin(alpha) + y * cos(alpha)")

        pi = np.pi
        W = numexpr.evaluate("(1. - 2. * (pi * f * xr) ** 2.) * exp(-(pi * f * xr) ** 2.)")

        #mask = numexpr.evaluate("(abs(xr) < c) & (abs(yr) < d)")
        #W = numexpr.evaluate("W*mask")
        #W = W.T
        #W = np.flipud(np.fliplr(W))

        return W

class Channel(Ricker):
    pass

<|endoftext|>


<commit_before>import os

import h5py

from tools import build_transfer_net
from tools.datasets.urban_tribes import load_data

class Session(object):
    """Training session.

    This is a helper class to bundle model and history together and make it
    easy to resume training and recording history.
    """
    def __init__(self, model, history=None):
        self.model = model
        self.history = history

    def dump(self, path):
        W, b = self.model.layers[-1].get_weights()
        with h5py.File(path, 'w') as f:
            f.create_dataset('W', data=W)
            f.create_dataset('b', data=b)
            group = f.create_group('history')
            for k, v in self.history.items():
                group.create_dataset(k, data=v)

    @classmethod
    def load(cls, model, path):
        W, b = cls.load_weights(path)
        history = cls.load_history(path)
        model.layers[-1].set_weights([W, b])
        return cls(model, history=history)

    def train(self, *args, **kwargs):
        new_history = self.model.fit(*args, **kwargs)
        self._record(new_history.history)

    def _record(self, new_history):
        if self.history is None:
            self.history = new_history
        else:
            for key in self.history.keys():
                self.history[key].extend(new_history[key])

    @staticmethod
    def load_history(path):
        with h5py.File(path, 'r') as f:
            history = {}
            for k, v in f['history'].items():
                history[k] = list(v)
        return history

    @staticmethod
    def load_weights(path):
        with h5py.File(path, 'r') as f:
            W = f['W'][:]
            b = f['b'][:]
        return W, b

def transfer_learn(layer_name, nb_sample, nb_epoch, output_file):
    """Transfer learning for image classification.

    Args:
        layer_name: Transfer layer name.
        nb_sample: Number of samples per categories.
        nb_epoch: Number of epochs to train in total.
        output_file: Name of the output file to pick history to.
    """
    # Build model
    model = build_transfer_net(output_dim=11,
                               transfer_layer_name=layer_name)

    # Prepare data
    (x_train, y_train), (x_val, y_val), (x_test, y_test) = \
        load_data(images_per_category=nb_sample)

    # Train
    model.compile(optimizer='adadelta', loss='categorical_crossentropy',
                  metrics=['accuracy'])
    if os.path.exists(output_file):
        print('Resuming')
        session = Session.load(model, output_file)
        nb_epoch -= len(session.history['loss'])
        if nb_epoch < 0:
            return session
    else:
        print('Starting')
        session = Session(model)
    session.train(x_train, y_train, batch_size=nb_sample, nb_epoch=nb_epoch,
                  validation_data=(x_val, y_val))
    session.dump(output_file)
    return session
<commit_msg>Don't train when nb_epoch is 0<commit_after>import os

import h5py

from tools import build_transfer_net
from tools.datasets.urban_tribes import load_data

class Session(object):
    """Training session.

    This is a helper class to bundle model and history together and make it
    easy to resume training and recording history.
    """
    def __init__(self, model, history=None):
        self.model = model
        self.history = history

    def dump(self, path):
        W, b = self.model.layers[-1].get_weights()
        with h5py.File(path, 'w') as f:
            f.create_dataset('W', data=W)
            f.create_dataset('b', data=b)
            group = f.create_group('history')
            for k, v in self.history.items():
                group.create_dataset(k, data=v)

    @classmethod
    def load(cls, model, path):
        W, b = cls.load_weights(path)
        history = cls.load_history(path)
        model.layers[-1].set_weights([W, b])
        return cls(model, history=history)

    def train(self, *args, **kwargs):
        new_history = self.model.fit(*args, **kwargs)
        self._record(new_history.history)

    def _record(self, new_history):
        if self.history is None:
            self.history = new_history
        else:
            for key in self.history.keys():
                self.history[key].extend(new_history[key])

    @staticmethod
    def load_history(path):
        with h5py.File(path, 'r') as f:
            history = {}
            for k, v in f['history'].items():
                history[k] = list(v)
        return history

    @staticmethod
    def load_weights(path):
        with h5py.File(path, 'r') as f:
            W = f['W'][:]
            b = f['b'][:]
        return W, b

def transfer_learn(layer_name, nb_sample, nb_epoch, output_file):
    """Transfer learning for image classification.

    Args:
        layer_name: Transfer layer name.
        nb_sample: Number of samples per categories.
        nb_epoch: Number of epochs to train in total.
        output_file: Name of the output file to pick history to.
    """
    # Build model
    model = build_transfer_net(output_dim=11,
                               transfer_layer_name=layer_name)

    # Prepare data
    (x_train, y_train), (x_val, y_val), (x_test, y_test) = \
        load_data(images_per_category=nb_sample)

    # Train
    model.compile(optimizer='adadelta', loss='categorical_crossentropy',
                  metrics=['accuracy'])
    if os.path.exists(output_file):
        print('Resuming')
        session = Session.load(model, output_file)
        nb_epoch -= len(session.history['loss'])
        if nb_epoch <= 0:
            return session
    else:
        print('Starting')
        session = Session(model)
    session.train(x_train, y_train, batch_size=nb_sample, nb_epoch=nb_epoch,
                  validation_data=(x_val, y_val))
    session.dump(output_file)
    return session
<|endoftext|>


<commit_before>import urllib, requests, time, praw, xml.etree.ElementTree as ET, os, re

def redditescape(string):
        escape = '\\`*_{}[]()#+-.!^'
        for char in escape:
                string = string.replace(char, '\\' + char)
        return string

def getdata(mathstring,appid):
        url = 'http://api.wolframalpha.com/v2/query?input=' + urllib.quote(mathstring) + '&appid=' + appid + '&format=plaintext'
        r = requests.get(url)
        return r.text.encode('utf8')

def getimportant(xml):
        important = {}
        blacklist = ['PropertiesAsARealFunction']
        root = ET.fromstring(xml)
        for child in root.iter('pod'):
                try:
                        title = child.attrib['title']
                        pid = child.attrib['id']
                        if pid not in blacklist:
                                images = []
                                for subpod in child.findall('subpod'):
                                        for image in subpod.findall('img'):
                                                ititle = image.attrib['title']
                                                if ititle == '':
                                                        ititle = title
                                                images.append('[' + redditescape(ititle) + '](' + image.attrib['src'] + '.gif)')
                                if images != []:
                                        important[title] = '\r\n\r\n'.join(images).encode('utf8')
                except TypeError:
                        pass
        return important

def main():
        if os.path.exists("wolframids.txt") != True:
                with open("wolframids.txt",'w') as w:
                        alreadyposted = []                        #Creates database if it doesn't exist, reads if it does
        else:
                with open("wolframids.txt","a+") as w:
                        alreadyposted = w.read().splitlines()

        appid = '' # Your Wolfram Alpha appid
        r = praw.Reddit(user_agent='Wolfram Alpha Bot') # Useragent
        r.login('','') # Reddit username and password
        subreddit = '' #This is the sub or list of subs to scan for new posts. For a single sub, use "sub1". For multiple subreddits, use "sub1+sub2+sub3+..."
        while True:
                for comment in praw.helpers.comment_stream(r,subreddit,limit = None, verbosity=0):#in comments:
                        commentid = comment.id
                        if commentid not in alreadyposted:
                                body = comment.body.lower()
                                if body.find('@wolfram_bot[') != -1:
                                        try:
                                                print 'Found Comment: ' + str(commentid)
                                                regex = re.compile(r'@wolfram_bot\[.*\]')
                                                findall = regex.findall(body.encode('utf8'))
                                                redditinput = findall[0]
                                                redditinput = redditinput[13:-1]
                                                data = getimportant(getdata(redditinput,appid))
                                                rcomment = []
                                                for d in data:
                                                        rcomment.append('**' + d + ':' + '**')
                                                        rcomment.append(data[d])
                                                rcomment =  '\r\n\r\n'.join(rcomment)
                                                try:
                                                        comment.reply(rcomment)
                                                except:
                                                        comment.reply('Invalid Query')
                                                print 'Replied to ' + commentid
                                                with open("wolframids.txt","a+") as w:
                                                        w.write(commentid + '\n')
                                                        alreadyposted.append(commentid)
                                        except Exception as e:
                                                print 'Error: ' + str(e)
                print 'done'
                time.sleep(10)

if __name__ == '__main__':
        main()
<commit_msg>Encoding issue<commit_after>import urllib, requests, time, praw, xml.etree.ElementTree as ET, os, re

def redditescape(string):
        escape = '\\`*_{}[]()#+-.!^'
        for char in escape:
                string = string.replace(char, '\\' + char)
        return string

def getdata(mathstring,appid):
        url = 'http://api.wolframalpha.com/v2/query?input=' + urllib.quote(mathstring) + '&appid=' + appid + '&format=plaintext'
        r = requests.get(url)
        return r.text.encode('utf8')

def getimportant(xml):
        important = {}
        blacklist = ['PropertiesAsARealFunction']
        root = ET.fromstring(xml)
        for child in root.iter('pod'):
                try:
                        title = child.attrib['title']
                        pid = child.attrib['id']
                        if pid not in blacklist:
                                images = []
                                for subpod in child.findall('subpod'):
                                        for image in subpod.findall('img'):
                                                ititle = image.attrib['title']
                                                if ititle == '':
                                                        ititle = title
                                                images.append('[' + redditescape(ititle) + '](' + image.attrib['src'] + '.gif)')
                                if images != []:
                                        important[title] = '\r\n\r\n'.join(images).encode('utf8').decode('utf8')
                except TypeError:
                        pass
        return important

def main():
        if os.path.exists("wolframids.txt") != True:
                with open("wolframids.txt",'w') as w:
                        alreadyposted = []                        #Creates database if it doesn't exist, reads if it does
        else:
                with open("wolframids.txt","a+") as w:
                        alreadyposted = w.read().splitlines()

        appid = '' # Your Wolfram Alpha appid
        r = praw.Reddit(user_agent='Wolfram Alpha Bot') # Useragent
        r.login('','') # Reddit username and password
        subreddit = '' #This is the sub or list of subs to scan for new posts. For a single sub, use "sub1". For multiple subreddits, use "sub1+sub2+sub3+..."
        while True:
                for comment in praw.helpers.comment_stream(r,subreddit,limit = None, verbosity=0):#in comments:
                        commentid = comment.id
                        if commentid not in alreadyposted:
                                body = comment.body.lower()
                                if body.find('@wolfram_bot[') != -1:
                                        try:
                                                print 'Found Comment: ' + str(commentid)
                                                regex = re.compile(r'@wolfram_bot\[.*\]')
                                                findall = regex.findall(body.encode('utf8'))
                                                redditinput = findall[0]
                                                redditinput = redditinput[13:-1]
                                                data = getimportant(getdata(redditinput,appid))
                                                rcomment = []
                                                for d in data:
                                                        rcomment.append('**' + d + ':' + '**')
                                                        rcomment.append(data[d])
                                                rcomment =  '\r\n\r\n'.join(rcomment)
                                                try:
                                                        comment.reply(rcomment)
                                                except:
                                                        comment.reply('Invalid Query')
                                                print 'Replied to ' + commentid
                                                with open("wolframids.txt","a+") as w:
                                                        w.write(commentid + '\n')
                                                        alreadyposted.append(commentid)
                                        except Exception as e:
                                                print 'Error: ' + str(e)
                print 'done'
                time.sleep(10)

if __name__ == '__main__':
        main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import csv
import getopt
import urllib
import urlparse

import sys
from tabulate import tabulate

metrics = ["#QuerySize", "#TripleCountWithService"]

help = 'Usage: viewData.py -d <day> -s <startline> -e <endline>'
day = 0
start = 0
end = 0
day_given = False
start_given = False
end_given = False

processedPrefix = "QueryProcessedOpenRDF"
sourcePrefix = "queryCnt"

try:
    opts, args = getopt.getopt(sys.argv[1:], "hd:s:e:", ["day=", "startline=", "endline="])
except getopt.GetoptError:
    print help
    sys.exit(2)
for opt, arg in opts:
    if opt == "-h":
        print help
        sys.exit()
    elif opt in ("-d", "--day"):
        day = int(arg)
        day_given = True
    elif opt in ("-s", "--startline"):
        start = int(arg)
        start_given = True
    elif opt in ("-e", "--endline"):
        end = int(arg)
        end_given = True

if not (day_given and start_given and end_given):
    print help
    sys.exit()

with open(processedPrefix + "%02d" % day + ".tsv") as p, open(sourcePrefix + "%02d" % day + ".tsv") as s:
    pReader = csv.DictReader(p, delimiter="\t")
    sReader = csv.DictReader(s, delimiter="\t")

    i = 0
    for processed, source in zip(pReader, sReader):
        if start <= i <= end:
            data = [[]]
            for metric in metrics:
                data[0].append(processed[metric])
            print tabulate(data, headers=metrics)
            d = dict(urlparse.parse_qsl(urlparse.urlsplit(source['uri_query']).query))
            if 'query' in d.keys():
                print d['query']
            else:
                print "Could not find query in uri_query."

        i += 1
<commit_msg>FIX: Switched viewData to izip<commit_after># -*- coding: utf-8 -*-

import csv
import getopt
import urllib
import urlparse
import sys

from tabulate import tabulate
from itertools import izip

metrics = ["#QuerySize", "#TripleCountWithService"]

help = 'Usage: viewData.py -d <day> -s <startline> -e <endline>'
day = 0
start = 0
end = 0
day_given = False
start_given = False
end_given = False

processedPrefix = "QueryProcessedOpenRDF"
sourcePrefix = "queryCnt"

try:
    opts, args = getopt.getopt(sys.argv[1:], "hd:s:e:", ["day=", "startline=", "endline="])
except getopt.GetoptError:
    print help
    sys.exit(2)
for opt, arg in opts:
    if opt == "-h":
        print help
        sys.exit()
    elif opt in ("-d", "--day"):
        day = int(arg)
        day_given = True
    elif opt in ("-s", "--startline"):
        start = int(arg)
        start_given = True
    elif opt in ("-e", "--endline"):
        end = int(arg)
        end_given = True

if not (day_given and start_given and end_given):
    print help
    sys.exit()

with open(processedPrefix + "%02d" % day + ".tsv") as p, open(sourcePrefix + "%02d" % day + ".tsv") as s:
    pReader = csv.DictReader(p, delimiter="\t")
    sReader = csv.DictReader(s, delimiter="\t")

    i = 0
    for processed, source in izip(pReader, sReader):
        if start <= i <= end:
            data = [[]]
            for metric in metrics:
                data[0].append(processed[metric])
            print tabulate(data, headers=metrics)
            d = dict(urlparse.parse_qsl(urlparse.urlsplit(source['uri_query']).query))
            if 'query' in d.keys():
                print d['query']
            else:
                print "Could not find query in uri_query."

        i += 1
<|endoftext|>


<commit_before># IEEEbot
# Copyright (C) 2015 Rafael Bailón-Ruiz <rafaelbailon@ieee.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import telebot
import telebot.types
import time
import logging
import sys
import re

import os

from storage import Storage

# Regular expressions.
USERNAME_PLUS_REGEXP_SEARCH = '@([a-zA-Z0-9_]+)(\+{2,})'
USERNAME_MINUS_REGEXP_SEARCH = '@([a-zA-Z0-9_]+)(\-{2,})'
MAX_POINTS = 20

# Database file path
DATABASE_FILE = os.path.join(os.environ['OPENSHIFT_DATA_DIR'],'database.sqlite')

# Load token from environment variable 'IEEEBOT_TOKEN'.
TOKEN = os.environ['IEEEBOT_TOKEN']

# Create a database object to store points
storage = Storage(DATABASE_FILE)

# Check if database existe, if not create it
if not os.path.isfile(DATABASE_FILE):
    storage.initialize()

# Create a bot
bot = telebot.TeleBot(TOKEN)

# Logging
logger = telebot.logger
formatter = logging.Formatter('[%(asctime)s] %(thread)d \
  {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s', '%m-%d %H:%M:%S')
ch = logging.StreamHandler(sys.stdout)
logger.addHandler(ch)
logger.setLevel(logging.DEBUG)  # or use logging.INFO
ch.setFormatter(formatter)

# Keep last update timestamp
last_update_id = 0

def process_update(update):
    message = telebot.types.Message.de_json(update['message'])
    bot.process_new_messages([message])

def get_karma_ranking_message():
    logger.debug("entra get_karma_ranking_message")
    text = "📊 Ranking actual:\n\n"

    karma_ranking = storage.ranking

    if karma_ranking:
        for entry in karma_ranking:
            text += "‣ {0}: {1} puntos\n".format(entry[0], entry[1])
        logger.debug("sale get_karma_ranking_message")
        return text

@bot.message_handler(commands=['ranking'])
def ranking_handler(message):
    """
    Send karma ranking
    """
    logger.debug("entra ranking_handler")
    bot.reply_to(message, get_karma_ranking_message())
    logger.debug("sale ranking_handler")

def update_karma(user_name, points):
    # No more than 5 points
    if user_name != 'aroldan':
        points = MAX_POINTS if points > MAX_POINTS else points
        points = -MAX_POINTS if points < -MAX_POINTS else points

    # SQL query will add points and insert a new user if it does not exist.
    # It will returns the final karma points too.
    return storage.update_user_karma(user_name, points)

@bot.message_handler(regexp=USERNAME_PLUS_REGEXP_SEARCH)
def mas1_handler(message):
    """
    Increment karma
    """
    m = re.search(USERNAME_PLUS_REGEXP_SEARCH, message.text)
    if m:
        # Get user name and karma points.
        user_name = m.group(1)
        points = len(m.group(2)) - 1

        # One cannot give karma to itself
        if message.from_user.username == user_name:
            bot.reply_to(message, "Ni lo intentes... 😒")
            return

        karma = update_karma(user_name, points)
        bot.reply_to(message,
                     "El karma de {0} ha aumentado a {1} 👍\n"
                     .format(user_name, karma))

@bot.message_handler(regexp=USERNAME_MINUS_REGEXP_SEARCH)
def menos1_handler(message):
    """
    Decrement karma
    """
    m = re.match(USERNAME_MINUS_REGEXP_SEARCH, message.text)
    if m:
        # Get user name and karma points.
        user_name = m.group(1)
        points = (len(m.group(2)) - 1) * -1

        # One cannot give karma to itself
        if message.from_user.username == user_name:
            bot.reply_to(message, "Tontos hay en todos lados 😆")
            return

        karma = update_karma(user_name, points)
        bot.reply_to(message,
                     "El karma de {0} ha bajado a {1} 👎\n"
                     .format(user_name, karma))

if __name__ == '__main__':
    # Use none_stop flag to not stop when get new message occur error.
    # Interval setup. Sleep 3 secs between request new message.
    bot.polling(none_stop=True, interval=3)
    bot.polling_thread.join()
<commit_msg>Disallow karma change form private chats<commit_after># IEEEbot
# Copyright (C) 2015 Rafael Bailón-Ruiz <rafaelbailon@ieee.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import telebot
import telebot.types
import time
import logging
import sys
import re

import os

from storage import Storage

# Regular expressions.
USERNAME_PLUS_REGEXP_SEARCH = '@([a-zA-Z0-9_]+)(\+{2,})'
USERNAME_MINUS_REGEXP_SEARCH = '@([a-zA-Z0-9_]+)(\-{2,})'
MAX_POINTS = 20

# Database file path
DATABASE_FILE = os.path.join(os.environ['OPENSHIFT_DATA_DIR'],'database.sqlite')

# Load token from environment variable 'IEEEBOT_TOKEN'.
TOKEN = os.environ['IEEEBOT_TOKEN']

# Create a database object to store points
storage = Storage(DATABASE_FILE)

# Check if database existe, if not create it
if not os.path.isfile(DATABASE_FILE):
    storage.initialize()

# Create a bot
bot = telebot.TeleBot(TOKEN)

# Logging
logger = telebot.logger
formatter = logging.Formatter('[%(asctime)s] %(thread)d \
  {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s', '%m-%d %H:%M:%S')
ch = logging.StreamHandler(sys.stdout)
logger.addHandler(ch)
logger.setLevel(logging.DEBUG)  # or use logging.INFO
ch.setFormatter(formatter)

# Keep last update timestamp
last_update_id = 0

def process_update(update):
    message = telebot.types.Message.de_json(update['message'])
    bot.process_new_messages([message])

def get_karma_ranking_message():
    logger.debug("entra get_karma_ranking_message")
    text = "📊 Ranking actual:\n\n"

    karma_ranking = storage.ranking

    if karma_ranking:
        for entry in karma_ranking:
            text += "‣ {0}: {1} puntos\n".format(entry[0], entry[1])
        logger.debug("sale get_karma_ranking_message")
        return text

@bot.message_handler(commands=['ranking'])
def ranking_handler(message):
    """
    Send karma ranking
    """
    logger.debug("entra ranking_handler")
    bot.reply_to(message, get_karma_ranking_message())
    logger.debug("sale ranking_handler")

def update_karma(user_name, points):
    # No more than 5 points
    if user_name != 'aroldan':
        points = MAX_POINTS if points > MAX_POINTS else points
        points = -MAX_POINTS if points < -MAX_POINTS else points

    # SQL query will add points and insert a new user if it does not exist.
    # It will returns the final karma points too.
    return storage.update_user_karma(user_name, points)

@bot.message_handler(regexp=USERNAME_PLUS_REGEXP_SEARCH)
def mas1_handler(message):
    """
    Increment karma
    """
    m = re.search(USERNAME_PLUS_REGEXP_SEARCH, message.text)
    if m:
        # Get user name and karma points.
        user_name = m.group(1)
        points = len(m.group(2)) - 1

        # Don't allow karma changes in private conversations
        if not isinstance(message.chat, telebot.types.GroupChat):
            bot.reply_to(message, "Nada de subir karma por lo bajini ¿eh? 😒")
            return

        # One cannot give karma to itself
        if message.from_user.username == user_name:
            bot.reply_to(message, "Ni lo intentes... 😒")
            return

        karma = update_karma(user_name, points)
        bot.reply_to(message,
                     "El karma de {0} ha aumentado a {1} 👍\n"
                     .format(user_name, karma))

@bot.message_handler(regexp=USERNAME_MINUS_REGEXP_SEARCH)
def menos1_handler(message):
    """
    Decrement karma
    """
    m = re.match(USERNAME_MINUS_REGEXP_SEARCH, message.text)
    if m:
        # Get user name and karma points.
        user_name = m.group(1)
        points = (len(m.group(2)) - 1) * -1

        # Don't allow karma changes in private conversations
        if not isinstance(message.chat, telebot.types.GroupChat):
            bot.reply_to(message, "Las humillaciones sólo en público, por favor 😈")
            return

        # One cannot give karma to itself
        if message.from_user.username == user_name:
            bot.reply_to(message, "Tontos hay en todos lados 😆")
            return

        karma = update_karma(user_name, points)
        bot.reply_to(message,
                     "El karma de {0} ha bajado a {1} 👎\n"
                     .format(user_name, karma))

if __name__ == '__main__':
    # Use none_stop flag to not stop when get new message occur error.
    # Interval setup. Sleep 3 secs between request new message.
    bot.polling(none_stop=True, interval=3)
    bot.polling_thread.join()
<|endoftext|>


<commit_before>import pickle
from datetime import timedelta
from uuid import uuid4
from redis import Redis
from werkzeug.datastructures import CallbackDict
from flask.sessions import SessionInterface, SessionMixin

class RedisSession(CallbackDict, SessionMixin):

    def __init__(self, initial=None, sid=None, new=False):
        def on_update(self):
            self.modified = True
        CallbackDict.__init__(self, initial, on_update)
        self.sid = sid
        self.new = new
        self.modified = False

class RedisSessionInterface(SessionInterface):
    serializer = pickle
    session_class = RedisSession

    def __init__(self, redis=None, prefix='session:'):
        if redis is None:
            redis = Redis()
        self.redis = redis
        self.prefix = prefix

    def generate_sid(self):
        return str(uuid4())

    def get_redis_expiration_time(self, app, session):
        if session.permanent:
            return app.permanent_session_lifetime
        return timedelta(days=1)

    def open_session(self, app, request):
        sid = request.cookies.get(app.session_cookie_name)
        if not sid:
            sid = self.generate_sid()
            return self.session_class(sid=sid, new=True)
        val = self.redis.get(self.prefix + sid)
        if val is not None:
            data = self.serializer.loads(val)
            return self.session_class(data, sid=sid)
        return self.session_class(sid=sid, new=True)

    def save_session(self, app, session, response):
        domain = self.get_cookie_domain(app)
        if not session:
            self.redis.delete(self.prefix + session.sid)
            if session.modified:
                response.delete_cookie(app.session_cookie_name,
                                       domain=domain)
            return
        redis_exp = self.get_redis_expiration_time(app, session)
        cookie_exp = self.get_expiration_time(app, session)
        val = self.serializer.dumps(dict(session))
        self.redis.setex(self.prefix + session.sid, val,
                         int(redis_exp.total_seconds()))
        response.set_cookie(app.session_cookie_name, session.sid,
                            expires=cookie_exp, httponly=True,
                            domain=domain)
<commit_msg>add workaround for python <2.7<commit_after>import pickle
import sys
from datetime import timedelta
from uuid import uuid4
from redis import Redis
from werkzeug.datastructures import CallbackDict
from flask.sessions import SessionInterface, SessionMixin

class RedisSession(CallbackDict, SessionMixin):

    def __init__(self, initial=None, sid=None, new=False):
        def on_update(self):
            self.modified = True
        CallbackDict.__init__(self, initial, on_update)
        self.sid = sid
        self.new = new
        self.modified = False

class RedisSessionInterface(SessionInterface):
    serializer = pickle
    session_class = RedisSession

    def __init__(self, redis=None, prefix='session:'):
        if redis is None:
            redis = Redis()
        self.redis = redis
        self.prefix = prefix

    def generate_sid(self):
        return str(uuid4())

    def get_redis_expiration_time(self, app, session):
        if session.permanent:
            return app.permanent_session_lifetime
        return timedelta(days=1)

    def open_session(self, app, request):
        sid = request.cookies.get(app.session_cookie_name)
        if not sid:
            sid = self.generate_sid()
            return self.session_class(sid=sid, new=True)
        val = self.redis.get(self.prefix + sid)
        if val is not None:
            data = self.serializer.loads(val)
            return self.session_class(data, sid=sid)
        return self.session_class(sid=sid, new=True)

    def save_session(self, app, session, response):
        domain = self.get_cookie_domain(app)
        if not session:
            self.redis.delete(self.prefix + session.sid)
            if session.modified:
                response.delete_cookie(app.session_cookie_name,
                                       domain=domain)
            return
        redis_exp = self.get_redis_expiration_time(app, session)
        cookie_exp = self.get_expiration_time(app, session)
        val = self.serializer.dumps(dict(session))

        if sys.version_info < (2, 7, 0):
            def total_seconds(td):
                return td.days * 60 * 60 * 24 + td.seconds

            self.redis.setex(self.prefix + session.sid, val,
                             int(total_seconds(redis_exp)))
        else:
            self.redis.setex(self.prefix + session.sid, val,
                             int(redis_exp.total_seconds()))

        response.set_cookie(app.session_cookie_name, session.sid,
                            expires=cookie_exp, httponly=True,
                            domain=domain)
<|endoftext|>


<commit_before>import sys
import logging
import unittest
from unittest.case import _Outcome
from torba.orchstr8 import Conductor
from torba.orchstr8.node import BlockchainNode, WalletNode
from torba.client.baseledger import BaseLedger
from torba.client.baseaccount import BaseAccount
from torba.client.basemanager import BaseWalletManager
from torba.client.wallet import Wallet
from typing import Optional

try:
    import asyncio
    from asyncio.runners import _cancel_all_tasks  # type: ignore
except ImportError:
    import asyncio

    # this is only available in py3.7
    def _cancel_all_tasks(loop):
        pass

HANDLER = logging.StreamHandler(sys.stdout)
HANDLER.setFormatter(
    logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
)
logging.getLogger().addHandler(HANDLER)

class AsyncioTestCase(unittest.TestCase):
    # Implementation inspired by discussion:
    #  https://bugs.python.org/issue32972

    async def asyncSetUp(self):  # pylint: disable=C0103
        pass

    async def asyncTearDown(self):  # pylint: disable=C0103
        pass

    async def doAsyncCleanups(self):  # pylint: disable=C0103
        pass

    def run(self, result=None):  # pylint: disable=R0915
        orig_result = result
        if result is None:
            result = self.defaultTestResult()
            startTestRun = getattr(result, 'startTestRun', None)  # pylint: disable=C0103
            if startTestRun is not None:
                startTestRun()

        result.startTest(self)

        testMethod = getattr(self, self._testMethodName)  # pylint: disable=C0103
        if (getattr(self.__class__, "__unittest_skip__", False) or
                getattr(testMethod, "__unittest_skip__", False)):
            # If the class or method was skipped.
            try:
                skip_why = (getattr(self.__class__, '__unittest_skip_why__', '')
                            or getattr(testMethod, '__unittest_skip_why__', ''))
                self._addSkip(result, self, skip_why)
            finally:
                result.stopTest(self)
            return
        expecting_failure_method = getattr(testMethod,
                                           "__unittest_expecting_failure__", False)
        expecting_failure_class = getattr(self,
                                          "__unittest_expecting_failure__", False)
        expecting_failure = expecting_failure_class or expecting_failure_method
        outcome = _Outcome(result)
        try:
            self._outcome = outcome

            loop = asyncio.new_event_loop()
            try:
                asyncio.set_event_loop(loop)
                loop.set_debug(True)

                with outcome.testPartExecutor(self):
                    self.setUp()
                    loop.run_until_complete(self.asyncSetUp())
                if outcome.success:
                    outcome.expecting_failure = expecting_failure
                    with outcome.testPartExecutor(self, isTest=True):
                        possible_coroutine = testMethod()
                        if asyncio.iscoroutine(possible_coroutine):
                            loop.run_until_complete(possible_coroutine)
                    outcome.expecting_failure = False
                    with outcome.testPartExecutor(self):
                        loop.run_until_complete(self.asyncTearDown())
                        self.tearDown()
            finally:
                try:
                    _cancel_all_tasks(loop)
                    loop.run_until_complete(loop.shutdown_asyncgens())
                finally:
                    asyncio.set_event_loop(None)
                    loop.close()

            self.doCleanups()

            for test, reason in outcome.skipped:
                self._addSkip(result, test, reason)
            self._feedErrorsToResult(result, outcome.errors)
            if outcome.success:
                if expecting_failure:
                    if outcome.expectedFailure:
                        self._addExpectedFailure(result, outcome.expectedFailure)
                    else:
                        self._addUnexpectedSuccess(result)
                else:
                    result.addSuccess(self)
            return result
        finally:
            result.stopTest(self)
            if orig_result is None:
                stopTestRun = getattr(result, 'stopTestRun', None)  # pylint: disable=C0103
                if stopTestRun is not None:
                    stopTestRun()  # pylint: disable=E1102

            # explicitly break reference cycles:
            # outcome.errors -> frame -> outcome -> outcome.errors
            # outcome.expectedFailure -> frame -> outcome -> outcome.expectedFailure
            outcome.errors.clear()
            outcome.expectedFailure = None

            # clear the outcome, no more needed
            self._outcome = None

class IntegrationTestCase(AsyncioTestCase):

    LEDGER = None
    MANAGER = None
    VERBOSITY = logging.WARN

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.conductor: Optional[Conductor] = None
        self.blockchain: Optional[BlockchainNode] = None
        self.wallet_node: Optional[WalletNode] = None
        self.manager: Optional[BaseWalletManager] = None
        self.ledger: Optional[BaseLedger] = None
        self.wallet: Optional[Wallet] = None
        self.account: Optional[BaseAccount] = None

    async def asyncSetUp(self):
        self.conductor = Conductor(
            ledger_module=self.LEDGER, manager_module=self.MANAGER, verbosity=self.VERBOSITY
        )
        await self.conductor.start()
        self.blockchain = self.conductor.blockchain_node
        self.wallet_node = self.conductor.wallet_node
        self.manager = self.wallet_node.manager
        self.ledger = self.wallet_node.ledger
        self.wallet = self.wallet_node.wallet
        self.account = self.wallet_node.wallet.default_account

    async def asyncTearDown(self):
        await self.conductor.stop()

    def broadcast(self, tx):
        return self.ledger.broadcast(tx)

    def get_balance(self, account=None, confirmations=0):
        if account is None:
            return self.manager.get_balance(confirmations=confirmations)
        else:
            return account.get_balance(confirmations=confirmations)

    async def on_header(self, height):
        if self.ledger.headers.height < height:
            await self.ledger.on_header.where(
                lambda e: e.height == height
            )
        return True

    def on_transaction_id(self, txid):
        return self.ledger.on_transaction.where(
            lambda e: e.tx.id == txid
        )

    def on_transaction_address(self, tx, address):
        return self.ledger.on_transaction.where(
            lambda e: e.tx.id == tx.id and e.address == address
        )
<commit_msg>moved import to make pylint happy<commit_after>import sys
import logging
import unittest
from typing import Optional
from unittest.case import _Outcome
from torba.orchstr8 import Conductor
from torba.orchstr8.node import BlockchainNode, WalletNode
from torba.client.baseledger import BaseLedger
from torba.client.baseaccount import BaseAccount
from torba.client.basemanager import BaseWalletManager
from torba.client.wallet import Wallet

try:
    import asyncio
    from asyncio.runners import _cancel_all_tasks  # type: ignore
except ImportError:
    import asyncio

    # this is only available in py3.7
    def _cancel_all_tasks(loop):
        pass

HANDLER = logging.StreamHandler(sys.stdout)
HANDLER.setFormatter(
    logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
)
logging.getLogger().addHandler(HANDLER)

class AsyncioTestCase(unittest.TestCase):
    # Implementation inspired by discussion:
    #  https://bugs.python.org/issue32972

    async def asyncSetUp(self):  # pylint: disable=C0103
        pass

    async def asyncTearDown(self):  # pylint: disable=C0103
        pass

    async def doAsyncCleanups(self):  # pylint: disable=C0103
        pass

    def run(self, result=None):  # pylint: disable=R0915
        orig_result = result
        if result is None:
            result = self.defaultTestResult()
            startTestRun = getattr(result, 'startTestRun', None)  # pylint: disable=C0103
            if startTestRun is not None:
                startTestRun()

        result.startTest(self)

        testMethod = getattr(self, self._testMethodName)  # pylint: disable=C0103
        if (getattr(self.__class__, "__unittest_skip__", False) or
                getattr(testMethod, "__unittest_skip__", False)):
            # If the class or method was skipped.
            try:
                skip_why = (getattr(self.__class__, '__unittest_skip_why__', '')
                            or getattr(testMethod, '__unittest_skip_why__', ''))
                self._addSkip(result, self, skip_why)
            finally:
                result.stopTest(self)
            return
        expecting_failure_method = getattr(testMethod,
                                           "__unittest_expecting_failure__", False)
        expecting_failure_class = getattr(self,
                                          "__unittest_expecting_failure__", False)
        expecting_failure = expecting_failure_class or expecting_failure_method
        outcome = _Outcome(result)
        try:
            self._outcome = outcome

            loop = asyncio.new_event_loop()
            try:
                asyncio.set_event_loop(loop)
                loop.set_debug(True)

                with outcome.testPartExecutor(self):
                    self.setUp()
                    loop.run_until_complete(self.asyncSetUp())
                if outcome.success:
                    outcome.expecting_failure = expecting_failure
                    with outcome.testPartExecutor(self, isTest=True):
                        possible_coroutine = testMethod()
                        if asyncio.iscoroutine(possible_coroutine):
                            loop.run_until_complete(possible_coroutine)
                    outcome.expecting_failure = False
                    with outcome.testPartExecutor(self):
                        loop.run_until_complete(self.asyncTearDown())
                        self.tearDown()
            finally:
                try:
                    _cancel_all_tasks(loop)
                    loop.run_until_complete(loop.shutdown_asyncgens())
                finally:
                    asyncio.set_event_loop(None)
                    loop.close()

            self.doCleanups()

            for test, reason in outcome.skipped:
                self._addSkip(result, test, reason)
            self._feedErrorsToResult(result, outcome.errors)
            if outcome.success:
                if expecting_failure:
                    if outcome.expectedFailure:
                        self._addExpectedFailure(result, outcome.expectedFailure)
                    else:
                        self._addUnexpectedSuccess(result)
                else:
                    result.addSuccess(self)
            return result
        finally:
            result.stopTest(self)
            if orig_result is None:
                stopTestRun = getattr(result, 'stopTestRun', None)  # pylint: disable=C0103
                if stopTestRun is not None:
                    stopTestRun()  # pylint: disable=E1102

            # explicitly break reference cycles:
            # outcome.errors -> frame -> outcome -> outcome.errors
            # outcome.expectedFailure -> frame -> outcome -> outcome.expectedFailure
            outcome.errors.clear()
            outcome.expectedFailure = None

            # clear the outcome, no more needed
            self._outcome = None

class IntegrationTestCase(AsyncioTestCase):

    LEDGER = None
    MANAGER = None
    VERBOSITY = logging.WARN

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.conductor: Optional[Conductor] = None
        self.blockchain: Optional[BlockchainNode] = None
        self.wallet_node: Optional[WalletNode] = None
        self.manager: Optional[BaseWalletManager] = None
        self.ledger: Optional[BaseLedger] = None
        self.wallet: Optional[Wallet] = None
        self.account: Optional[BaseAccount] = None

    async def asyncSetUp(self):
        self.conductor = Conductor(
            ledger_module=self.LEDGER, manager_module=self.MANAGER, verbosity=self.VERBOSITY
        )
        await self.conductor.start()
        self.blockchain = self.conductor.blockchain_node
        self.wallet_node = self.conductor.wallet_node
        self.manager = self.wallet_node.manager
        self.ledger = self.wallet_node.ledger
        self.wallet = self.wallet_node.wallet
        self.account = self.wallet_node.wallet.default_account

    async def asyncTearDown(self):
        await self.conductor.stop()

    def broadcast(self, tx):
        return self.ledger.broadcast(tx)

    def get_balance(self, account=None, confirmations=0):
        if account is None:
            return self.manager.get_balance(confirmations=confirmations)
        else:
            return account.get_balance(confirmations=confirmations)

    async def on_header(self, height):
        if self.ledger.headers.height < height:
            await self.ledger.on_header.where(
                lambda e: e.height == height
            )
        return True

    def on_transaction_id(self, txid):
        return self.ledger.on_transaction.where(
            lambda e: e.tx.id == txid
        )

    def on_transaction_address(self, tx, address):
        return self.ledger.on_transaction.where(
            lambda e: e.tx.id == tx.id and e.address == address
        )
<|endoftext|>


<commit_before>#-*- coding: utf-8 -*-

##########################################################
# © 2013/2014 Daniel 'grindhold' Brendle with torservers.net
#
# This file is part of torcollect
#
# torcollect is free software: you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public License
# as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later
# version.
#
# torcollect is distributed in the hope that it will be
# useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
# PURPOSE. See the GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with torcollect.
# If not, see http://www.gnu.org/licenses/.
###########################################################

import datetime
import torcollect.database
import StringIO
import json
import re
import pygal
from pygal.style import Style

from torcollect.paths import REPORTPAGE, REPORTS

main_page = """
<!DOCTYPE html>

    <title>Torservers.net :: Statistics</title>
    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
    <link rel="stylesheet" href="bootstrap/css/bootstrap.min.css" type="text/css">
    <link rel="stylesheet" href="bootstrap/css/bootstrap-theme.min.css" type="text/css">
    <link rel="stylesheet" href="torcollect.css" type="text/css">

Torservers.net Statistics

            <svg width="1024" height="100" class="tc_graph" id="tc_graph">
                <defs>
                    <linearGradient id="grad1" x1="0%%" y1="0%%" x2="0%%" y2="100%%">
                    <stop offset="0%%" style="stop-color:rgb(160,255,0);stop-opacity:1" />
                    <stop offset="100%%" style="stop-color:rgb(128,255,0);stop-opacity:0.25" />
                    </linearGradient>
                </defs>
            </svg>

            %(usage_graph)s

Please select a day to see detailed statistics

"""

report_header = """

Statistics of Bridge usage at %(date)s

"""

country_line = """

     %(name)s
     %(users)d

    %(sparkline)s

"""

country_table = """

Country Statistics

%(worldmap)s

%(content)s

"""

transport_line = """

     %(transport)s
     %(users)d

"""

transport_table = """

Pluggable Transports Statistics

%(content)s

"""

bridge_line = """

     Bridge
     %(users)d

    %(sparkline)s

"""

bridge_table = """

Users by Bridge

%(content)s

"""

RE_XML_COMMENT = re.compile(r"")

class TorcollectStyle(Style):
    def __init__(self):
        Style.__init__(self)
        self.background = 'transparent'
        self.plot_background = 'transparent'
        self.colors = ('#aaff00', '#550055')
        self.foreground_dark = '#010101'

def clean_graph(xml):
    xml = re.sub(RE_XML_COMMENT, "", xml)
    return xml.replace ("<?xml version='1.0' encoding='utf-8'?>","",1)

def escape(plain):
    html = plain.replace("<", "<")
    return html.replace(">", ">")

def generate_main_graph(data):
    chart = pygal.Line(width=900, height=100, fill=True, spacing=20,
                       margin=10, style=TorcollectStyle())
    chart.add('Users', data)
    return clean_graph(chart.render())

def generate_main_page():
    graphdata = []
    db = torcollect.database.Database()
    stmnt = "SELECT SUM(CRP_USERS), REP_DATE \
            FROM Report INNER JOIN CountryReport \
                ON (REP_ID = CRP_REP_ID) \
            GROUP BY (REP_DATE) \
            ORDER BY REP_DATE ASC LIMIT 365;"
    cur = db.cursor()
    cur.execute(stmnt)
    count = 0
    pygal_graph = []
    for dataset in cur.fetchall():
        graphdata.append({'d': dataset[1], 'u': dataset[0]})
        pygal_graph.append(dataset[0])
    page = main_page % {'graphdata': json.dumps(graphdata),
                        'usage_graph': generate_main_graph(pygal_graph)}
    mainpage = open(REPORTPAGE, "w")
    mainpage.write(page)
    mainpage.close()

def generate_worldmap(data):
    wm = pygal.Worldmap(width=300, height=200, show_legend=False, margin=0, style=TorcollectStyle())
    wm.add('Tor Usage', data)
    return clean_graph(wm.render())

def generate_country_sparkline(data):
    chart = pygal.StackedLine(fill=True, show_x_labels=False,
                              show_y_labels=False, margin=0, style=TorcollectStyle())
    chart.add('', data)
    return clean_graph(chart.render_sparkline(interpolate="cubic"))

def generate_countryreport(date):
    db = torcollect.database.Database()
    stmnt = "SELECT CCO_SHORT, CCO_LONG, SUM(CRP_USERS) AS USAGE\
            FROM CountryReport INNER JOIN CountryCode \
                ON (CCO_ID = CRP_CCO_ID) \
            INNER JOIN Report \
                ON (REP_ID = CRP_REP_ID) \
            WHERE REP_DATE = %(date)s \
            GROUP BY CCO_SHORT, CCO_LONG \
            ORDER BY USAGE DESC;"
    stmnt_history = "SELECT CCO_SHORT, SUM(CRP_USERS) AS USAGE\
            FROM CountryReport INNER JOIN CountryCode \
                ON (CCO_ID = CRP_CCO_ID) \
            INNER JOIN Report \
                ON (REP_ID = CRP_REP_ID) \
            WHERE REP_DATE > DATE %(date)s-7 \
            GROUP BY CCO_SHORT, CCO_LONG, REP_DATE \
            ORDER BY CCO_SHORT, REP_DATE ASC;"
    cur = db.cursor()

    cur.execute(stmnt_history, {'date': date.isoformat()})
    country_history = {}
    for dataset in cur.fetchall():
        ccode = dataset[0].lower()
        if not country_history.has_key(ccode):
            country_history[ccode] = []
        country_history[ccode].append(dataset[1])

    cur.execute(stmnt, {'date': date.isoformat()})
    country_lines = StringIO.StringIO()

    worldmap_data = {}

    for dataset in cur.fetchall():
        ccode = dataset[0].lower()
        line = country_line % {'code': ccode,
                               'name': dataset[1],
                               'users': dataset[2],
                               'sparkline': generate_country_sparkline(country_history[ccode])}
        worldmap_data[dataset[0].lower()] = dataset[2]
        country_lines.write(line)

    return country_table % {'date': date.isoformat(),
                            'content': country_lines.getvalue(),
                            'worldmap': generate_worldmap(worldmap_data)}

def generate_transportreport(date):
    db = torcollect.database.Database()
    stmnt = "SELECT TRA_NAME, SUM(TRP_USERS) AS USAGE\
             FROM TransportReport INNER JOIN Transport \
                 ON (TRA_ID = TRP_TRA_ID) \
             INNER JOIN Report \
                 ON (REP_ID = TRP_REP_ID) \
             WHERE REP_DATE = %(date)s \
             GROUP BY TRA_NAME \
             ORDER BY USAGE DESC;"
    cur = db.cursor()
    cur.execute(stmnt, {'date': date.isoformat()})
    transport_lines = StringIO.StringIO()
    for dataset in cur.fetchall():
        line = transport_line % {'transport': escape(dataset[0]),
                                 'users': dataset[1]}
        transport_lines.write(line)
    return transport_table % {'date': date.isoformat(),
                              'content': transport_lines.getvalue()}

def generate_bridge_sparkline(data):
    style = Style()
    style.background = 'transparent'
    style.plot_background = 'transparent'
    chart = pygal.StackedLine(fill=True, show_x_labels=False, show_y_labels=False,
                               margin=0, style=TorcollectStyle())
    chart.add('', data['sent'])
    chart.add('', data['received'])
    return clean_graph(chart.render_sparkline(interpolate="cubic"))

def generate_bridgereport(date):
    db = torcollect.database.Database()
    stmnt_usage = "SELECT REP_BRG_ID, SUM(CRP_USERS) AS USAGE \
             FROM CountryReport LEFT JOIN Report \
                ON (REP_ID = CRP_REP_ID) \
             WHERE REP_DATE = %(date)s \
             GROUP BY REP_BRG_ID \
             ORDER BY USAGE DESC;"
    stmnt_traffic = "SELECT REP_BRG_ID, REP_TRAFFIC_SENT, \
                        REP_TRAFFIC_RECEIVED \
                     FROM Report \
                     WHERE REP_DATE > DATE %(date)s-7 \
                     ORDER BY REP_BRG_ID, REP_DATE ASC;"
    cur = db.cursor()

    traffic_data = {}
    cur.execute(stmnt_traffic, {'date': date.isoformat()})
    for dataset in cur.fetchall():
        if not traffic_data.has_key(dataset[0]):
            traffic_data[dataset[0]] = {'sent':[],'received':[]}
        traffic_data[dataset[0]]['sent'].append(dataset[1])
        traffic_data[dataset[0]]['received'].append(dataset[2])

    cur.execute(stmnt_usage, {'date': date.isoformat()})
    bridge_lines = StringIO.StringIO()
    for dataset in cur.fetchall():
        line = bridge_line % {'users': dataset[1],
                              'sparkline': generate_bridge_sparkline(traffic_data[dataset[0]])}
        bridge_lines.write(line)

    return bridge_table % {'date': date.isoformat(),
                           'content': bridge_lines.getvalue()}

def generate_report_for_day(date):
    content = report_header%{'date': date.isoformat()}
    country_html = generate_countryreport(date)
    bridge_html = generate_bridgereport(date)

    content += country_html
    content += bridge_html
    content += generate_transportreport(date)

    reportfile = open("%s%s%s" % (REPORTS, date.isoformat(), ".html"), "w")
    reportfile.write(content)
    reportfile.close()

<commit_msg>re-added trafficvalues as text<commit_after>#-*- coding: utf-8 -*-

##########################################################
# © 2013/2014 Daniel 'grindhold' Brendle with torservers.net
#
# This file is part of torcollect
#
# torcollect is free software: you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public License
# as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later
# version.
#
# torcollect is distributed in the hope that it will be
# useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
# PURPOSE. See the GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with torcollect.
# If not, see http://www.gnu.org/licenses/.
###########################################################

import datetime
import torcollect.database
import StringIO
import json
import re
import pygal
from pygal.style import Style

from torcollect.paths import REPORTPAGE, REPORTS

main_page = """
<!DOCTYPE html>

    <title>Torservers.net :: Statistics</title>
    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
    <link rel="stylesheet" href="bootstrap/css/bootstrap.min.css" type="text/css">
    <link rel="stylesheet" href="bootstrap/css/bootstrap-theme.min.css" type="text/css">
    <link rel="stylesheet" href="torcollect.css" type="text/css">

Torservers.net Statistics

            <svg width="1024" height="100" class="tc_graph" id="tc_graph">
                <defs>
                    <linearGradient id="grad1" x1="0%%" y1="0%%" x2="0%%" y2="100%%">
                    <stop offset="0%%" style="stop-color:rgb(160,255,0);stop-opacity:1" />
                    <stop offset="100%%" style="stop-color:rgb(128,255,0);stop-opacity:0.25" />
                    </linearGradient>
                </defs>
            </svg>

            %(usage_graph)s

Please select a day to see detailed statistics

"""

report_header = """

Statistics of Bridge usage at %(date)s

"""

country_line = """

     %(name)s
     %(users)d

    %(sparkline)s

"""

country_table = """

Country Statistics

%(worldmap)s

%(content)s

"""

transport_line = """

     %(transport)s
     %(users)d

"""

transport_table = """

Pluggable Transports Statistics

%(content)s

"""

bridge_line = """

     Bridge
     %(users)d

    %(sparkline)s

        recv: %(received)d
        sent: %(sent)d

"""

bridge_table = """

Users and traffic by Bridge

%(content)s

"""

RE_XML_COMMENT = re.compile(r"")

class TorcollectStyle(Style):
    def __init__(self):
        Style.__init__(self)
        self.background = 'transparent'
        self.plot_background = 'transparent'
        self.colors = ('#aaff00', '#550055')
        self.foreground_dark = '#010101'

def clean_graph(xml):
    xml = re.sub(RE_XML_COMMENT, "", xml)
    return xml.replace ("<?xml version='1.0' encoding='utf-8'?>","",1)

def escape(plain):
    html = plain.replace("<", "<")
    return html.replace(">", ">")

def generate_main_graph(data):
    chart = pygal.Line(width=900, height=100, fill=True, spacing=20,
                       margin=10, style=TorcollectStyle())
    chart.add('Users', data)
    return clean_graph(chart.render())

def generate_main_page():
    graphdata = []
    db = torcollect.database.Database()
    stmnt = "SELECT SUM(CRP_USERS), REP_DATE \
            FROM Report INNER JOIN CountryReport \
                ON (REP_ID = CRP_REP_ID) \
            GROUP BY (REP_DATE) \
            ORDER BY REP_DATE ASC LIMIT 365;"
    cur = db.cursor()
    cur.execute(stmnt)
    count = 0
    pygal_graph = []
    for dataset in cur.fetchall():
        graphdata.append({'d': dataset[1], 'u': dataset[0]})
        pygal_graph.append(dataset[0])
    page = main_page % {'graphdata': json.dumps(graphdata),
                        'usage_graph': generate_main_graph(pygal_graph)}
    mainpage = open(REPORTPAGE, "w")
    mainpage.write(page)
    mainpage.close()

def generate_worldmap(data):
    wm = pygal.Worldmap(width=300, height=200, show_legend=False, margin=0, style=TorcollectStyle())
    wm.add('Tor Usage', data)
    return clean_graph(wm.render())

def generate_country_sparkline(data):
    chart = pygal.StackedLine(fill=True, show_x_labels=False,
                              show_y_labels=False, margin=0, style=TorcollectStyle())
    chart.add('', data)
    return clean_graph(chart.render_sparkline(interpolate="cubic"))

def generate_countryreport(date):
    db = torcollect.database.Database()
    stmnt = "SELECT CCO_SHORT, CCO_LONG, SUM(CRP_USERS) AS USAGE\
            FROM CountryReport INNER JOIN CountryCode \
                ON (CCO_ID = CRP_CCO_ID) \
            INNER JOIN Report \
                ON (REP_ID = CRP_REP_ID) \
            WHERE REP_DATE = %(date)s \
            GROUP BY CCO_SHORT, CCO_LONG \
            ORDER BY USAGE DESC;"
    stmnt_history = "SELECT CCO_SHORT, SUM(CRP_USERS) AS USAGE\
            FROM CountryReport INNER JOIN CountryCode \
                ON (CCO_ID = CRP_CCO_ID) \
            INNER JOIN Report \
                ON (REP_ID = CRP_REP_ID) \
            WHERE REP_DATE > DATE %(date)s-7 \
            GROUP BY CCO_SHORT, CCO_LONG, REP_DATE \
            ORDER BY CCO_SHORT, REP_DATE ASC;"
    cur = db.cursor()

    cur.execute(stmnt_history, {'date': date.isoformat()})
    country_history = {}
    for dataset in cur.fetchall():
        ccode = dataset[0].lower()
        if not country_history.has_key(ccode):
            country_history[ccode] = []
        country_history[ccode].append(dataset[1])

    cur.execute(stmnt, {'date': date.isoformat()})
    country_lines = StringIO.StringIO()

    worldmap_data = {}

    for dataset in cur.fetchall():
        ccode = dataset[0].lower()
        line = country_line % {'code': ccode,
                               'name': dataset[1],
                               'users': dataset[2],
                               'sparkline': generate_country_sparkline(country_history[ccode])}
        worldmap_data[dataset[0].lower()] = dataset[2]
        country_lines.write(line)

    return country_table % {'date': date.isoformat(),
                            'content': country_lines.getvalue(),
                            'worldmap': generate_worldmap(worldmap_data)}

def generate_transportreport(date):
    db = torcollect.database.Database()
    stmnt = "SELECT TRA_NAME, SUM(TRP_USERS) AS USAGE\
             FROM TransportReport INNER JOIN Transport \
                 ON (TRA_ID = TRP_TRA_ID) \
             INNER JOIN Report \
                 ON (REP_ID = TRP_REP_ID) \
             WHERE REP_DATE = %(date)s \
             GROUP BY TRA_NAME \
             ORDER BY USAGE DESC;"
    cur = db.cursor()
    cur.execute(stmnt, {'date': date.isoformat()})
    transport_lines = StringIO.StringIO()
    for dataset in cur.fetchall():
        line = transport_line % {'transport': escape(dataset[0]),
                                 'users': dataset[1]}
        transport_lines.write(line)
    return transport_table % {'date': date.isoformat(),
                              'content': transport_lines.getvalue()}

def generate_bridge_sparkline(data):
    style = Style()
    style.background = 'transparent'
    style.plot_background = 'transparent'
    chart = pygal.StackedLine(fill=True, show_x_labels=False, show_y_labels=False,
                               margin=0, style=TorcollectStyle())
    chart.add('', data['sent'])
    chart.add('', data['received'])
    return clean_graph(chart.render_sparkline(interpolate="cubic"))

def generate_bridgereport(date):
    db = torcollect.database.Database()
    stmnt_usage = "SELECT REP_BRG_ID, SUM(CRP_USERS) AS USAGE, \
                          REP_TRAFFIC_SENT, REP_TRAFFIC_RECEIVED \
             FROM CountryReport LEFT JOIN Report \
                ON (REP_ID = CRP_REP_ID) \
             WHERE REP_DATE = %(date)s \
             GROUP BY REP_BRG_ID, REP_TRAFFIC_SENT, REP_TRAFFIC_RECEIVED \
             ORDER BY USAGE DESC;"
    stmnt_traffic = "SELECT REP_BRG_ID, REP_TRAFFIC_SENT, \
                        REP_TRAFFIC_RECEIVED \
                     FROM Report \
                     WHERE REP_DATE > DATE %(date)s-7 \
                     ORDER BY REP_BRG_ID, REP_DATE ASC;"
    cur = db.cursor()

    traffic_data = {}
    cur.execute(stmnt_traffic, {'date': date.isoformat()})
    for dataset in cur.fetchall():
        if not traffic_data.has_key(dataset[0]):
            traffic_data[dataset[0]] = {'sent':[],'received':[]}
        traffic_data[dataset[0]]['sent'].append(dataset[1])
        traffic_data[dataset[0]]['received'].append(dataset[2])

    cur.execute(stmnt_usage, {'date': date.isoformat()})
    bridge_lines = StringIO.StringIO()
    for dataset in cur.fetchall():
        line = bridge_line % {'users': dataset[1],
                              'sent': dataset[2],
                              'received': dataset[3],
                              'sparkline': generate_bridge_sparkline(traffic_data[dataset[0]])}
        bridge_lines.write(line)

    return bridge_table % {'date': date.isoformat(),
                           'content': bridge_lines.getvalue()}

def generate_report_for_day(date):
    content = report_header%{'date': date.isoformat()}
    country_html = generate_countryreport(date)
    bridge_html = generate_bridgereport(date)

    content += country_html
    content += bridge_html
    content += generate_transportreport(date)

    reportfile = open("%s%s%s" % (REPORTS, date.isoformat(), ".html"), "w")
    reportfile.write(content)
    reportfile.close()

<|endoftext|>


<commit_before>import random
import sys

import json
import motor
import tornado.ioloop
import tornado.web
from tornado import gen
import tornado.options
from tornado.options import options
import tornado.httpserver

PY3 = False
if sys.version_info[0] == 3:
    PY3 = True
    xrange = range

tornado.options.define('port', default=8888, type=int, help="Server port")
tornado.options.define('mongo', default='localhost', type=str, help="MongoDB host")

class BaseHandler(tornado.web.RequestHandler):
    def compute_etag(self):
        return None

class JsonSerializeTestHandler(BaseHandler):
    def get(self):
        obj = dict(message="Hello, World!")
        self.write(obj)

class PlaintextHandler(BaseHandler):
    def get(self):
        self.set_header('Content-Type', 'text/plain')
        self.write(b"Hello, World!")

class QueryTestHandler(BaseHandler):
    @tornado.web.asynchronous
    @gen.coroutine
    def get(self):
        queries = int(self.get_argument("queries", 0))

        if queries == 0:
            random_id = random.randint(1, 10000)
            world = yield motor.Op(db.World.find_one, random_id)
            # Get first postion on arguments, and so first postion in mongo return
            world['id'] = world.pop('_id')
            response = json.dumps(world)
        else:
            worlds = []
            for i in xrange(int(queries)):
                random_id = random.randint(1, 10000)
                world = yield motor.Op(db.World.find_one, random_id)
                # Get first postion on arguments, and so first postion in mongo return
                world['id'] = world.pop('_id')
                worlds.append(world)
            response = json.dumps(worlds)
        self.set_header("Content-Type", "application/json; charset=UTF-8")
        self.finish(response)

application = tornado.web.Application([
    (r"/json", JsonSerializeTestHandler),
    (r"/plaintext", PlaintextHandler),
    (r"/db", QueryTestHandler),
])

if __name__ == "__main__":
    tornado.options.parse_command_line()
    server = tornado.httpserver.HTTPServer(application)
    server.bind(options.port)
    server.start(0)
    db = motor.MotorClient(options.mongo).open_sync().hello_world
    tornado.ioloop.IOLoop.instance().start()
<commit_msg>remove unnecessary decorator web.asynchronous, call self.write instead.<commit_after>import random
import sys

import json
import motor
import tornado.ioloop
import tornado.web
from tornado import gen
import tornado.options
from tornado.options import options
import tornado.httpserver

PY3 = False
if sys.version_info[0] == 3:
    PY3 = True
    xrange = range

tornado.options.define('port', default=8888, type=int, help="Server port")
tornado.options.define('mongo', default='localhost', type=str, help="MongoDB host")

class BaseHandler(tornado.web.RequestHandler):
    def compute_etag(self):
        return None

class JsonSerializeTestHandler(BaseHandler):
    def get(self):
        obj = dict(message="Hello, World!")
        self.write(obj)

class PlaintextHandler(BaseHandler):
    def get(self):
        self.set_header('Content-Type', 'text/plain')
        self.write(b"Hello, World!")

class QueryTestHandler(BaseHandler):
    @gen.coroutine
    def get(self):
        queries = int(self.get_argument("queries", 0))

        if queries == 0:
            random_id = random.randint(1, 10000)
            world = yield motor.Op(db.World.find_one, random_id)
            # Get first postion on arguments, and so first postion in mongo return
            world['id'] = world.pop('_id')
            response = json.dumps(world)
        else:
            worlds = []
            for i in xrange(int(queries)):
                random_id = random.randint(1, 10000)
                world = yield motor.Op(db.World.find_one, random_id)
                # Get first postion on arguments, and so first postion in mongo return
                world['id'] = world.pop('_id')
                worlds.append(world)
            response = json.dumps(worlds)
        self.set_header("Content-Type", "application/json; charset=UTF-8")
        self.write(response)

application = tornado.web.Application([
    (r"/json", JsonSerializeTestHandler),
    (r"/plaintext", PlaintextHandler),
    (r"/db", QueryTestHandler),
])

if __name__ == "__main__":
    tornado.options.parse_command_line()
    server = tornado.httpserver.HTTPServer(application)
    server.bind(options.port)
    server.start(0)
    db = motor.MotorClient(options.mongo).open_sync().hello_world
    tornado.ioloop.IOLoop.instance().start()
<|endoftext|>


<commit_before>import tables as pt, numpy as np
from models.PositionModel import PositionModel
'''
Based on the model:
PositionModel:
    timestamp = pt.Time64Col()
    symbol = pt.StringCol(4)
    shares = pt.Int32Col()
    purchase_price = pt.Int32Col()
    closed = pt.Int32Col()
'''

class Position:
    def __init__(self, isTable = True):
        self.isTable = isTable
        self.positionFile = pt.openFile('PositionModel.h5', mode = "w")
        self.position = self.positionFile.createTable('/', 'position', PositionModel)
        if isTable == False:
            self.positionArray = np.array([])

    def getPositions(self, isTable = True):
        output = []
        for row in self.position.iterrows():
            output.append(self.cloneRow(row))
        return output

    def addPosition(self,timestamp,symbol,shares,purchase_price,isTable = True):
        '''
        Adds a position to the current list of positions
        '''
        if isTable:
            row = self.position.row
            row['timestamp'] = timestamp
            row['symbol'] = symbol
            row['shares'] = shares
            row['purchase_price'] = purchase_price
            row.append()
            self.position.flush()
        else:
            self.addPositionArray(timestamp,symbol,shares,purchase_price)

    def removePosition(self, symbol, shares, closeType, isTable = True):
        '''
        symbol: the representation of the shares to be removed
        shares: the number of shares to remove
        closeType: removal order "lifo" or "fifo"
        Removes/modifies positions until the total number of shares have been removed
        NOTE: Method assumes that verification of valid sell has already been completed
        '''
        if isTable:
            rowIndexes = []
            rows = []
            debug = False
            if debug:
                print 'REMOVING POSITIONS'
                print 'REMOVE:',symbol,shares,closeType
                for row in self.position.iterrows():
                    print 'CURRROWS:', row
            for row in self.position.where("symbol=='%s'"%symbol):
                if debug:
                    print 'SELEROWS:', row
                rows.append(self.cloneRow(row))
                rowIndexes.append(row.nrow)
            if(closeType=='fifo'):
                i = 0
                row = rows[i]
                if debug:
                    print 'FIFO', row
                posShares = row['shares']
                while(shares>posShares):
                    shares-=posShares
                    i+=1
                    row = rows[i]
                    posShares = row['shares']
                cnt=0
                while cnt<i:
                    for row in self.position.iterrows(rowIndexes[cnt]-cnt,rowIndexes[cnt]-cnt+1):
                        row['closed'] = 1
                        row.update()
                    self.position.flush()
                    cnt+=1
                    if debug:
                        print 'ROWCLOSED', row
                cnt=0
                for newRow in self.position.where('(symbol=="%s") & (timestamp==%i)'%(symbol,row['timestamp'])):
                    if(cnt==0):
                        newShares = posShares-shares
                        newRow['shares'] = newShares
                        newRow.update()
                        if debug:
                            print 'UPDATEDROW(FIFO):', newRow
                    cnt+=1
                self.position.flush()

            elif(closeType=='lifo'):
                i = len(rows)-1
                row = rows[i]
                if debug:
                    print "LIFO",row
                posShares = row['shares']
                while(shares>posShares):
                    shares-=posShares
                    i-=1
                    row = rows[i]
                    posShares = row['shares']
                cnt=0
                i+=1
                while i<len(rows):
                    for row in self.position.removeRows(rowIndexes[i]-cnt,rowIndexes[i]-cnt+1):
                        row['closed'] = 1
                        row.update()
                    self.position.flush()
                    i+=1
                    cnt+=1
                    if debug:
                        print 'ROWREMOVED', row
                cnt=0
                for newRow in self.position.where('(symbol=="%s") & (timestamp==%i)'%(symbol,row['timestamp'])):
                    if(cnt==0):
                        newShares = posShares-shares
                        newRow['shares'] = newShares
                        newRow.update()
                        if debug:
                            print 'UPDATEDROW(LIFO):', newRow
                    cnt+=1
                self.position.flush()
            else:
                #invalid type
                raise TypeError("Not an existing close type '%s'." % str(newOrder.order_type))
        else:
            self.removePositionArray(symbol, shares, closeType)

    def cloneRow(self,row):
        '''
        Makes a copy of the row so that the correct information will be appended to the list
        '''
        dct = {}
        dct['symbol'] = row['symbol']
        dct['shares'] = row['shares']
        dct['timestamp'] = row['timestamp']
        dct['purchase_price'] = row['purchase_price']
        return dct

    def addPositionArray(self,timestamp,symbol,shares,purchase_price):
        row = {}
        row['timestamp'] = timestamp
        row['symbol'] = symbol
        row['shares'] = shares
        row['purchase_price'] = purchase_price
        np.append(self.positionArray, row)

    def removePositionArray(self, symbol, shares, closeType):
        '''
        symbol: the representation of the shares to be removed
        shares: the number of shares to remove
        closeType: removal order "lifo" or "fifo"
        Removes/modifies positions until the total number of shares have been removed
        NOTE: Method assumes that verification of valid sell has already been completed
        '''
        rowIndexes = []
        rows = []
        debug = False
        if debug:
            print 'REMOVING POSITIONS'
            print 'REMOVE:',symbol,shares,closeType
            for row in self.positionArray:
                print 'CURRROWS:', row
        rows = self.positionArray
        print 'RPA rows', rows
        if(closeType=='fifo'):
            i = 0
            row = rows[i]
            if debug:
                print 'FIFO', row
            posShares = row['shares']
            while(shares>posShares):
                shares-=posShares
                i+=1
                row = rows[i]
                posShares = row['shares']
            cnt=0
            while cnt<i:
                rows[cnt]['closed']=1
                cnt+=1
                if debug:
                    print 'ROWCLOSED', rows[cnt]
            newRow = rows[i]
            newShares = posShares-shares
            newRow['shares'] = newShares
            if debug:
                print 'UPDATEDROW(FIFO):', newRow

        elif(closeType=='lifo'):
            i = len(rows)-1
            row = rows[i]
            if debug:
                print "LIFO",row
            posShares = row['shares']
            while(shares>posShares):
                shares-=posShares
                i-=1
                row = rows[i]
                posShares = row['shares']
            cnt=i+1
            while cnt<len(rows):
                rows[cnt]['closed']=1
                cnt+=1
                if debug:
                    print 'ROWREMOVED', row
            newRow = rows[i]
            newShares = posShares-shares
            newRow['shares'] = newShares
            if debug:
                print 'UPDATEDROW(FIFO):', newRow
        else:
            #invalid type
            raise TypeError("Not an existing close type '%s'." % str(newOrder.order_type))

    def fillTable(self):
        for arrRow in self.positionArray:
            row = self.position.row
            row['timestamp'] = arrRow['timestamp']
            row['symbol'] = arrRow['symbol']
            row['shares'] = arrRow['shares']
            row['purchase_price'] = arrRow['purchase_price']
            row.append()
        self.position.flush()

    def close(self, isTable = True):
        if isTable:
            self.positionFile.close()
        else:
            self.fillTable()
            self.positionFile.close()
<commit_msg>Fixed error in error handling in Position: removePosition - had newOrder.order_type when it should have been closeType<commit_after>import tables as pt, numpy as np
from models.PositionModel import PositionModel
'''
Based on the model:
PositionModel:
    timestamp = pt.Time64Col()
    symbol = pt.StringCol(4)
    shares = pt.Int32Col()
    purchase_price = pt.Int32Col()
    closed = pt.Int32Col()
'''

class Position:
    def __init__(self, isTable = True):
        self.isTable = isTable
        self.positionFile = pt.openFile('PositionModel.h5', mode = "w")
        self.position = self.positionFile.createTable('/', 'position', PositionModel)
        if isTable == False:
            self.positionArray = np.array([])

    def getPositions(self, isTable = True):
        output = []
        for row in self.position.iterrows():
            output.append(self.cloneRow(row))
        return output

    def addPosition(self,timestamp,symbol,shares,purchase_price,isTable = True):
        '''
        Adds a position to the current list of positions
        '''
        if isTable:
            row = self.position.row
            row['timestamp'] = timestamp
            row['symbol'] = symbol
            row['shares'] = shares
            row['purchase_price'] = purchase_price
            row.append()
            self.position.flush()
        else:
            self.addPositionArray(timestamp,symbol,shares,purchase_price)

    def removePosition(self, symbol, shares, closeType, isTable = True):
        '''
        symbol: the representation of the shares to be removed
        shares: the number of shares to remove
        closeType: removal order "lifo" or "fifo"
        Removes/modifies positions until the total number of shares have been removed
        NOTE: Method assumes that verification of valid sell has already been completed
        '''
        if isTable:
            rowIndexes = []
            rows = []
            debug = False
            if debug:
                print 'REMOVING POSITIONS'
                print 'REMOVE:',symbol,shares,closeType
                for row in self.position.iterrows():
                    print 'CURRROWS:', row
            for row in self.position.where("symbol=='%s'"%symbol):
                if debug:
                    print 'SELEROWS:', row
                rows.append(self.cloneRow(row))
                rowIndexes.append(row.nrow)
            if(closeType=='fifo'):
                i = 0
                row = rows[i]
                if debug:
                    print 'FIFO', row
                posShares = row['shares']
                while(shares>posShares):
                    shares-=posShares
                    i+=1
                    row = rows[i]
                    posShares = row['shares']
                cnt=0
                while cnt<i:
                    for row in self.position.iterrows(rowIndexes[cnt]-cnt,rowIndexes[cnt]-cnt+1):
                        row['closed'] = 1
                        row.update()
                    self.position.flush()
                    cnt+=1
                    if debug:
                        print 'ROWCLOSED', row
                cnt=0
                for newRow in self.position.where('(symbol=="%s") & (timestamp==%i)'%(symbol,row['timestamp'])):
                    if(cnt==0):
                        newShares = posShares-shares
                        newRow['shares'] = newShares
                        newRow.update()
                        if debug:
                            print 'UPDATEDROW(FIFO):', newRow
                    cnt+=1
                self.position.flush()

            elif(closeType=='lifo'):
                i = len(rows)-1
                row = rows[i]
                if debug:
                    print "LIFO",row
                posShares = row['shares']
                while(shares>posShares):
                    shares-=posShares
                    i-=1
                    row = rows[i]
                    posShares = row['shares']
                cnt=0
                i+=1
                while i<len(rows):
                    for row in self.position.removeRows(rowIndexes[i]-cnt,rowIndexes[i]-cnt+1):
                        row['closed'] = 1
                        row.update()
                    self.position.flush()
                    i+=1
                    cnt+=1
                    if debug:
                        print 'ROWREMOVED', row
                cnt=0
                for newRow in self.position.where('(symbol=="%s") & (timestamp==%i)'%(symbol,row['timestamp'])):
                    if(cnt==0):
                        newShares = posShares-shares
                        newRow['shares'] = newShares
                        newRow.update()
                        if debug:
                            print 'UPDATEDROW(LIFO):', newRow
                    cnt+=1
                self.position.flush()
            else:
                #invalid type
                raise TypeError("Not an existing close type '%s'." % str(closeType))
        else:
            self.removePositionArray(symbol, shares, closeType)

    def cloneRow(self,row):
        '''
        Makes a copy of the row so that the correct information will be appended to the list
        '''
        dct = {}
        dct['symbol'] = row['symbol']
        dct['shares'] = row['shares']
        dct['timestamp'] = row['timestamp']
        dct['purchase_price'] = row['purchase_price']
        return dct

    def addPositionArray(self,timestamp,symbol,shares,purchase_price):
        row = {}
        row['timestamp'] = timestamp
        row['symbol'] = symbol
        row['shares'] = shares
        row['purchase_price'] = purchase_price
        np.append(self.positionArray, row)

    def removePositionArray(self, symbol, shares, closeType):
        '''
        symbol: the representation of the shares to be removed
        shares: the number of shares to remove
        closeType: removal order "lifo" or "fifo"
        Removes/modifies positions until the total number of shares have been removed
        NOTE: Method assumes that verification of valid sell has already been completed
        '''
        rowIndexes = []
        rows = []
        debug = False
        if debug:
            print 'REMOVING POSITIONS'
            print 'REMOVE:',symbol,shares,closeType
            for row in self.positionArray:
                print 'CURRROWS:', row
        rows = self.positionArray
        print 'RPA rows', rows
        if(closeType=='fifo'):
            i = 0
            row = rows[i]
            if debug:
                print 'FIFO', row
            posShares = row['shares']
            while(shares>posShares):
                shares-=posShares
                i+=1
                row = rows[i]
                posShares = row['shares']
            cnt=0
            while cnt<i:
                rows[cnt]['closed']=1
                cnt+=1
                if debug:
                    print 'ROWCLOSED', rows[cnt]
            newRow = rows[i]
            newShares = posShares-shares
            newRow['shares'] = newShares
            if debug:
                print 'UPDATEDROW(FIFO):', newRow

        elif(closeType=='lifo'):
            i = len(rows)-1
            row = rows[i]
            if debug:
                print "LIFO",row
            posShares = row['shares']
            while(shares>posShares):
                shares-=posShares
                i-=1
                row = rows[i]
                posShares = row['shares']
            cnt=i+1
            while cnt<len(rows):
                rows[cnt]['closed']=1
                cnt+=1
                if debug:
                    print 'ROWREMOVED', row
            newRow = rows[i]
            newShares = posShares-shares
            newRow['shares'] = newShares
            if debug:
                print 'UPDATEDROW(FIFO):', newRow
        else:
            #invalid type
            raise TypeError("Not an existing close type '%s'." % str(newOrder.order_type))

    def fillTable(self):
        for arrRow in self.positionArray:
            row = self.position.row
            row['timestamp'] = arrRow['timestamp']
            row['symbol'] = arrRow['symbol']
            row['shares'] = arrRow['shares']
            row['purchase_price'] = arrRow['purchase_price']
            row.append()
        self.position.flush()

    def close(self, isTable = True):
        if isTable:
            self.positionFile.close()
        else:
            self.fillTable()
            self.positionFile.close()
<|endoftext|>