<commit_before>#!/usr/bin/env python3

# This is a python script that emulates a terminal session and runs
# commands from a supplied markdown file..

import optparse
import os
import sys
import time

from cli import Ui
from web import WebUi
import config
from demo import Demo
from environment import Environment

def get_bash_script(script_dir, is_simulation = True, is_automated=False, is_testing=False):
    """
    Reads a README.md file in the indicated directoy and builds an
    executable bash script from the commands contained within.
    """
    if not script_dir.endswith('/'):
        script_dir = script_dir + "/"

    script = ""
    env = Environment(script_dir, False).get()
    for key, value in env.items():
        script += key + "='" + value + "'\n"

    filename = env.get_script_file_name(script_dir)
    in_code_block = False
    in_results_section = False
    lines = list(open(script_dir + filename))
    for line in lines:
        if line.startswith("Results:"):
            # Entering results section
            in_results_section = True
        elif line.startswith("```") and not in_code_block:
            # Entering a code block, if in_results_section = True then it's a results block
            in_code_block = True
        elif line.startswith("```") and in_code_block:
            # Finishing code block
            in_results_section = False
            in_code_block = False
        elif in_code_block and not in_results_section:
            # Executable line
            script += line
        elif line.startswith("#") and not in_code_block and not in_results_section and not is_automated:
            # Heading in descriptive text
            script += "\n"
    return script

def main():
    """SimDem CLI interpreter"""

    commands = config.modes
    command_string = ""
    for command in commands:
        command_string = command_string + command + "|"
    command_string = command_string[0:len(command_string)-1]

    p = optparse.OptionParser("%prog [" + command_string + "] <options> DEMO_NAME", version=config.SIMDEM_VERSION)
    p.add_option('--style', '-s', default="tutorial",
                 help="The style of simulation you want to run. 'tutorial' (the default) will print out all text and pause for user input before running commands. 'simulate' will not print out the text but will still pause for input.")
    p.add_option('--path', '-p', default="demo_scripts/",
                 help="The Path to the demo scripts directory.")
    p.add_option('--auto', '-a', default="False",
                 help="Set to 'true' (or 'yes') to prevent the application waiting for user keypresses between commands. Set to 'no' when running in test mode to allow users to step through each test.")
    p.add_option('--test', '-t', default="False",
                 help="If set to anything other than False the output of the command will be compared to the expected results in the sript. Any failures will be reported")
    p.add_option('--fastfail', default="True",
                 help="If set to anything other than True test execution has will stop on the first failure. This has no affect if running in any mode other than 'test'.")
    p.add_option('--debug', '-d', default="False",
                 help="Turn on debug logging by setting to True.")
    p.add_option('--webui', '-w', default="False",
                 help="If set to anything other than False will interact with the user through a Web UI rather than the CLI.")

    options, arguments = p.parse_args()

    if not options.path.endswith("/"):
        options.path += "/"

    if options.auto == "False":
        is_automatic = False
    else:
        is_automatic = True

    if options.test == "False":
        is_test = False
    else:
        is_test = True

    if options.fastfail == "True":
        is_fast_fail= True
    else:
        is_fast_fail= False

    if options.style == "simulate":
        simulate = True
    elif options.style == 'tutorial':
        simulate = False
    else:
        print("Unknown style (--style, -s): " + options.style)
        exit(1)

    if options.debug.lower() == "true":
        config.is_debug = True

    if len(arguments) == 2:
        script_dir = options.path + arguments[1]
    else:
        script_dir = options.path

    filename = "script.md"
    is_docker = os.path.isfile('/.dockerenv')
    demo = Demo(is_docker, script_dir, filename, simulate, is_automatic, is_test);

    if options.webui == "False":
        ui = Ui()
        if len(arguments) > 0:
            cmd = arguments[0]
            # 'run' is deprecated in the CLI, but not yet removed from code
            if cmd == "tutorial":
                cmd = "run"
    else:
        ui = WebUi()
        while not ui.ready:
            time.sleep(0.25)
            print("Waiting for client connection")
        cmd = None

    demo.set_ui(ui)
    demo.run(cmd)

main()
<commit_msg>initialize cmd<commit_after>#!/usr/bin/env python3

# This is a python script that emulates a terminal session and runs
# commands from a supplied markdown file..

import optparse
import os
import sys
import time

from cli import Ui
from web import WebUi
import config
from demo import Demo
from environment import Environment

def get_bash_script(script_dir, is_simulation = True, is_automated=False, is_testing=False):
    """
    Reads a README.md file in the indicated directoy and builds an
    executable bash script from the commands contained within.
    """
    if not script_dir.endswith('/'):
        script_dir = script_dir + "/"

    script = ""
    env = Environment(script_dir, False).get()
    for key, value in env.items():
        script += key + "='" + value + "'\n"

    filename = env.get_script_file_name(script_dir)
    in_code_block = False
    in_results_section = False
    lines = list(open(script_dir + filename))
    for line in lines:
        if line.startswith("Results:"):
            # Entering results section
            in_results_section = True
        elif line.startswith("```") and not in_code_block:
            # Entering a code block, if in_results_section = True then it's a results block
            in_code_block = True
        elif line.startswith("```") and in_code_block:
            # Finishing code block
            in_results_section = False
            in_code_block = False
        elif in_code_block and not in_results_section:
            # Executable line
            script += line
        elif line.startswith("#") and not in_code_block and not in_results_section and not is_automated:
            # Heading in descriptive text
            script += "\n"
    return script

def main():
    """SimDem CLI interpreter"""

    commands = config.modes
    command_string = ""
    for command in commands:
        command_string = command_string + command + "|"
    command_string = command_string[0:len(command_string)-1]

    p = optparse.OptionParser("%prog [" + command_string + "] <options> DEMO_NAME", version=config.SIMDEM_VERSION)
    p.add_option('--style', '-s', default="tutorial",
                 help="The style of simulation you want to run. 'tutorial' (the default) will print out all text and pause for user input before running commands. 'simulate' will not print out the text but will still pause for input.")
    p.add_option('--path', '-p', default="demo_scripts/",
                 help="The Path to the demo scripts directory.")
    p.add_option('--auto', '-a', default="False",
                 help="Set to 'true' (or 'yes') to prevent the application waiting for user keypresses between commands. Set to 'no' when running in test mode to allow users to step through each test.")
    p.add_option('--test', '-t', default="False",
                 help="If set to anything other than False the output of the command will be compared to the expected results in the sript. Any failures will be reported")
    p.add_option('--fastfail', default="True",
                 help="If set to anything other than True test execution has will stop on the first failure. This has no affect if running in any mode other than 'test'.")
    p.add_option('--debug', '-d', default="False",
                 help="Turn on debug logging by setting to True.")
    p.add_option('--webui', '-w', default="False",
                 help="If set to anything other than False will interact with the user through a Web UI rather than the CLI.")

    options, arguments = p.parse_args()

    if not options.path.endswith("/"):
        options.path += "/"

    if options.auto == "False":
        is_automatic = False
    else:
        is_automatic = True

    if options.test == "False":
        is_test = False
    else:
        is_test = True

    if options.fastfail == "True":
        is_fast_fail= True
    else:
        is_fast_fail= False

    if options.style == "simulate":
        simulate = True
    elif options.style == 'tutorial':
        simulate = False
    else:
        print("Unknown style (--style, -s): " + options.style)
        exit(1)

    if options.debug.lower() == "true":
        config.is_debug = True

    if len(arguments) == 2:
        script_dir = options.path + arguments[1]
    else:
        script_dir = options.path

    filename = "script.md"
    is_docker = os.path.isfile('/.dockerenv')
    demo = Demo(is_docker, script_dir, filename, simulate, is_automatic, is_test);

    cmd = None
    if options.webui == "False":
        ui = Ui()
        if len(arguments) > 0:
            cmd = arguments[0]
            # 'run' is deprecated in the CLI, but not yet removed from code
            if cmd == "tutorial":
                cmd = "run"
    else:
        ui = WebUi()
        while not ui.ready:
            time.sleep(0.25)
            print("Waiting for client connection")
        cmd = None

    demo.set_ui(ui)
    demo.run(cmd)

main()
<|endoftext|>


<commit_before>import json
import logging
import urllib

from flask import Flask, jsonify, request
from google.appengine.api import urlfetch, urlfetch_errors

app = Flask(__name__)
app.config['DEBUG'] = True

@app.route('/')
def hello():
    return 'The Yoda API, this is. Mmmmm.'

@app.route('/api/v1/yodish', methods=['GET'])
def get_yodish():
    return jsonify({'yodish': translate(request.args.get('text'))})

def translate(source):
    pos_tagged = fetch_pos(source.lower().strip(' .'))
    words = word_list(pos_tagged.strip('()'))
    return words

def fetch_pos(source):
    """ Parts of speech tagging from text-processing.com. """
    try:
        response = urlfetch.fetch(
            'http://text-processing.com/api/tag/',
            payload = urllib.urlencode({'text': source}),
            method = urlfetch.POST,
            headers = {'Content-Type': 'application/x-www-form-urlencoded'}
        )
    except urlfetch_errors.DeadlineExceededError:
        return "Timed out, your request has. Mmmmmm. Try again later, you must."

    if response.status_code == 200:
        return json.loads(response.content)['text']

def word_list(sentence):
    words = sentence.split()
    del(words[0])

    for i in range(0, len(words)):
        word, tag = words[i].split('/')
        words[i] = (word, tag)
    logging.warning(words)
    return words
<commit_msg>Apply rule: "You are strong" -> "Strong, you are"<commit_after>import json
import logging
import re
import urllib

from flask import Flask, jsonify, request
from google.appengine.api import urlfetch, urlfetch_errors

app = Flask(__name__)
app.config['DEBUG'] = True

@app.route('/')
def hello():
    return 'The Yoda API, this is. Mmmmm.'

@app.route('/api/v1/yodish', methods=['GET'])
def get_yodish():
    return jsonify({'yodish': translate(request.args.get('text'))})

def translate(source):
    pos_tagged = fetch_pos(source.lower().strip(' .'))
    words = word_list(pos_tagged.strip('()'))
    yodish = apply_yodish_grammar(words)
    sentence = assemble(yodish)
    return sentence

def fetch_pos(source):
    """ Parts of speech tagging from text-processing.com. """
    try:
        response = urlfetch.fetch(
            'http://text-processing.com/api/tag/',
            payload = urllib.urlencode({'text': source}),
            method = urlfetch.POST,
            headers = {'Content-Type': 'application/x-www-form-urlencoded'}
        )
    except urlfetch_errors.DeadlineExceededError:
        return "Timed out, your request has. Mmmmmm. Try again later, you must."

    if response.status_code == 200:
        return json.loads(response.content)['text']

def word_list(sentence):
    words = sentence.split()
    del(words[0])

    for i in range(0, len(words)):
        word, tag = words[i].split('/')
        words[i] = (word, tag)
    return words

def apply_yodish_grammar(words):
    yodish = []
    current = 0

    for i in range(0, len(words)):
        tag = words[i][1]

        # RULE: PRP + VBP pairings go to the end, prepended with comma
        # Example: "You are sad." -> "Sad, you are."
        if tag == 'PRP':
            if words[i+1][1] == 'VBP':
                yodish.append(',')
                yodish.append(words[i][0])
                yodish.append(words[i+1][0])
        elif tag == 'VBP':
            continue
        else:
            yodish.insert(current, words[i][0])
            current += 1

    return yodish

def assemble(words):
    words[0] = words[0].capitalize()
    words[-1] += '.'
    # Remove whitespace before punctuation
    return re.sub(
        r'\s+(\W)',
        r'\1',
        ' '.join(words)
    )

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Tue Mar 14 02:10:46 2017

@author: guida
"""
from telebot import *
from sonarr import *
import time
import config

TOKEN = config.TOKEN
SONARR_API = config.SONARR_API
CHAT_ID = config.CHAT_ID

telegram = bot(TOKEN)
sonarr = sonarr(SONARR_API)

def tel_sonarr_calendar():
    text = []
    calendar = sonarr.get_calendar()
    for series in calendar:
        text.append("--{} - {}  -{}".format(series['title'] \
        ,series['episode'],series['date']))
    message = "\n".join(text)
    telegram.send_message(message,CHAT_ID)

def tel_sonarr_grabbed():
    grabbed = sonarr.get_history()
    text = []
    for episode in grabbed:
        text.append("--{} - {}".format(episode['title'],episode['episode']))
    message = "\n".join(text)
    telegram.send_message(message,CHAT_ID)

def handle_updates(updates):
    for update in updates["result"]:
        text = update["message"]["text"]
        if text == "/sonarr":
            keyboard = telegram.build_keyboard(['Sonarr grabbed','Sonarr calendar'])
            telegram.send_message("Select an option", CHAT_ID, keyboard)

        elif text == "/start":
            keyboard = telegram.build_keyboard(['/sonarr'])
            telegram.send_message("Server Manager", CHAT_ID, keyboard)
        elif text == "Sonarr calendar":
            tel_sonarr_calendar()
        elif text == "Sonarr grabbed":
            tel_sonarr_grabbed()
        else:
            telegram.send_message("Sorry, I didn't understand you",CHAT_ID)

def main():
    last_update_id = None
    while True:
        updates = telegram.get_updates(last_update_id)
        if len(updates["result"]) > 0:
            last_update_id = telegram.get_last_update_id(updates) + 1
            handle_updates(updates)
        time.sleep(0.5)

if __name__ == '__main__':
  main()<commit_msg>Added simple logger<commit_after># -*- coding: utf-8 -*-
"""
Created on Tue Mar 14 02:10:46 2017

@author: guida
"""
from telebot import *
from sonarr import *
import time
import config
import logging

#Logging system Setup
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
# create a file handler
handler = logging.handlers.RotatingFileHandler('telegram-raspy.log',mode='w',
                                       maxBytes=20000,backupCount=3)
handler.setLevel(logging.INFO)
# create a logging format
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.info('---------------Telegram-Raspy------------' )
logger.info('Adding personal info(Token,API and Chat_id)')
TOKEN = config.TOKEN
SONARR_API = config.SONARR_API
CHAT_ID = config.CHAT_ID

logger.info('Initializing telegram bot and sonarr parser')
telegram = bot(TOKEN)
sonarr = sonarr(SONARR_API)

def tel_sonarr_calendar():
    text = []
    calendar = sonarr.get_calendar()
    for series in calendar:
        text.append("--{} - {}  -{}".format(series['title'] \
        ,series['episode'],series['date']))
    message = "\n".join(text)
    telegram.send_message(message,CHAT_ID)

def tel_sonarr_grabbed():
    grabbed = sonarr.get_history()
    text = []
    for episode in grabbed:
        text.append("--{} - {}".format(episode['title'],episode['episode']))
    message = "\n".join(text)
    telegram.send_message(message,CHAT_ID)

def handle_updates(updates):
    for update in updates["result"]:
        text = update["message"]["text"]
        logger.info('Message says: "%s"', text)
        if text == "/sonarr":
            keyboard = telegram.build_keyboard(['Sonarr grabbed','Sonarr calendar'])
            telegram.send_message("Select an option", CHAT_ID, keyboard)

        elif text == "/start":
            keyboard = telegram.build_keyboard(['/sonarr'])
            telegram.send_message("Server Manager", CHAT_ID, keyboard)
        elif text == "Sonarr calendar":
            logger.info('SCRIPT: Sending Sonarr calendar')
            tel_sonarr_calendar()
        elif text == "Sonarr grabbed":
            logger.info('SCRIPT: Sending Sonarr grabbed history')
            tel_sonarr_grabbed()
        else:
            logger.info('Message not understood')
            telegram.send_message("Sorry, I didn't understand you",CHAT_ID)

def main():
    logger.info('Initialized')
    last_update_id = None
    while True:
        updates = telegram.get_updates(last_update_id)
        if len(updates["result"]) > 0:
            logger.info('Message received, Update_id: %s', str(last_update_id))
            last_update_id = telegram.get_last_update_id(updates) + 1
            handle_updates(updates)
        time.sleep(0.5)

if __name__ == '__main__':
  main()<|endoftext|>


<commit_before>import bisect
import collections
import torch
from .interp import _interp_evaluate, _interp_fit
from .misc import (_compute_error_ratio,
                   _select_initial_step,
                   _optimal_step_size)
from .solvers import AdaptiveStepsizeODESolver

_ButcherTableau = collections.namedtuple('_ButcherTableau', 'alpha, beta, c_sol, c_error')

_RungeKuttaState = collections.namedtuple('_RungeKuttaState', 'y1, f1, t0, t1, dt, interp_coeff')
# Saved state of the Runge Kutta solver.
#
# Attributes:
#     y1: Tensor giving the function value at the end of the last time step.
#     f1: Tensor giving derivative at the end of the last time step.
#     t0: scalar float64 Tensor giving start of the last time step.
#     t1: scalar float64 Tensor giving end of the last time step.
#     dt: scalar float64 Tensor giving the size for the next time step.
#     interp_coeff: list of Tensors giving coefficients for polynomial
#         interpolation between `t0` and `t1`.

class _UncheckedAssign(torch.autograd.Function):
    @staticmethod
    def forward(ctx, scratch, value, index):
        ctx.index = index
        scratch.data[index] = value  # sneak past the version checker
        return scratch

    @staticmethod
    def backward(ctx, grad_scratch):
        return grad_scratch, grad_scratch[ctx.index], None

def _runge_kutta_step(func, y0, f0, t0, dt, tableau):
    """Take an arbitrary Runge-Kutta step and estimate error.

    Args:
        func: Function to evaluate like `func(t, y)` to compute the time derivative of `y`.
        y0: Tensor initial value for the state.
        f0: Tensor initial value for the derivative, computed from `func(t0, y0)`.
        t0: float64 scalar Tensor giving the initial time.
        dt: float64 scalar Tensor giving the size of the desired time step.
        tableau: _ButcherTableau describing how to take the Runge-Kutta step.

    Returns:
        Tuple `(y1, f1, y1_error, k)` giving the estimated function value after
        the Runge-Kutta step at `t1 = t0 + dt`, the derivative of the state at `t1`,
        estimated error at `t1`, and a list of Runge-Kutta coefficients `k` used for
        calculating these terms.
    """

    t0 = t0.type_as(y0)
    dt = dt.type_as(y0)

    # We use an unchecked assign to put data into k without incrementing its _version counter, so that the backward
    # doesn't throw an (overzealous) error about in-place correctness. We know that it's actually correct.
    k = torch.empty(*f0.shape, len(tableau.alpha) + 1, dtype=y0.dtype, device=y0.device)
    k = _UncheckedAssign.apply(k, f0, (..., 0))
    for i, (alpha_i, beta_i) in enumerate(zip(tableau.alpha, tableau.beta)):
        ti = t0 + alpha_i * dt
        yi = y0 + k[..., :i + 1].matmul(beta_i * dt).view_as(f0)
        f = func(ti, yi)
        k = _UncheckedAssign.apply(k, f, (..., i + 1))

    if not (tableau.c_sol[-1] == 0 and (tableau.c_sol[:-1] == tableau.beta[-1]).all()):
        # This property (true for Dormand-Prince) lets us save a few FLOPs.
        yi = y0 + k.matmul(dt * tableau.c_sol).view_as(f0)

    y1 = yi
    f1 = k[..., -1]
    y1_error = k.matmul(dt * tableau.c_error)
    return y1, f1, y1_error, k

# Precompute divisions
_one_third = 1 / 3
_two_thirds = 2 / 3
_one_sixth = 1 / 6

def rk4_step_func(func, t, dt, y, k1=None):
    if k1 is None:
        k1 = func(t, y)
    half_dt = dt * 0.5
    k2 = func(t + half_dt, y + half_dt * k1)
    k3 = func(t + half_dt, y + half_dt * k2)
    k4 = func(t + dt, y + dt * k3)
    return (k1 + 2 * (k2 + k3) + k4) * dt * _one_sixth

def rk4_alt_step_func(func, t, dt, y, k1=None):
    """Smaller error with slightly more compute."""
    if k1 is None:
        k1 = func(t, y)
    k2 = func(t + dt * _one_third, y + dt * k1 * _one_third)
    k3 = func(t + dt * _two_thirds, y + dt * (k2 - k1 * _one_third))
    k4 = func(t + dt, y + dt * (k1 - k2 + k3))
    return (k1 + 3 * (k2 + k3) + k4) * dt * 0.125

class RKAdaptiveStepsizeODESolver(AdaptiveStepsizeODESolver):
    order: int
    tableau: _ButcherTableau
    mid: torch.Tensor

    def __init__(self, func, y0, rtol, atol, first_step=None, safety=0.9, ifactor=10.0, dfactor=0.2,
                 max_num_steps=2 ** 31 - 1, grid_points=None, eps=0., dtype=torch.float64, **kwargs):
        super(RKAdaptiveStepsizeODESolver, self).__init__(dtype=dtype, y0=y0, **kwargs)

        # We use mixed precision. y has its original dtype (probably float32), whilst all 'time'-like objects use
        # `dtype` (defaulting to float64).
        dtype = torch.promote_types(dtype, y0.dtype)
        device = y0.device

        self.func = lambda t, y: func(t.type_as(y), y)
        self.rtol = torch.as_tensor(rtol, dtype=dtype, device=device)
        self.atol = torch.as_tensor(atol, dtype=dtype, device=device)
        self.first_step = None if first_step is None else torch.as_tensor(first_step, dtype=dtype, device=device)
        self.safety = torch.as_tensor(safety, dtype=dtype, device=device)
        self.ifactor = torch.as_tensor(ifactor, dtype=dtype, device=device)
        self.dfactor = torch.as_tensor(dfactor, dtype=dtype, device=device)
        self.max_num_steps = torch.as_tensor(max_num_steps, dtype=torch.int32, device=device)
        grid_points = torch.tensor([], dtype=dtype, device=device) if grid_points is None else grid_points.to(dtype)
        self.grid_points = grid_points
        self.eps = torch.as_tensor(eps, dtype=dtype, device=device)
        self.dtype = dtype

        # Copy from class to instance to set device
        self.tableau = _ButcherTableau(alpha=self.tableau.alpha.to(device=device, dtype=y0.dtype),
                                       beta=[b.to(device=device, dtype=y0.dtype) for b in self.tableau.beta],
                                       c_sol=self.tableau.c_sol.to(device=device, dtype=y0.dtype),
                                       c_error=self.tableau.c_error.to(device=device, dtype=y0.dtype))
        self.mid = self.mid.to(device=device, dtype=y0.dtype)

    def _before_integrate(self, t):
        f0 = self.func(t[0], self.y0)
        if self.first_step is None:
            first_step = _select_initial_step(self.func, t[0], self.y0, self.order - 1, self.rtol, self.atol,
                                              self.norm, f0=f0)
        else:
            first_step = self.first_step
        self.rk_state = _RungeKuttaState(self.y0, f0, t[0], t[0], first_step, [self.y0] * 5)
        self.next_grid_index = min(bisect.bisect(self.grid_points.tolist(), t[0]), len(self.grid_points) - 1)

    def _advance(self, next_t):
        """Interpolate through the next time point, integrating as necessary."""
        n_steps = 0
        while next_t > self.rk_state.t1:
            assert n_steps < self.max_num_steps, 'max_num_steps exceeded ({}>={})'.format(n_steps, self.max_num_steps)
            self.rk_state = self._adaptive_step(self.rk_state)
            n_steps += 1
        return _interp_evaluate(self.rk_state.interp_coeff, self.rk_state.t0, self.rk_state.t1, next_t)

    def _adaptive_step(self, rk_state):
        """Take an adaptive Runge-Kutta step to integrate the ODE."""
        y0, f0, _, t0, dt, interp_coeff = rk_state
        # dtypes: self.y0.dtype (probably float32); self.dtype (probably float64)
        # used for state and timelike objects respectively.
        # Then:
        # y0.dtype == self.y0.dtype
        # f0.dtype == self.y0.dtype
        # t0.dtype == self.dtype
        # dt.dtype == self.dtype
        # for coeff in interp_coeff: coeff.dtype == self.y0.dtype

        ########################################################
        #                      Assertions                      #
        ########################################################
        assert t0 + dt > t0, 'underflow in dt {}'.format(dt.item())
        assert torch.isfinite(y0).all(), 'non-finite values in state `y`: {}'.format(y0)

        ########################################################
        #     Make step, respecting prescribed grid points     #
        ########################################################
        on_grid = len(self.grid_points) and t0 < self.grid_points[self.next_grid_index] < t0 + dt
        if on_grid:
            dt = self.grid_points[self.next_grid_index] - t0
            eps = min(0.5 * dt, self.eps)
            dt = dt - eps
        else:
            eps = 0

        y1, f1, y1_error, k = _runge_kutta_step(self.func, y0, f0, t0, dt, tableau=self.tableau)
        # dtypes:
        # y1.dtype == self.y0.dtype
        # f1.dtype == self.y0.dtype
        # y1_error.dtype == self.dtype
        # k.dtype == self.y0.dtype

        ########################################################
        #                     Error Ratio                      #
        ########################################################
        error_ratio = _compute_error_ratio(y1_error, self.rtol, self.atol, y0, y1, self.norm)
        accept_step = error_ratio <= 1
        # dtypes:
        # error_ratio.dtype == self.dtype

        ########################################################
        #                   Update RK State                    #
        ########################################################
        t_next = t0 + dt + 2 * eps if accept_step else t0
        y_next = y1 if accept_step else y0
        if on_grid and accept_step:
            # We've just passed a discontinuity in f; we should update f to match the side of the discontinuity we're
            # now on.
            if eps != 0:
                f1 = self.func(t_next, y_next)
            if self.next_grid_index != len(self.grid_points) - 1:
                self.next_grid_index += 1
        f_next = f1 if accept_step else f0
        interp_coeff = self._interp_fit(y0, y1, k, dt) if accept_step else interp_coeff
        dt_next = _optimal_step_size(dt, error_ratio, self.safety, self.ifactor, self.dfactor, self.order)
        rk_state = _RungeKuttaState(y_next, f_next, t0, t_next, dt_next, interp_coeff)
        return rk_state

    def _interp_fit(self, y0, y1, k, dt):
        """Fit an interpolating polynomial to the results of a Runge-Kutta step."""
        dt = dt.type_as(y0)
        y_mid = y0 + k.matmul(dt * self.mid).view_as(y0)
        f0 = k[..., 0]
        f1 = k[..., -1]
        return _interp_fit(y0, y1, y_mid, f0, f1, dt)
<commit_msg>use torch.nextafter as default for grid_points with eps=None<commit_after>import bisect
import collections
import torch
from .interp import _interp_evaluate, _interp_fit
from .misc import (_compute_error_ratio,
                   _select_initial_step,
                   _optimal_step_size)
from .solvers import AdaptiveStepsizeODESolver

_ButcherTableau = collections.namedtuple('_ButcherTableau', 'alpha, beta, c_sol, c_error')

_RungeKuttaState = collections.namedtuple('_RungeKuttaState', 'y1, f1, t0, t1, dt, interp_coeff')
# Saved state of the Runge Kutta solver.
#
# Attributes:
#     y1: Tensor giving the function value at the end of the last time step.
#     f1: Tensor giving derivative at the end of the last time step.
#     t0: scalar float64 Tensor giving start of the last time step.
#     t1: scalar float64 Tensor giving end of the last time step.
#     dt: scalar float64 Tensor giving the size for the next time step.
#     interp_coeff: list of Tensors giving coefficients for polynomial
#         interpolation between `t0` and `t1`.

class _UncheckedAssign(torch.autograd.Function):
    @staticmethod
    def forward(ctx, scratch, value, index):
        ctx.index = index
        scratch.data[index] = value  # sneak past the version checker
        return scratch

    @staticmethod
    def backward(ctx, grad_scratch):
        return grad_scratch, grad_scratch[ctx.index], None

def _runge_kutta_step(func, y0, f0, t0, dt, tableau):
    """Take an arbitrary Runge-Kutta step and estimate error.

    Args:
        func: Function to evaluate like `func(t, y)` to compute the time derivative of `y`.
        y0: Tensor initial value for the state.
        f0: Tensor initial value for the derivative, computed from `func(t0, y0)`.
        t0: float64 scalar Tensor giving the initial time.
        dt: float64 scalar Tensor giving the size of the desired time step.
        tableau: _ButcherTableau describing how to take the Runge-Kutta step.

    Returns:
        Tuple `(y1, f1, y1_error, k)` giving the estimated function value after
        the Runge-Kutta step at `t1 = t0 + dt`, the derivative of the state at `t1`,
        estimated error at `t1`, and a list of Runge-Kutta coefficients `k` used for
        calculating these terms.
    """

    t0 = t0.type_as(y0)
    dt = dt.type_as(y0)

    # We use an unchecked assign to put data into k without incrementing its _version counter, so that the backward
    # doesn't throw an (overzealous) error about in-place correctness. We know that it's actually correct.
    k = torch.empty(*f0.shape, len(tableau.alpha) + 1, dtype=y0.dtype, device=y0.device)
    k = _UncheckedAssign.apply(k, f0, (..., 0))
    for i, (alpha_i, beta_i) in enumerate(zip(tableau.alpha, tableau.beta)):
        ti = t0 + alpha_i * dt
        yi = y0 + k[..., :i + 1].matmul(beta_i * dt).view_as(f0)
        f = func(ti, yi)
        k = _UncheckedAssign.apply(k, f, (..., i + 1))

    if not (tableau.c_sol[-1] == 0 and (tableau.c_sol[:-1] == tableau.beta[-1]).all()):
        # This property (true for Dormand-Prince) lets us save a few FLOPs.
        yi = y0 + k.matmul(dt * tableau.c_sol).view_as(f0)

    y1 = yi
    f1 = k[..., -1]
    y1_error = k.matmul(dt * tableau.c_error)
    return y1, f1, y1_error, k

# Precompute divisions
_one_third = 1 / 3
_two_thirds = 2 / 3
_one_sixth = 1 / 6

def rk4_step_func(func, t, dt, y, k1=None):
    if k1 is None:
        k1 = func(t, y)
    half_dt = dt * 0.5
    k2 = func(t + half_dt, y + half_dt * k1)
    k3 = func(t + half_dt, y + half_dt * k2)
    k4 = func(t + dt, y + dt * k3)
    return (k1 + 2 * (k2 + k3) + k4) * dt * _one_sixth

def rk4_alt_step_func(func, t, dt, y, k1=None):
    """Smaller error with slightly more compute."""
    if k1 is None:
        k1 = func(t, y)
    k2 = func(t + dt * _one_third, y + dt * k1 * _one_third)
    k3 = func(t + dt * _two_thirds, y + dt * (k2 - k1 * _one_third))
    k4 = func(t + dt, y + dt * (k1 - k2 + k3))
    return (k1 + 3 * (k2 + k3) + k4) * dt * 0.125

class RKAdaptiveStepsizeODESolver(AdaptiveStepsizeODESolver):
    order: int
    tableau: _ButcherTableau
    mid: torch.Tensor

    def __init__(self, func, y0, rtol, atol, first_step=None, safety=0.9, ifactor=10.0, dfactor=0.2,
                 max_num_steps=2 ** 31 - 1, grid_points=None, eps=None, dtype=torch.float64, **kwargs):
        super(RKAdaptiveStepsizeODESolver, self).__init__(dtype=dtype, y0=y0, **kwargs)

        # We use mixed precision. y has its original dtype (probably float32), whilst all 'time'-like objects use
        # `dtype` (defaulting to float64).
        dtype = torch.promote_types(dtype, y0.dtype)
        device = y0.device

        self.func = lambda t, y: func(t.type_as(y), y)
        self.rtol = torch.as_tensor(rtol, dtype=dtype, device=device)
        self.atol = torch.as_tensor(atol, dtype=dtype, device=device)
        self.first_step = None if first_step is None else torch.as_tensor(first_step, dtype=dtype, device=device)
        self.safety = torch.as_tensor(safety, dtype=dtype, device=device)
        self.ifactor = torch.as_tensor(ifactor, dtype=dtype, device=device)
        self.dfactor = torch.as_tensor(dfactor, dtype=dtype, device=device)
        self.max_num_steps = torch.as_tensor(max_num_steps, dtype=torch.int32, device=device)
        grid_points = torch.tensor([], dtype=dtype, device=device) if grid_points is None else grid_points.to(dtype)
        self.grid_points = grid_points
        self.eps = eps if eps is None else torch.as_tensor(eps, dtype=dtype, device=device)
        self.dtype = dtype

        # Copy from class to instance to set device
        self.tableau = _ButcherTableau(alpha=self.tableau.alpha.to(device=device, dtype=y0.dtype),
                                       beta=[b.to(device=device, dtype=y0.dtype) for b in self.tableau.beta],
                                       c_sol=self.tableau.c_sol.to(device=device, dtype=y0.dtype),
                                       c_error=self.tableau.c_error.to(device=device, dtype=y0.dtype))
        self.mid = self.mid.to(device=device, dtype=y0.dtype)

    def _before_integrate(self, t):
        f0 = self.func(t[0], self.y0)
        if self.first_step is None:
            first_step = _select_initial_step(self.func, t[0], self.y0, self.order - 1, self.rtol, self.atol,
                                              self.norm, f0=f0)
        else:
            first_step = self.first_step
        self.rk_state = _RungeKuttaState(self.y0, f0, t[0], t[0], first_step, [self.y0] * 5)
        self.next_grid_index = min(bisect.bisect(self.grid_points.tolist(), t[0]), len(self.grid_points) - 1)

    def _advance(self, next_t):
        """Interpolate through the next time point, integrating as necessary."""
        n_steps = 0
        while next_t > self.rk_state.t1:
            assert n_steps < self.max_num_steps, 'max_num_steps exceeded ({}>={})'.format(n_steps, self.max_num_steps)
            self.rk_state = self._adaptive_step(self.rk_state)
            n_steps += 1
        return _interp_evaluate(self.rk_state.interp_coeff, self.rk_state.t0, self.rk_state.t1, next_t)

    def _adaptive_step(self, rk_state):
        """Take an adaptive Runge-Kutta step to integrate the ODE."""
        y0, f0, _, t0, dt, interp_coeff = rk_state
        # dtypes: self.y0.dtype (probably float32); self.dtype (probably float64)
        # used for state and timelike objects respectively.
        # Then:
        # y0.dtype == self.y0.dtype
        # f0.dtype == self.y0.dtype
        # t0.dtype == self.dtype
        # dt.dtype == self.dtype
        # for coeff in interp_coeff: coeff.dtype == self.y0.dtype

        ########################################################
        #                      Assertions                      #
        ########################################################
        assert t0 + dt > t0, 'underflow in dt {}'.format(dt.item())
        assert torch.isfinite(y0).all(), 'non-finite values in state `y`: {}'.format(y0)

        ########################################################
        #     Make step, respecting prescribed grid points     #
        ########################################################
        on_grid = len(self.grid_points) and t0 < self.grid_points[self.next_grid_index] < t0 + dt
        eps = 0
        _func = self.func
        if on_grid:
            t_grid = self.grid_points[self.next_grid_index].type_as(y0)
            print("on grid")
            dt = self.grid_points[self.next_grid_index] - t0
            if self.eps is None:
                dt = torch.nextafter(dt.type_as(y0), dt.type_as(y0) - 1.0).type_as(dt)
                # Ensure the time value never exceeds or equals the grid time.
                _func = lambda t, y: self.func(min(t, torch.nextafter(t_grid, t_grid - 1.0)), y)
            else:
                eps = min(0.5 * dt, self.eps)
                dt = dt - eps

        y1, f1, y1_error, k = _runge_kutta_step(_func, y0, f0, t0, dt, tableau=self.tableau)
        # dtypes:
        # y1.dtype == self.y0.dtype
        # f1.dtype == self.y0.dtype
        # y1_error.dtype == self.dtype
        # k.dtype == self.y0.dtype

        ########################################################
        #                     Error Ratio                      #
        ########################################################
        error_ratio = _compute_error_ratio(y1_error, self.rtol, self.atol, y0, y1, self.norm)
        accept_step = error_ratio <= 1
        # dtypes:
        # error_ratio.dtype == self.dtype

        ########################################################
        #                   Update RK State                    #
        ########################################################
        t_next = t0 + dt + 2 * eps if accept_step else t0
        y_next = y1 if accept_step else y0
        if on_grid and accept_step:
            # We've just passed a discontinuity in f; we should update f to match the side of the discontinuity we're
            # now on.
            if self.eps is None:
                print("eps is None")
                t_next = torch.nextafter(t_grid, t_grid + 1.0)
            if self.eps != 0:
                f1 = self.func(t_next, y_next)
            if self.next_grid_index != len(self.grid_points) - 1:
                self.next_grid_index += 1
        f_next = f1 if accept_step else f0
        interp_coeff = self._interp_fit(y0, y1, k, dt) if accept_step else interp_coeff
        dt_next = _optimal_step_size(dt, error_ratio, self.safety, self.ifactor, self.dfactor, self.order)
        rk_state = _RungeKuttaState(y_next, f_next, t0, t_next, dt_next, interp_coeff)
        return rk_state

    def _interp_fit(self, y0, y1, k, dt):
        """Fit an interpolating polynomial to the results of a Runge-Kutta step."""
        dt = dt.type_as(y0)
        y_mid = y0 + k.matmul(dt * self.mid).view_as(y0)
        f0 = k[..., 0]
        f1 = k[..., -1]
        return _interp_fit(y0, y1, y_mid, f0, f1, dt)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import vklogic as vkl

from io import BytesIO
import random
import pycurl
import time
import json
import re

__version__ = '0.1.0'
__author__ = 'Eugene Ershov - http://vk.com/fogapod'

__info__ = '''
Версия: {ver}
Я умею:
*Говорить то, что вы попросите
(/say text|/скажи текст)
*Вызывать помощь
(/help|/помощь)

Автор: {author}'''.format(\
	ver = __version__, author = __author__
)

def animate_loading(text, delay):
	loading_symbols = ('|', '/', '-', '\\')
	for i, symbol in enumerate(loading_symbols):
		print('#{} {}\r'.format(text, symbol), end='')
		time.sleep(delay/len(loading_symbols))

def main():
	client = vkl.Client()

	while not client.authorization():
		continue

	#client.save_full_message_history()

	url = client.make_url()
	c = pycurl.Curl()
	m = pycurl.CurlMulti()

	last_rnd_id = 1
	while True:
		s = BytesIO()
		c.setopt(c.URL, url)
		c.setopt(c.WRITEFUNCTION, s.write)
		m.add_handle(c)

		while True:
			ret, num_handles = m.perform()
			if ret != pycurl.E_CALL_MULTI_PERFORM:
				break

		while num_handles: # main loop
			animate_loading('Listening long poll...', 1)
			while 1: # main loop (2)
				ret, num_handles = m.perform()
				if ret != pycurl.E_CALL_MULTI_PERFORM:
					break

		m.remove_handle(c)
		response = s.getvalue()
		response = response.decode('utf-8')
		response = json.loads(response)

		url = client.make_url(keep_ts=response['ts'])

		for update in response['updates']:
			if update[0] is 4 and update[7] != last_rnd_id and update[3]:
				text = update[6]
				mark_msg = True
				if text.lower() == 'ершов' or\
						text.lower() == 'женя' or\
						text.lower() == 'жень' or\
						text.lower() == 'женька' or\
						text.lower() == 'жека' or\
						text.lower() == 'жэка':
					text = 'А'
					mark_msg = False

				elif 'HALP' in text:
					text = 'Кому нужна помощь?!'

				elif re.sub('^( )*', '', text).startswith('/'):
					text = text[1:]
					words = text.split()
					if not words:
						words = ' '

					if re.match('^((help)|(помощь))', words[0].lower()):
						text = __info__

					if words[0].startswith('/'):
						words[0] = words[0][1:]
						mark_msg = False

					if re.match('^((help)|(помощь)|(info)|(инфо)|(информация)|\?)',\
							words[0].lower()):
						text = __info__
					elif re.match('^((скажи)|(say))', words[0].lower()):
						del words[0]
						text = ' '.join(words)
					else:
						text = 'Попка молодец🐔' if random.randint(0,1) else 'Попка дурак🐔'

				else:
					continue

				last_rnd_id = update + 1
				client.reply(
					uid = update[3],
					text = text + "'" if mark_msg else text,
					rnd_id = last_rnd_id
				)

if __name__ == '__main__':
	main()
<commit_msg>Улучшение анимации работы<commit_after># -*- coding: utf-8 -*-
import vklogic as vkl

from io import BytesIO
import random
import pycurl
import time
import json
import re

__version__ = '0.1.0'
__author__ = 'Eugene Ershov - http://vk.com/fogapod'

__info__ = '''
Версия: {ver}
Я умею:
*Говорить то, что вы попросите
(/say text|/скажи текст)
*Вызывать помощь
(/help|/помощь)

Автор: {author}'''.format(\
	ver = __version__, author = __author__
)

def animate_loading(text, delay):
	loading_symbols = ('|', '/', '-', '\\')
	for i, symbol in enumerate(loading_symbols):
		print('#{} {}\r'.format(text, symbol), end='')
		time.sleep(delay/len(loading_symbols))

def main():
	client = vkl.Client()

	while not client.authorization():
		continue

	client.save_full_message_history()

	url = client.make_url()
	c = pycurl.Curl()
	m = pycurl.CurlMulti()

	last_rnd_id = 1
	reply_count = 0
	while True:
		s = BytesIO()
		c.setopt(c.URL, url)
		c.setopt(c.WRITEFUNCTION, s.write)
		m.add_handle(c)

		while True:
			ret, num_handles = m.perform()
			if ret != pycurl.E_CALL_MULTI_PERFORM:
				break

		while num_handles: # main loop
			animate_loading(
				'Listening long poll... {} replyes'.format(reply_count), 1
				)
			while 1: # main loop (2)
				ret, num_handles = m.perform()
				if ret != pycurl.E_CALL_MULTI_PERFORM:
					break

		m.remove_handle(c)
		response = s.getvalue()
		response = response.decode('utf-8')
		response = json.loads(response)

		url = client.make_url(keep_ts=response['ts'])

		for update in response['updates']:
			if update[0] is 4 and update[7] != last_rnd_id and update[3]:
				text = update[6]
				mark_msg = True
				if text.lower() == 'ершов' or\
						text.lower() == 'женя' or\
						text.lower() == 'жень' or\
						text.lower() == 'женька' or\
						text.lower() == 'жека' or\
						text.lower() == 'жэка':
					text = 'А'
					mark_msg = False

				elif 'HALP' in text:
					text = 'Кому нужна помощь?!'

				elif re.sub('^( )*', '', text).startswith('/'):
					text = text[1:]
					words = text.split()
					if not words:
						words = ' '

					if words[0].startswith('/'):
						words[0] = words[0][1:]
						mark_msg = False

					if re.match('^((help)|(помощь)|(info)|(инфо)|(информация)|\?)',\
							words[0].lower()):
						text = __info__
					elif re.match('^((скажи)|(say))', words[0].lower()):
						del words[0]
						text = ' '.join(words)
					else:
						text = 'Попка молодец🐔' if random.randint(0,1) else 'Попка дурак🐔'

				else:
					continue

				last_rnd_id = update[7] + 1
				client.reply(
					uid = update[3],
					text = text + "'" if mark_msg else text,
					rnd_id = last_rnd_id
				)
				reply_count += 1

if __name__ == '__main__':
	main()<|endoftext|>


<commit_before>import os
from torchtext.data.datasets_utils import (
    _download_extract_validate,
    _RawTextIterableDataset,
    _wrap_split_argument,
    _create_dataset_directory,
    _read_text_iterator,
)

URL = {
    'train': r'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/training.tar.gz',
    'valid': r'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz',
    'test': r'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/mmt16_task1_test.tar.gz',
}

MD5 = {
    'train': '20140d013d05dd9a72dfde46478663ba05737ce983f478f960c1123c6671be5e',
    'valid': 'a7aa20e9ebd5ba5adce7909498b94410996040857154dab029851af3a866da8c',
    'test': '0681be16a532912288a91ddd573594fbdd57c0fbb81486eff7c55247e35326c2',
}

_EXTRACTED_FILES_INFO = {
    'train': {
        'file_prefix': 'train',
        'md5': {
            'de': '695df46f6fd14567e69970408a2c129a50e778a910ecb1585a92eb25b2c7accc',
            'en': '4b4d37e774976ef44fecca1738cdeb3b3ba384851a59a755b9c5e6aa7d87b13c',
        },
    },
    'valid': {
        'file_prefix': 'val',
        'md5': {
            'de': 'fd0fc009db2446cfc12d96a382aff0d3122cb47577b352d0f7e0bb3a38e2e552',
            'en': '40cd20974079d9afb0e3d27c659a8e059cc2fcf850b4bc23ede13fc36dd8a865',
        },
    },
    'test': {
        'file_prefix': 'test',
        'md5': {
            'de': 'c1d2f544471a7387e37d15f1adf075ff7d6fe57a30840bb969281ae102d24cb1',
            'en': '399a4382932c1aadd3ceb9bef1008d388a64c76d4ae4e9d4728c6f4301cac182',
        },
    },
}

NUM_LINES = {
    'train': 29000,
    'valid': 1014,
    'test': 1000,
}

DATASET_NAME = "Multi30k"

@_create_dataset_directory(dataset_name=DATASET_NAME)
@_wrap_split_argument(('train', 'valid', 'test'))
def Multi30k(root, split, language_pair=('de', 'en')):
    """Multi30k dataset

    Reference: http://www.statmt.org/wmt16/multimodal-task.html#task1

    Args:
        root: Directory where the datasets are saved. Default: ".data"
        split: split or splits to be returned. Can be a string or tuple of strings. Default: (‘train’, ‘valid’, ‘test’)
        language_pair: tuple or list containing src and tgt language. Available options are ('de','en') and ('en', 'de')
    """

    assert (len(language_pair) == 2), 'language_pair must contain only 2 elements: src and tgt language respectively'
    assert (tuple(sorted(language_pair)) == ('de', 'en')), "language_pair must be either ('de','en') or ('en', 'de')"

    downloaded_file = os.path.basename(URL[split])

    src_path = _download_extract_validate(root, URL[split], MD5[split],
                                          os.path.join(root, downloaded_file),
                                          os.path.join(root, _EXTRACTED_FILES_INFO[split]['file_prefix'] + '.' + language_pair[0]),
                                          _EXTRACTED_FILES_INFO[split]['md5'][language_pair[0]])
    trg_path = _download_extract_validate(root, URL[split], MD5[split],
                                          os.path.join(root, downloaded_file),
                                          os.path.join(root, _EXTRACTED_FILES_INFO[split]['file_prefix'] + '.' + language_pair[1]),
                                          _EXTRACTED_FILES_INFO[split]['md5'][language_pair[1]])

    src_data_iter = _read_text_iterator(src_path)
    trg_data_iter = _read_text_iterator(trg_path)

    return _RawTextIterableDataset(DATASET_NAME, NUM_LINES[split], zip(src_data_iter, trg_data_iter))
<commit_msg>Update multi30k.py (#1351)<commit_after>import os
from torchtext.data.datasets_utils import (
    _download_extract_validate,
    _RawTextIterableDataset,
    _wrap_split_argument,
    _create_dataset_directory,
    _read_text_iterator,
)

URL = {
    'train': r'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/training.tar.gz',
    'valid': r'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz',
    'test': r'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/mmt16_task1_test.tar.gz',
}

MD5 = {
    'train': '20140d013d05dd9a72dfde46478663ba05737ce983f478f960c1123c6671be5e',
    'valid': 'a7aa20e9ebd5ba5adce7909498b94410996040857154dab029851af3a866da8c',
    'test': '0681be16a532912288a91ddd573594fbdd57c0fbb81486eff7c55247e35326c2',
}

_EXTRACTED_FILES_INFO = {
    'train': {
        'file_prefix': 'train',
        'md5': {
            'de': '695df46f6fd14567e69970408a2c129a50e778a910ecb1585a92eb25b2c7accc',
            'en': '4b4d37e774976ef44fecca1738cdeb3b3ba384851a59a755b9c5e6aa7d87b13c',
        },
    },
    'valid': {
        'file_prefix': 'val',
        'md5': {
            'de': 'fd0fc009db2446cfc12d96a382aff0d3122cb47577b352d0f7e0bb3a38e2e552',
            'en': '40cd20974079d9afb0e3d27c659a8e059cc2fcf850b4bc23ede13fc36dd8a865',
        },
    },
    'test': {
        'file_prefix': 'test',
        'md5': {
            'de': 'c1d2f544471a7387e37d15f1adf075ff7d6fe57a30840bb969281ae102d24cb1',
            'en': '399a4382932c1aadd3ceb9bef1008d388a64c76d4ae4e9d4728c6f4301cac182',
        },
    },
}

NUM_LINES = {
    'train': 29000,
    'valid': 1014,
    'test': 1000,
}

DATASET_NAME = "Multi30k"

@_create_dataset_directory(dataset_name=DATASET_NAME)
@_wrap_split_argument(('train', 'valid', 'test'))
def Multi30k(root, split, language_pair=('de', 'en')):
    """Multi30k dataset

    Reference: http://www.statmt.org/wmt16/multimodal-task.html#task1

    Args:
        root: Directory where the datasets are saved. Default: ".data"
        split: split or splits to be returned. Can be a string or tuple of strings. Default: ('train', 'valid', 'test')
        language_pair: tuple or list containing src and tgt language. Available options are ('de','en') and ('en', 'de')
    """

    assert (len(language_pair) == 2), 'language_pair must contain only 2 elements: src and tgt language respectively'
    assert (tuple(sorted(language_pair)) == ('de', 'en')), "language_pair must be either ('de','en') or ('en', 'de')"

    downloaded_file = os.path.basename(URL[split])

    src_path = _download_extract_validate(root, URL[split], MD5[split],
                                          os.path.join(root, downloaded_file),
                                          os.path.join(root, _EXTRACTED_FILES_INFO[split]['file_prefix'] + '.' + language_pair[0]),
                                          _EXTRACTED_FILES_INFO[split]['md5'][language_pair[0]])
    trg_path = _download_extract_validate(root, URL[split], MD5[split],
                                          os.path.join(root, downloaded_file),
                                          os.path.join(root, _EXTRACTED_FILES_INFO[split]['file_prefix'] + '.' + language_pair[1]),
                                          _EXTRACTED_FILES_INFO[split]['md5'][language_pair[1]])

    src_data_iter = _read_text_iterator(src_path)
    trg_data_iter = _read_text_iterator(trg_path)

    return _RawTextIterableDataset(DATASET_NAME, NUM_LINES[split], zip(src_data_iter, trg_data_iter))
<|endoftext|>


<commit_before># -*- coding:utf-8 -*-

import peewee
from peewee import SQL
from torcms.core import tools
from torcms.model.core_tab import TabPost
from torcms.model.core_tab import TabRel
from torcms.model.core_tab import TabPost2Tag
from torcms.model.abc_model import Mabc
from torcms.model.label_model import MPost2Label

class MRelation(Mabc):
    @staticmethod
    def add_relation(app_f, app_t, weight=1):
        '''
        Adding relation between two posts.
        '''
        recs = TabRel.select().where(
            (TabRel.post_f_id == app_f) & (TabRel.post_t_id == app_t)
        )
        if recs.count() > 1:
            for record in recs:
                MRelation.delete(record.uid)

        if recs.count() == 0:
            uid = tools.get_uuid()
            entry = TabRel.create(
                uid=uid,
                post_f_id=app_f,
                post_t_id=app_t,
                count=1,
            )
            return entry.uid
        elif recs.count() == 1:
            MRelation.update_relation(app_f, app_t, weight)
        else:
            return False

    @staticmethod
    def delete(uid):
        entry = TabRel.delete().where(
            TabRel.uid == uid
        )
        entry.execute()

    @staticmethod
    def update_relation(app_f, app_t, weight=1):
        try:
            postinfo = TabRel.get(
                (TabRel.post_f_id == app_f) & (TabRel.post_t_id == app_t)
            )
        except:
            return False
        entry = TabRel.update(
            count=postinfo.count + weight
        ).where(
            (TabRel.post_f_id == app_f) & (TabRel.post_t_id == app_t)
        )
        entry.execute()

    @staticmethod
    def get_app_relations(app_id, num=20, kind='1'):
        '''
        The the related infors. 如有标签按标签推荐，如无标签按标题推荐
        '''

        tag_info = filter(lambda x: not x.tag_name.startswith('_'),
                          MPost2Label.get_by_uid(app_id).objects())

        info =TabPost.get_by_id(app_id)

        tag_arr = []
        for tag in tag_info:
            tag_arr.append(tag.tag_uid)

        if len(tag_arr) > 0:

            rec = TabPost2Tag.select(
                TabPost2Tag,
                TabPost.title.alias('post_title'),
                TabPost.valid.alias('post_valid')
            ).join(
                TabPost, on=(TabPost2Tag.post_id == TabPost.uid)
            ).where(
                (TabPost2Tag.tag_id << tag_arr) &
                (TabPost.uid != app_id) &
                (TabPost.kind == kind) &
                (TabPost.valid == 1)
            ).distinct(TabPost2Tag.post_id).order_by(
                TabPost2Tag.post_id
            ).limit(num)
        else:

            rec = TabPost.select(
                TabPost.title.alias('post_title'),
                TabPost.valid.alias('post_valid'),
                TabPost.uid.alias('post_id')
            ).where(
                (SQL(("title like '%%{0}%%' ").format(info.title))) &
                (TabPost.uid != app_id) &
                (TabPost.kind == kind) &
                (TabPost.valid == 1)

            ).order_by(peewee.fn.Random()).limit(num)
        return rec
<commit_msg>xiugai`<commit_after># -*- coding:utf-8 -*-

import peewee
from peewee import SQL
from torcms.core import tools
from torcms.model.core_tab import TabPost
from torcms.model.core_tab import TabRel
from torcms.model.core_tab import TabPost2Tag
from torcms.model.abc_model import Mabc
from torcms.model.label_model import MPost2Label

class MRelation(Mabc):
    @staticmethod
    def add_relation(app_f, app_t, weight=1):
        '''
        Adding relation between two posts.
        '''
        recs = TabRel.select().where(
            (TabRel.post_f_id == app_f) & (TabRel.post_t_id == app_t)
        )
        if recs.count() > 1:
            for record in recs:
                MRelation.delete(record.uid)

        if recs.count() == 0:
            uid = tools.get_uuid()
            entry = TabRel.create(
                uid=uid,
                post_f_id=app_f,
                post_t_id=app_t,
                count=1,
            )
            return entry.uid
        elif recs.count() == 1:
            MRelation.update_relation(app_f, app_t, weight)
        else:
            return False

    @staticmethod
    def delete(uid):
        entry = TabRel.delete().where(
            TabRel.uid == uid
        )
        entry.execute()

    @staticmethod
    def update_relation(app_f, app_t, weight=1):
        try:
            postinfo = TabRel.get(
                (TabRel.post_f_id == app_f) & (TabRel.post_t_id == app_t)
            )
        except:
            return False
        entry = TabRel.update(
            count=postinfo.count + weight
        ).where(
            (TabRel.post_f_id == app_f) & (TabRel.post_t_id == app_t)
        )
        entry.execute()

    @staticmethod
    def get_app_relations(app_id, num=20, kind='1'):
        '''
        The the related infors. 如有标签按标签推荐，如无标签按标题推荐
        '''

        tag_info = filter(lambda x: not x.tag_name.startswith('_'),
                          MPost2Label.get_by_uid(app_id).objects())

        info = TabPost.get_by_id(app_id)

        tag_arr = []
        for tag in tag_info:
            tag_arr.append(tag.tag_uid)

        if len(tag_arr) > 0:

            recs = TabPost2Tag.select(
                TabPost2Tag,
                TabPost.title.alias('post_title'),
                TabPost.valid.alias('post_valid')
            ).join(
                TabPost, on=(TabPost2Tag.post_id == TabPost.uid)
            ).where(
                (TabPost2Tag.tag_id << tag_arr) &
                (TabPost.uid != app_id) &
                (TabPost.kind == kind) &
                (TabPost.valid == 1)
            ).distinct(TabPost2Tag.post_id).order_by(
                TabPost2Tag.post_id
            ).limit(num)
        else:

            recs = TabPost.select(
                TabPost.title.alias('post_title'),
                TabPost.valid.alias('post_valid'),
                TabPost.uid.alias('post_id')
            ).where(
                (SQL(("title like '%%{0}%%' ").format(info.title))) &
                (TabPost.uid != app_id) &
                (TabPost.kind == kind) &
                (TabPost.valid == 1)

            ).order_by(peewee.fn.Random()).limit(num)
        return recs
<|endoftext|>


<commit_before>#-*- coding: utf-8 -*-

##########################################################
# © 2013/2014 Daniel 'grindhold' Brendle with torservers.net
#
# This file is part of torcollect
#
# torcollect is free software: you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public License
# as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later
# version.
#
# torcollect is distributed in the hope that it will be
# useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
# PURPOSE. See the GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with torcollect.
# If not, see http://www.gnu.org/licenses/.
###########################################################

import torcollect.database
import torcollect.graphs
import datetime
import pygal
import StringIO

class MonthlyReport(object):
    """ Generates a Monthly report of the bridge-usage in HTML-format
    """
    _USAGE_STMNT = "SELECT COALESCE(TT.USAGE, 0) FROM \
                        (SELECT SUM(TRP_USERS) AS USAGE, REP_DATE \
                         FROM Report INNER JOIN TransportReport \
                            ON (REP_ID = TRP_REP_ID) \
                         WHERE REP_DATE >= %(start_date)s AND REP_DATE <= %(end_date)s \
                         GROUP BY REP_DATE) AS TT\
                    RIGHT JOIN \
                        (SELECT CURRENT_DATE + i AS DATE FROM GENERATE_SERIES( \
                            DATE %(start_date)s - CURRENT_DATE, \
                            DATE %(end_date)s - CURRENT_DATE) i) AS DT \
                    ON (DT.DATE = TT.REP_DATE) \
                    ORDER BY DT.DATE;"

    _TRFFC_STMNT = "SELECT COALESCE(TT.RECEIVED, 0), COALESCE(TT.SENT, 0) FROM \
                        (SELECT SUM(REP_TRAFFIC_RECEIVED) AS RECEIVED, \
                                SUM(REP_TRAFFIC_SENT) AS SENT, \
                                REP_DATE \
                         FROM Report \
                         WHERE REP_DATE >= %(start_date)s AND REP_DATE <= %(end_date)s \
                         GROUP BY REP_DATE) AS TT \
                    RIGHT JOIN \
                        (SELECT CURRENT_DATE + i AS DATE FROM GENERATE_SERIES( \
                            DATE %(start_date)s - CURRENT_DATE, \
                            DATE %(end_date)s - CURRENT_DATE) i) AS DT \
                    ON (DT.DATE = TT.REP_DATE) \
                    ORDER BY DT.DATE;"

    _CNMAP_STMNT = "SELECT TC.CCO_SHORT, COALESCE(TT.USAGE, 0) FROM \
                        (SELECT CCO_SHORT, SUM(CRP_USERS) AS USAGE \
                        FROM Countryreport INNER JOIN CountryCode \
                            ON (CCO_ID = CRP_CCO_ID) \
                        INNER JOIN Report \
                            ON (REP_ID = CRP_REP_ID) \
                        WHERE REP_DATE >= %(start_date)s AND REP_DATE <= %(end_date)s \
                        GROUP BY CCO_SHORT ) AS TT \
                    RIGHT JOIN CountryCode as TC \
                        ON (TT.CCO_SHORT = TC.CCO_SHORT) \
                    ORDER BY TC.CCO_SHORT ASC;"

    _CNHIS_STMNT = "SELECT CT.CCO_SHORT, RT.USAGE, CT.DATE FROM \
                        (SELECT CCO_SHORT, DT.DATE AS DATE FROM \
                            (SELECT CURRENT_DATE + i AS DATE FROM \
                                GENERATE_SERIES(DATE %(start_date)s - CURRENT_DATE, \
                                                DATE %(end_date)s - CURRENT_DATE) i) \
                            AS DT \
                            CROSS JOIN CountryCode) AS CT \
                        LEFT JOIN \
                        (SELECT CCO_SHORT, SUM(CRP_USERS) AS USAGE, REP_DATE  \
                         FROM CountryReport INNER JOIN CountryCode \
                            ON (CCO_ID = CRP_CCO_ID)  \
                        INNER JOIN Report \
                            ON (REP_ID = CRP_REP_ID) \
                        WHERE REP_DATE >= %(start_date)s  AND REP_DATE <= %(end_date)s \
                        GROUP BY CCO_SHORT, REP_DATE \
                        ORDER BY CCO_SHORT, REP_DATE ASC) \
                    AS RT ON (RT.CCO_SHORT = CT.CCO_SHORT AND RT.REP_DATE = CT.DATE);"

    _HTML_FRAME = """
    <!DOCTYPE html>
    <html>
    <head>
        <title>%(title)s</title>
        <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
        <link rel="stylesheet" href="/torcollect.css" type="text/css">
    </head>
    <body>
        <h1> %(title)s </title>
        <p> This report is concering data from %(start_date)s to %(end_date)s </p>
        %(overall_usage_graph)s
        %(overall_traffic_graph)s
        <h2> Distribution by Country </h2>
        %(worldmap)s
        %(country_graph)s
    </body>
    </html>
    """

    def __init__(self, year=None, month=None):
        """ The Monthly Report is initialized by giving it a Month it should represent """
        # if no date and year given, assume its this date and year
        if month is None or year is None:
            date = datetime.datetime.now()
            date.day = 1
        else:
            date = datetime.datetime(year, month, 1)

        self.start_date = date

        # determine end date
        self.days_of_month = (datetime.datetime(date.year, date.month+1, 1) - date).days
        self.end_date = datetime.datetime(date.year, date.month, self.days_of_month)

        # define structures to save data in
        self.usage_data = []
        self.traffic_sent = []
        self.traffic_received = []
        self.country_overall_data = {}
        self.country_usage_data = {}

        # proceed with gathering the needed data
        self.gather_data()

    def gather_data(self):
        """ Gathering the data needed to generate the report """
        self.check_validity()
        db = torcollect.database.Database()
        cur = db.cursor()

        cur.execute(MonthlyReport._USAGE_STMNT, {'start_date': self.start_date.isoformat(),
                                   'end_date'  : self.end_date.isoformat()})
        for dataset in cur.fetchall():
            self.usage_data.append(dataset[0])

        cur.execute(MonthlyReport._TRFFC_STMNT, {'start_date': self.start_date.isoformat(),
                                   'end_date'  : self.end_date.isoformat()})
        for dataset in cur.fetchall():
            self.traffic_sent.append(dataset[1])
            self.traffic_received.append(dataset[0])

        cur.execute(MonthlyReport._CNMAP_STMNT, {'start_date': self.start_date.isoformat(),
                                   'end_date'  : self.end_date.isoformat()})
        for dataset in cur.fetchall():
            self.country_overall_data[dataset[0].lower()] = dataset[1]

        cur.execute(MonthlyReport._CNHIS_STMNT, {'start_date': self.start_date.isoformat(),
                                   'end_date'  : self.end_date.isoformat()})
        for dataset in cur.fetchall():
            cc = dataset[0].lower()
            if not self.country_usage_data.has_key(cc):
                self.country_usage_data[cc] = []
            self.country_usage_data[cc].append(dataset[1])

    def render(self):
        overall_usage_graph = self.generate_overall_usage_graph(self.usage_data)
        overall_traffic_graph = self.generate_overall_traffic_graph(self.traffic_sent,
                                                               self.traffic_received)
        worldmap = self.generate_worldmap(self.country_overall_data)
        country_history_graph = self.generate_country_graph(self.country_usage_data)
        html = MonthlyReport._HTML_FRAME%{'overall_usage_graph': overall_usage_graph,
                                          'overall_traffic_graph': overall_traffic_graph,
                                          'worldmap': worldmap,
                                          'country_graph': country_history_graph,
                                          'title': "Torserver - Monthly Report",
                                          'start_date': self.start_date.isoformat(),
                                          'end_date': self.end_date.isoformat()}
        return html

    def generate_overall_usage_graph(self, usagedata):
        g = pygal.Line(fill=True, height=400)
        g.add('Usage',usagedata)
        g.x_labels = map(str, range(1,len(usagedata)+1))
        g.title = "Overall Usage"
        return torcollect.graphs.clean_graph(g.render())

    def generate_overall_traffic_graph(self, traffic_sent, traffic_received):
        g = pygal.StackedLine(fill=True, height=400)
        g.x_labels = map(str, range(1,len(traffic_sent)+1))
        g.add('Sent', [int(x or 0) for x in traffic_sent])
        g.add('Received', [int(x or 0) for x in traffic_received])
        g.title = "Overall Traffic"
        return torcollect.graphs.clean_graph(g.render())

    def generate_worldmap(self, worldmap_data):
        g = pygal.Worldmap()
        g.add('', worldmap_data)
        g.title = "Distribution by country"
        return torcollect.graphs.clean_graph(g.render())

    def generate_country_graph(self, usage_data):
        g = pygal.StackedLine(fill=True)
        g.x_labels = map(str, range(1,len(usage_data['us'])+1))
        for ccode, data in usage_data.items():
            g.add(ccode, data)
        return torcollect.graphs.clean_graph(g.render())

    def check_validity(self):
        """ Check if there is all data needed to generate the report """
        if self.end_date > datetime.datetime.now():
            difference = (self.end_date - datetime.datetime.now()).days
            raise Exception("This month is not finished yet. Try again in %d days"%difference)
<commit_msg>implemented organization-depended report generation<commit_after>#-*- coding: utf-8 -*-

##########################################################
# © 2013/2014 Daniel 'grindhold' Brendle with torservers.net
#
# This file is part of torcollect
#
# torcollect is free software: you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public License
# as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later
# version.
#
# torcollect is distributed in the hope that it will be
# useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
# PURPOSE. See the GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with torcollect.
# If not, see http://www.gnu.org/licenses/.
###########################################################

import torcollect.database
import torcollect.graphs
import datetime
import pygal
import StringIO

class MonthlyReport(object):
    """ Generates a Monthly report of the bridge-usage in HTML-format
    """
    _USAGE_STMNT = "SELECT COALESCE(TT.USAGE, 0) FROM \
                        (SELECT SUM(TRP_USERS) AS USAGE, REP_DATE \
                         FROM Report INNER JOIN TransportReport \
                            ON (REP_ID = TRP_REP_ID) \
                         WHERE REP_DATE >= %(start_date)s AND REP_DATE <= %(end_date)s \
                         GROUP BY REP_DATE) AS TT\
                    RIGHT JOIN \
                        (SELECT CURRENT_DATE + i AS DATE FROM GENERATE_SERIES( \
                            DATE %(start_date)s - CURRENT_DATE, \
                            DATE %(end_date)s - CURRENT_DATE) i) AS DT \
                    ON (DT.DATE = TT.REP_DATE) \
                    ORDER BY DT.DATE;"

    _TRFFC_STMNT = "SELECT COALESCE(TT.RECEIVED, 0), COALESCE(TT.SENT, 0) FROM \
                        (SELECT SUM(REP_TRAFFIC_RECEIVED) AS RECEIVED, \
                                SUM(REP_TRAFFIC_SENT) AS SENT, \
                                REP_DATE \
                         FROM Report \
                         WHERE REP_DATE >= %(start_date)s AND REP_DATE <= %(end_date)s \
                         GROUP BY REP_DATE) AS TT \
                    RIGHT JOIN \
                        (SELECT CURRENT_DATE + i AS DATE FROM GENERATE_SERIES( \
                            DATE %(start_date)s - CURRENT_DATE, \
                            DATE %(end_date)s - CURRENT_DATE) i) AS DT \
                    ON (DT.DATE = TT.REP_DATE) \
                    ORDER BY DT.DATE;"

    _CNMAP_STMNT = "SELECT TC.CCO_SHORT, COALESCE(TT.USAGE, 0) FROM \
                        (SELECT CCO_SHORT, SUM(CRP_USERS) AS USAGE \
                        FROM Countryreport INNER JOIN CountryCode \
                            ON (CCO_ID = CRP_CCO_ID) \
                        INNER JOIN Report \
                            ON (REP_ID = CRP_REP_ID) \
                        WHERE REP_DATE >= %(start_date)s AND REP_DATE <= %(end_date)s \
                        GROUP BY CCO_SHORT ) AS TT \
                    RIGHT JOIN CountryCode as TC \
                        ON (TT.CCO_SHORT = TC.CCO_SHORT) \
                    ORDER BY TC.CCO_SHORT ASC;"

    _CNHIS_STMNT = "SELECT CT.CCO_SHORT, RT.USAGE, CT.DATE FROM \
                        (SELECT CCO_SHORT, DT.DATE AS DATE FROM \
                            (SELECT CURRENT_DATE + i AS DATE FROM \
                                GENERATE_SERIES(DATE %(start_date)s - CURRENT_DATE, \
                                                DATE %(end_date)s - CURRENT_DATE) i) \
                            AS DT \
                            CROSS JOIN CountryCode) AS CT \
                        LEFT JOIN \
                        (SELECT CCO_SHORT, SUM(CRP_USERS) AS USAGE, REP_DATE  \
                         FROM CountryReport INNER JOIN CountryCode \
                            ON (CCO_ID = CRP_CCO_ID)  \
                        INNER JOIN Report \
                            ON (REP_ID = CRP_REP_ID) \
                        WHERE REP_DATE >= %(start_date)s  AND REP_DATE <= %(end_date)s \
                        GROUP BY CCO_SHORT, REP_DATE \
                        ORDER BY CCO_SHORT, REP_DATE ASC) \
                    AS RT ON (RT.CCO_SHORT = CT.CCO_SHORT AND RT.REP_DATE = CT.DATE);"

    _HTML_FRAME = """
    <!DOCTYPE html>
    <html>
    <head>
        <title>%(title)s</title>
        <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
        <link rel="stylesheet" href="/torcollect.css" type="text/css">
    </head>
    <body>
        <h1> %(title)s </title>
        <p> This report is concering data from %(start_date)s to %(end_date)s </p>
        %(overall_usage_graph)s
        %(overall_traffic_graph)s
        <h2> Distribution by Country </h2>
        %(worldmap)s
        %(country_graph)s
    </body>
    </html>
    """

    def __init__(self, year=None, month=None):
        """ The Monthly Report is initialized by giving it a Month it should represent """
        # if no date and year given, assume its this date and year
        if month is None or year is None:
            date = datetime.datetime.now()
            date.day = 1
        else:
            date = datetime.datetime(year, month, 1)

        self.start_date = date

        # determine end date
        self.days_of_month = (datetime.datetime(date.year, date.month+1, 1) - date).days
        self.end_date = datetime.datetime(date.year, date.month, self.days_of_month)

        # define structures to save data in
        self.usage_data = []
        self.traffic_sent = []
        self.traffic_received = []
        self.country_overall_data = {}
        self.country_usage_data = {}

        # proceed with gathering the needed data
        self.gather_data()

    def gather_data(self):
        """ Gathering the data needed to generate the report """
        self.check_validity()
        db = torcollect.database.Database()
        cur = db.cursor()

        cur.execute(MonthlyReport._USAGE_STMNT, {'start_date': self.start_date.isoformat(),
                                   'end_date'  : self.end_date.isoformat()})
        for dataset in cur.fetchall():
            self.usage_data.append(dataset[0])

        cur.execute(MonthlyReport._TRFFC_STMNT, {'start_date': self.start_date.isoformat(),
                                   'end_date'  : self.end_date.isoformat()})
        for dataset in cur.fetchall():
            self.traffic_sent.append(dataset[1])
            self.traffic_received.append(dataset[0])

        cur.execute(MonthlyReport._CNMAP_STMNT, {'start_date': self.start_date.isoformat(),
                                   'end_date'  : self.end_date.isoformat()})
        for dataset in cur.fetchall():
            self.country_overall_data[dataset[0].lower()] = dataset[1]

        cur.execute(MonthlyReport._CNHIS_STMNT, {'start_date': self.start_date.isoformat(),
                                   'end_date'  : self.end_date.isoformat()})
        for dataset in cur.fetchall():
            cc = dataset[0].lower()
            if not self.country_usage_data.has_key(cc):
                self.country_usage_data[cc] = []
            self.country_usage_data[cc].append(dataset[1])

    def render(self):
        overall_usage_graph = self.generate_overall_usage_graph(self.usage_data)
        overall_traffic_graph = self.generate_overall_traffic_graph(self.traffic_sent,
                                                               self.traffic_received)
        worldmap = self.generate_worldmap(self.country_overall_data)
        country_history_graph = self.generate_country_graph(self.country_usage_data)
        html = MonthlyReport._HTML_FRAME%{'overall_usage_graph': overall_usage_graph,
                                          'overall_traffic_graph': overall_traffic_graph,
                                          'worldmap': worldmap,
                                          'country_graph': country_history_graph,
                                          'title': "Torserver - Monthly Report",
                                          'start_date': self.start_date.isoformat(),
                                          'end_date': self.end_date.isoformat()}
        return html

    def generate_overall_usage_graph(self, usagedata):
        g = pygal.Line(fill=True, height=400)
        g.add('Usage',usagedata)
        g.x_labels = map(str, range(1,len(usagedata)+1))
        g.title = "Overall Usage"
        return torcollect.graphs.clean_graph(g.render())

    def generate_overall_traffic_graph(self, traffic_sent, traffic_received):
        g = pygal.StackedLine(fill=True, height=400)
        g.x_labels = map(str, range(1,len(traffic_sent)+1))
        g.add('Sent', [int(x or 0) for x in traffic_sent])
        g.add('Received', [int(x or 0) for x in traffic_received])
        g.title = "Overall Traffic"
        return torcollect.graphs.clean_graph(g.render())

    def generate_worldmap(self, worldmap_data):
        g = pygal.Worldmap()
        g.add('', worldmap_data)
        g.title = "Distribution by country"
        return torcollect.graphs.clean_graph(g.render())

    def generate_country_graph(self, usage_data):
        g = pygal.StackedLine(fill=True)
        g.x_labels = map(str, range(1,len(usage_data['us'])+1))
        for ccode, data in usage_data.items():
            g.add(ccode, data)
        return torcollect.graphs.clean_graph(g.render())

    def check_validity(self):
        """ Check if there is all data needed to generate the report """
        if self.end_date > datetime.datetime.now():
            difference = (self.end_date - datetime.datetime.now()).days
            raise Exception("This month is not finished yet. Try again in %d days"%difference)

class MonthlyOrganizationReport(MonthlyReport):
    """ This class modifies the behaviour of MonthlyReport to generate
        reports that only concern bridges that have only been disclosed to
        a specific organization
    """
    _USAGE_STMNT = "SELECT COALESCE(TT.USAGE, 0) FROM \
                        (SELECT SUM(TRP_USERS) AS USAGE, REP_DATE \
                         FROM Report INNER JOIN TransportReport \
                            ON (REP_ID = TRP_REP_ID) \
                         INNER JOIN Bridge \
                            ON (BRG_ID = REP_BRG_ID) \
                         INNER JOIN DisclosureTo \
                            ON (BRG_ID = DSC_BRG_ID) \
                         WHERE REP_DATE >= %(start_date)s AND REP_DATE <= %(end_date)s \
                            AND DSC_ORG_ID = %(org_id)d \
                         GROUP BY REP_DATE) AS TT\
                    RIGHT JOIN \
                        (SELECT CURRENT_DATE + i AS DATE FROM GENERATE_SERIES( \
                            DATE %(start_date)s - CURRENT_DATE, \
                            DATE %(end_date)s - CURRENT_DATE) i) AS DT \
                    ON (DT.DATE = TT.REP_DATE) \
                    ORDER BY DT.DATE;"

    _TRFFC_STMNT = "SELECT COALESCE(TT.RECEIVED, 0), COALESCE(TT.SENT, 0) FROM \
                        (SELECT SUM(REP_TRAFFIC_RECEIVED) AS RECEIVED, \
                                SUM(REP_TRAFFIC_SENT) AS SENT, \
                                REP_DATE \
                         FROM Report INNER JOIN Bridge \
                            ON (BRG_ID = REP_BRG_ID) \
                         INNER JOIN DisclosureTo \
                            ON (BRG_ID = DSC_BRG_ID) \
                         WHERE REP_DATE >= %(start_date)s AND REP_DATE <= %(end_date)s \
                            AND DSC_ORG_ID = %(org_id)d \
                         GROUP BY REP_DATE) AS TT \
                    RIGHT JOIN \
                        (SELECT CURRENT_DATE + i AS DATE FROM GENERATE_SERIES( \
                            DATE %(start_date)s - CURRENT_DATE, \
                            DATE %(end_date)s - CURRENT_DATE) i) AS DT \
                    ON (DT.DATE = TT.REP_DATE) \
                    ORDER BY DT.DATE;"

    _CNMAP_STMNT = "SELECT TC.CCO_SHORT, COALESCE(TT.USAGE, 0) FROM \
                        (SELECT CCO_SHORT, SUM(CRP_USERS) AS USAGE \
                        FROM Countryreport INNER JOIN CountryCode \
                            ON (CCO_ID = CRP_CCO_ID) \
                        INNER JOIN Report \
                            ON (REP_ID = CRP_REP_ID) \
                        INNER JOIN Bridge \
                            ON (BRG_ID = BRG_REP_ID) \
                        INNER JOIN DisclosureTo \
                            ON (BRG_ID = DSC_BRG_ID) \
                        WHERE REP_DATE >= %(start_date)s AND REP_DATE <= %(end_date)s \
                            AND DSC_ORG_ID = %(org_id)d \
                        GROUP BY CCO_SHORT ) AS TT \
                    RIGHT JOIN CountryCode as TC \
                        ON (TT.CCO_SHORT = TC.CCO_SHORT) \
                    ORDER BY TC.CCO_SHORT ASC;"

    _CNHIS_STMNT = "SELECT CT.CCO_SHORT, RT.USAGE, CT.DATE FROM \
                        (SELECT CCO_SHORT, DT.DATE AS DATE FROM \
                            (SELECT CURRENT_DATE + i AS DATE FROM \
                                GENERATE_SERIES(DATE %(start_date)s - CURRENT_DATE, \
                                                DATE %(end_date)s - CURRENT_DATE) i) \
                            AS DT \
                            CROSS JOIN CountryCode) AS CT \
                        LEFT JOIN \
                        (SELECT CCO_SHORT, SUM(CRP_USERS) AS USAGE, REP_DATE  \
                         FROM CountryReport INNER JOIN CountryCode \
                            ON (CCO_ID = CRP_CCO_ID)  \
                        INNER JOIN Report \
                            ON (REP_ID = CRP_REP_ID) \
                        INNER JOIN Bridge \
                            ON (BRG_ID = REP_BRG_ID) \
                        INNER JOIN DisclosureTo \
                            ON (BRG_ID = DSC_BRG_ID) \
                        WHERE REP_DATE >= %(start_date)s  AND REP_DATE <= %(end_date)s \
                            AND DSC_ORG_ID = %(org_id)d \
                        GROUP BY CCO_SHORT, REP_DATE \
                        ORDER BY CCO_SHORT, REP_DATE ASC) \
                    AS RT ON (RT.CCO_SHORT = CT.CCO_SHORT AND RT.REP_DATE = CT.DATE);"

    def __init__(self, organization, year=None, month=None):
        self.organization = organization
        MonthlyReport.__init__(self, year, month)

    def render(self):
        overall_usage_graph = self.generate_overall_usage_graph(self.usage_data)
        overall_traffic_graph = self.generate_overall_traffic_graph(self.traffic_sent,
                                                               self.traffic_received)
        worldmap = self.generate_worldmap(self.country_overall_data)
        country_history_graph = self.generate_country_graph(self.country_usage_data)
        title = "Torserver - Monthly Report for %s" % self.organization.get_name()
        html = MonthlyReport._HTML_FRAME%{'overall_usage_graph': overall_usage_graph,
                                          'overall_traffic_graph': overall_traffic_graph,
                                          'worldmap': worldmap,
                                          'country_graph': country_history_graph,
                                          'title': title,
                                          'start_date': self.start_date.isoformat(),
                                          'end_date': self.end_date.isoformat()}
        return html

    def gather_data(self):
        """ Gathering the data needed to generate the report """
        self.check_validity()
        db = torcollect.database.Database()
        cur = db.cursor()

        cur.execute(MonthlyReport._USAGE_STMNT, {'start_date': self.start_date.isoformat(),
                                   'end_date'  : self.end_date.isoformat(),
                                   'org_id'    : self.organization.get_id()})
        for dataset in cur.fetchall():
            self.usage_data.append(dataset[0])

        cur.execute(MonthlyReport._TRFFC_STMNT, {'start_date': self.start_date.isoformat(),
                                   'end_date'  : self.end_date.isoformat(),
                                   'org_id'    : self.organization.get_id()})
        for dataset in cur.fetchall():
            self.traffic_sent.append(dataset[1])
            self.traffic_received.append(dataset[0])

        cur.execute(MonthlyReport._CNMAP_STMNT, {'start_date': self.start_date.isoformat(),
                                   'end_date'  : self.end_date.isoformat(),
                                   'org_id'    : self.organization.get_id()})
        for dataset in cur.fetchall():
            self.country_overall_data[dataset[0].lower()] = dataset[1]

        cur.execute(MonthlyReport._CNHIS_STMNT, {'start_date': self.start_date.isoformat(),
                                   'end_date'  : self.end_date.isoformat(),
                                   'org_id'    : self.organization.get_id()})
        for dataset in cur.fetchall():
            cc = dataset[0].lower()
            if not self.country_usage_data.has_key(cc):
                self.country_usage_data[cc] = []
            self.country_usage_data[cc].append(dataset[1])

<|endoftext|>