<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Zoe Agent Manager - https://github.com/RMed/zoe_agent_manager
#
# Copyright (c) 2014 Rafael Medina García <rafamedgar@gmail.com>
#
# The MIT License (MIT)
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import configparser
import os
import shutil
import stat
import subprocess
import zoe
from zoe.deco import *

@Agent(name = "manager")
class AgentManager:

    @Message(tags = ["add"])
    def add(self, ag_name, ag_source):
        """ Add an agent to the repository. """
        rpath = os.path.join(os.environ["ZOE_HOME"], "etc", "agents_repo.conf")
        repo = configparser.ConfigParser()
        repo.read(rpath)

        # Check if already installed
        if ag_name in repo.sections():
            if repo[ag_name]["installed"] == "1":
                print("Agent %s is already installed" % ag_name)
                return

        repo.add_section(ag_name)
        repo[ag_name]["source"] = str(ag_source)
        repo[ag_name]["installed"] = "0"
        repo[ag_name]["version"] = ""

        with open(rpath, 'w') as repofile:
            repo.write(repofile)

        print("Added new agent %s to the repo" % ag_name)

    @Message(tags = ["install"])
    def install(self, ag_name):
        """ Install an agent from source. """
        rpath = os.path.join(os.environ["ZOE_HOME"], "etc", "agents_repo.conf")
        repo = configparser.ConfigParser()
        repo.read(rpath)

        # Check if already installed
        if ag_name in repo.sections():
            if repo[ag_name]["installed"] == "1":
                print("Agent %s is already installed" % ag_name)
                return
        else:
            print("Agent not found in repository")
            return

        temp = os.path.join(os.environ["ZOE_VAR"], "manager", ag_name)
        git_code = self._fetch_source(ag_name)

        if git_code != 0:
            print("Could not fetch source")
            return

        a_setup = configparser.ConfigParser()
        a_setup.read(os.path.join(temp, "setup.zoe"))

        # Move agent files
        for dest in a_setup["INSTALL"]:
            shutil.move(
                os.path.join(temp, a_setup["INSTALL"][dest]),
                os.path.join(os.environ["ZOE_HOME"], dest))

        # Add agent to the zoe.conf file
        conf_path = os.path.join(os.environ["ZOE_HOME"], "etc", "zoe.conf")
        zconf = configparser.ConfigParser()
        zconf.read(conf_path)

        ports = []
        for sec in zconf.sections():
            if 'port' in zconf[sec]:
                ports.append(int(zconf[sec]["port"]))
        ports = sorted(ports)

        free_port = ports[0]
        while free_port in ports:
            free_port += 1

        zconf.add_section("agent " + ag_name)
        zconf["agent " + ag_name]["port"] = str(free_port)
        with open(conf_path, 'w') as configfile:
            zconf.write(configfile)

        # Update agent repo
        repo[ag_name]["installed"] = "1"
        repo[ag_name]["version"] = a_setup["INFO"]["version"]

        with open(rpath, 'w') as repofile:
            repo.write(repofile)

        print("Agent %s installed correctly." % ag_name)

        # Launch the agent
        # TODO: downloaded files do not have
        #self.launch_agent(ag_name)

        # Cleanup
        shutil.rmtree(os.path.join(os.environ["ZOE_VAR"], "manager"))

    @Message(tags = ["launch"])
    def launch_agent(self, ag_name):
        """ Launch an agent. """
        agent_dir = os.path.join(os.environ["ZOE_HOME"], "agents", ag_name)
        if not os.path.isdir(agent_dir):
            print("Agent %s does not exist!" %ag_name)

        # Get executable script
        # Obtained from http://stackoverflow.com/a/8957768
        executable = stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH
        for filename in os.listdir(agent_dir):
            if os.path.isfile(filename):
                st = os.stat(filename)
                mode = st.st_mode
                if mode & executable:
                    script = filename

        print("Launching agent %s..." % ag_name)
        log_path = os.path.join(os.environ["ZOE_LOGS"], ag_name + ".log")
        log_file = open(log_path, "w+")
        proc = subprocess.Popen([os.path.join(agent_dir, script)], stdout=log_file, stderr=log_file)
        print("Launched agent %s with PID %i" % (ag_name, proc.pid))

        pid_path = os.path.join(os.environ["ZOE_VAR"], ag_name + ".pid")
        with open(pid_path, "w+") as pf:
            print(proc.pid, file=pf)

    @Message(tags = ["remove"])
    def remove(self, ag_name):
        """ Remove an agent from the agent repository. """
        rpath = os.path.join(os.environ["ZOE_HOME"], "etc", "agents_repo.conf")
        repo = configparser.ConfigParser()
        repo.read(rpath)

        # Check if installed
        if ag_name in repo.sections():
            if repo[ag_name]["installed"] == "0":
                print("Agent %s is not installed" % ag_name)
                return
        else:
            print("Cannot find agent %s in the repository" % ag_name)

    @Message(tags = ["stop"])
    def stop_agent(self, ag_name):
        """ Stop an agent's execution. """
        pid_path = os.path.join(os.environ["ZOE_VAR"], ag_name + ".pid")
        if not os.path.isfile(pid_path):
            print("Agent %s is not running" % ag_name)

        with open(pid_path, "r") as pf:
            pid = str(int(pf.read()))

        killed = subprocess.call(["kill", pid])
        if killed != 0:
            print("Oops, something happened while stopping %s" % ag_name)
            return

        print("Stopped agent %s" % ag_name)

    def _fetch_source(self, ag_name):
        """ Download the source of the agent to var/manager/ag_name. """
        temp = os.path.join(os.environ["ZOE_VAR"], "manager", ag_name)
        rpath = os.path.join(os.environ["ZOE_HOME"], "etc", "agents_repo.conf")
        repo = configparser.ConfigParser()
        repo.read(rpath)

        source = repo[ag_name]["source"]
        return subprocess.call(["git", "clone", source, temp])
<commit_msg>New agent starts successfully when installed<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Zoe Agent Manager - https://github.com/RMed/zoe_agent_manager
#
# Copyright (c) 2014 Rafael Medina García <rafamedgar@gmail.com>
#
# The MIT License (MIT)
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import configparser
import os
import shutil
import stat
import subprocess
import zoe
from zoe.deco import *

@Agent(name = "manager")
class AgentManager:

    @Message(tags = ["add"])
    def add(self, ag_name, ag_source):
        """ Add an agent to the repository. """
        rpath = os.path.join(os.environ["ZOE_HOME"], "etc", "agents_repo.conf")
        repo = configparser.ConfigParser()
        repo.read(rpath)

        # Check if already installed
        if ag_name in repo.sections():
            if repo[ag_name]["installed"] == "1":
                print("Agent %s is already installed" % ag_name)
                return

        repo.add_section(ag_name)
        repo[ag_name]["source"] = str(ag_source)
        repo[ag_name]["installed"] = "0"
        repo[ag_name]["version"] = ""

        with open(rpath, 'w') as repofile:
            repo.write(repofile)

        print("Added new agent %s to the repo" % ag_name)

    @Message(tags = ["install"])
    def install(self, ag_name):
        """ Install an agent from source. """
        rpath = os.path.join(os.environ["ZOE_HOME"], "etc", "agents_repo.conf")
        repo = configparser.ConfigParser()
        repo.read(rpath)

        # Check if already installed
        if ag_name in repo.sections():
            if repo[ag_name]["installed"] == "1":
                print("Agent %s is already installed" % ag_name)
                return
        else:
            print("Agent not found in repository")
            return

        temp = os.path.join(os.environ["ZOE_VAR"], "manager", ag_name)
        git_code = self._fetch_source(ag_name)

        if git_code != 0:
            print("Could not fetch source")
            return

        a_setup = configparser.ConfigParser()
        a_setup.read(os.path.join(temp, "setup.zoe"))

        # Move agent files
        for dest in a_setup["INSTALL"]:
            shutil.move(
                os.path.join(temp, a_setup["INSTALL"][dest]),
                os.path.join(os.environ["ZOE_HOME"], dest))

        # Make script executable
        script = os.path.join(os.environ["ZOE_HOME"], "agents",
            ag_name, a_setup["RUN"]["script"])
        st = os.stat(script)
        os.chmod(script, st.st_mode | stat.S_IEXEC)

        # Add agent to the zoe.conf file
        conf_path = os.path.join(os.environ["ZOE_HOME"], "etc", "zoe.conf")
        zconf = configparser.ConfigParser()
        zconf.read(conf_path)

        ports = []
        for sec in zconf.sections():
            if 'port' in zconf[sec]:
                ports.append(int(zconf[sec]["port"]))
        ports = sorted(ports)

        free_port = ports[0]
        while free_port in ports:
            free_port += 1

        zconf.add_section("agent " + ag_name)
        zconf["agent " + ag_name]["port"] = str(free_port)
        with open(conf_path, 'w') as configfile:
            zconf.write(configfile)

        # Update agent repo
        repo[ag_name]["installed"] = "1"
        repo[ag_name]["version"] = a_setup["INFO"]["version"]

        with open(rpath, 'w') as repofile:
            repo.write(repofile)

        print("Agent %s installed correctly." % ag_name)

        # Launch the agent
        self.launch_agent(ag_name, a_setup["RUN"]["script"])

        # Cleanup
        self._clean_temp()

    @Message(tags = ["launch"])
    def launch_agent(self, ag_name, script):
        """ Launch an agent. """
        agent_dir = os.path.join(os.environ["ZOE_HOME"], "agents", ag_name)
        if not os.path.isdir(agent_dir):
            print("Agent %s does not exist!" % ag_name)

        print("Launching agent %s..." % ag_name)
        log_path = os.path.join(os.environ["ZOE_LOGS"], ag_name + ".log")
        log_file = open(log_path, "w+")
        proc = subprocess.Popen([os.path.join(agent_dir, script)],
            stdout=log_file, stderr=log_file)
        print("Launched agent %s with PID %i" % (ag_name, proc.pid))

        pid_path = os.path.join(os.environ["ZOE_VAR"], ag_name + ".pid")
        with open(pid_path, "w+") as pf:
            print(proc.pid, file=pf)

    @Message(tags = ["remove"])
    def remove(self, ag_name):
        """ Remove an agent from the agent repository. """
        rpath = os.path.join(os.environ["ZOE_HOME"], "etc", "agents_repo.conf")
        repo = configparser.ConfigParser()
        repo.read(rpath)

        # Check if installed
        if ag_name in repo.sections():
            if repo[ag_name]["installed"] == "0":
                print("Agent %s is not installed" % ag_name)
                return
        else:
            print("Cannot find agent %s in the repository" % ag_name)

    @Message(tags = ["stop"])
    def stop_agent(self, ag_name):
        """ Stop an agent's execution. """
        pid_path = os.path.join(os.environ["ZOE_VAR"], ag_name + ".pid")
        if not os.path.isfile(pid_path):
            print("Agent %s is not running" % ag_name)

        with open(pid_path, "r") as pf:
            pid = str(int(pf.read()))

        killed = subprocess.call(["kill", pid])
        if killed != 0:
            print("Oops, something happened while stopping %s" % ag_name)
            return

        print("Stopped agent %s" % ag_name)

    def _clean_temp(self):
        """ Clean the temporal data stored in var/manager. """
        try:
            shutil.rmtree(os.path.join(os.environ["ZOE_VAR"], "manager"))
        except:
            pass

    def _fetch_source(self, ag_name):
        """ Download the source of the agent to var/manager/ag_name. """
        temp = os.path.join(os.environ["ZOE_VAR"], "manager", ag_name)
        rpath = os.path.join(os.environ["ZOE_HOME"], "etc", "agents_repo.conf")
        repo = configparser.ConfigParser()
        repo.read(rpath)

        source = repo[ag_name]["source"]
        return subprocess.call(["git", "clone", source, temp])
<|endoftext|>


<commit_before>import os
import urlparse
import smtplib
from werkzeug.wrappers import Request, Response
from werkzeug.routing import Map, Rule
from werkzeug.exceptions import HTTPException, NotFound
from werkzeug.wsgi import SharedDataMiddleware
from werkzeug.utils import redirect
from jinja2 import Environment, FileSystemLoader
from email.mime.text import MIMEText
from conf import EMAIL, TOKN, CEILING
from validate_email import validate_email
from datetime import datetime

#WSGI Application
class Forms(object):

    def __init__(self):
        global rater
        rater.increment_rate()
        template_path = os.path.join(os.path.dirname(__file__), 'templates')
        self.error = None
        self.jinja_env = Environment(loader=FileSystemLoader(template_path),
                                     autoescape=True)
        self.url_map = Map([Rule('/', endpoint='form_page')])

    def render_template(self, template_name, status, **context):
        t = self.jinja_env.get_template(template_name)
        return Response(t.render(context), mimetype='text/html', status=status)

    def dispatch_request(self, request):
        adapter = self.url_map.bind_to_environ(request.environ)
        try:
            endpoint, values = adapter.match()
            return getattr(self, 'on_' + endpoint)(request, **values)
        except HTTPException, e:
            return e

    def wsgi_app(self, environ, start_response):
        request = Request(environ)
        response = self.dispatch_request(request)
        return response(environ, start_response)

    def __call__(self, environ, start_response):
        return self.wsgi_app(environ, start_response)

    def send_email(self, msg):
        msg_send = MIMEText(str(msg))
        s = smtplib.SMTP('localhost')
        try:
            s.sendmail('theform', EMAIL, msg_send.as_string())
            s.quit()
        except:
            s.quit()

    def on_form_page(self, request):
        global rater
        self.error = None
        message = None
        status = 200
        if request.method == 'POST':
            if not is_valid_email(request):
                self.error = 'Invalid Email'
                message = None
                status = 400
            elif not validate_name(request):
                self.error = 'Invalid Name'
                message = None
                status = 400
            elif not is_hidden_field_empty(request) or not is_valid_token(request):
                self.error = 'Improper Form Submission'
                message = None
                status = 400
            elif rater.is_rate_violation():
                self.error = 'Too Many Requests'
                message = None
                status = 429
            else:
                message = create_msg(request)
                if message:
                    self.send_email(message)
                    return self.render_template('submitted.html',
                                                error=self.error,
                                                url=message,
                                                status=status)
        return self.render_template('index.html',
                                    error=self.error,
                                    url=message, status=status)

class RateLimiter(object):

    def __init__(self):
        self.rate = 0
        self.start_time = datetime.now()
        self.time_diff = 0

    def set_time_diff(self):
        time_d = datetime.now() - self.start_time
        self.time_diff = time_d.seconds

    def increment_rate(self):
        self.rate += 1;

    def reset_rate(self):
        self.rate = 0
        self.start_time = datetime.now()
        self.time_diff = 0

    def is_rate_violation(self):
        """
        False if rate does not violate CEILING in 1 second (no violation)
        and True otherwise (violation)
        """
        self.set_time_diff()
        if self.time_diff < 1 and self.rate > CEILING:
            self.reset_rate()
            return True
        elif self.time_diff > 1:
            self.reset_rate()
        return False

rater = RateLimiter();

# Standalone/helper functions
def create_app(with_static=True):
    app = Forms()
    if with_static:
        app.wsgi_app = SharedDataMiddleware(app.wsgi_app, {
            '/static':  os.path.join(os.path.dirname(__file__), 'static')
        })
    return app

def create_msg(request):
    message = dict()
    if request.method == 'POST':
        for key in request.form:
            message[key] = request.form[key]
        if message:
            return message
        return None
    return None

def is_valid_email(request):
    valid_email = validate_email(request.form['email'],
                                 check_mx=True,
                                 verify=True)
    if valid_email:
        return valid_email
    return False

def validate_name(request):
    name = request.form['name']
    if name.strip():
        return True
    return False

def is_hidden_field_empty(request):
    if request.form['hidden'] == "":
        return True
    return False

def is_valid_token(request):
    if request.form['tokn'] == TOKN:
        return True
    return False

# Application logic
if __name__ == '__main__':
    from werkzeug.serving import run_simple
    app = create_app()
    run_simple('127.0.0.1', 5000, app, use_debugger=True, use_reloader=True)
<commit_msg>Documented RateLimiter class<commit_after>import os
import urlparse
import smtplib
from werkzeug.wrappers import Request, Response
from werkzeug.routing import Map, Rule
from werkzeug.exceptions import HTTPException, NotFound
from werkzeug.wsgi import SharedDataMiddleware
from werkzeug.utils import redirect
from jinja2 import Environment, FileSystemLoader
from email.mime.text import MIMEText
from conf import EMAIL, TOKN, CEILING
from validate_email import validate_email
from datetime import datetime

#WSGI Application
class Forms(object):

    def __init__(self):
        global rater
        rater.increment_rate()
        template_path = os.path.join(os.path.dirname(__file__), 'templates')
        self.error = None
        self.jinja_env = Environment(loader=FileSystemLoader(template_path),
                                     autoescape=True)
        self.url_map = Map([Rule('/', endpoint='form_page')])

    def render_template(self, template_name, status, **context):
        t = self.jinja_env.get_template(template_name)
        return Response(t.render(context), mimetype='text/html', status=status)

    def dispatch_request(self, request):
        adapter = self.url_map.bind_to_environ(request.environ)
        try:
            endpoint, values = adapter.match()
            return getattr(self, 'on_' + endpoint)(request, **values)
        except HTTPException, e:
            return e

    def wsgi_app(self, environ, start_response):
        request = Request(environ)
        response = self.dispatch_request(request)
        return response(environ, start_response)

    def __call__(self, environ, start_response):
        return self.wsgi_app(environ, start_response)

    def send_email(self, msg):
        msg_send = MIMEText(str(msg))
        s = smtplib.SMTP('localhost')
        try:
            s.sendmail('theform', EMAIL, msg_send.as_string())
            s.quit()
        except:
            s.quit()

    def on_form_page(self, request):
        global rater
        self.error = None
        message = None
        status = 200
        if request.method == 'POST':
            if not is_valid_email(request):
                self.error = 'Invalid Email'
                message = None
                status = 400
            elif not validate_name(request):
                self.error = 'Invalid Name'
                message = None
                status = 400
            elif not is_hidden_field_empty(request) or not is_valid_token(request):
                self.error = 'Improper Form Submission'
                message = None
                status = 400
            elif rater.is_rate_violation():
                self.error = 'Too Many Requests'
                message = None
                status = 429
            else:
                message = create_msg(request)
                if message:
                    self.send_email(message)
                    return self.render_template('submitted.html',
                                                error=self.error,
                                                url=message,
                                                status=status)
        return self.render_template('index.html',
                                    error=self.error,
                                    url=message, status=status)

class RateLimiter(object):
    """Track number of form submissions per second

    __init__
    set_time_diff
    increment_rate
    reset_rate
    is_rate_violation
    """

    def __init__(self):
        self.rate = 0
        self.start_time = datetime.now()
        self.time_diff = 0

    def set_time_diff(self):
        # Sets time_diff in seconds
        time_d = datetime.now() - self.start_time
        self.time_diff = time_d.seconds

    def increment_rate(self):
        self.rate += 1;

    def reset_rate(self):
        # Reset rate to initial values
        self.rate = 0
        self.start_time = datetime.now()
        self.time_diff = 0

    def is_rate_violation(self):
        """
        False if rate does not violate CEILING in 1 second (no violation)
        and True otherwise (violation)
        """
        self.set_time_diff()
        if self.time_diff < 1 and self.rate > CEILING:
            self.reset_rate()
            return True
        elif self.time_diff > 1:
            self.reset_rate()
        return False

# Global RateLimiter object
rater = RateLimiter();

# Standalone/helper functions
def create_app(with_static=True):
    app = Forms()
    if with_static:
        app.wsgi_app = SharedDataMiddleware(app.wsgi_app, {
            '/static':  os.path.join(os.path.dirname(__file__), 'static')
        })
    return app

def create_msg(request):
    message = dict()
    if request.method == 'POST':
        for key in request.form:
            message[key] = request.form[key]
        if message:
            return message
        return None
    return None

def is_valid_email(request):
    valid_email = validate_email(request.form['email'],
                                 check_mx=True,
                                 verify=True)
    if valid_email:
        return valid_email
    return False

def validate_name(request):
    name = request.form['name']
    if name.strip():
        return True
    return False

def is_hidden_field_empty(request):
    if request.form['hidden'] == "":
        return True
    return False

def is_valid_token(request):
    if request.form['tokn'] == TOKN:
        return True
    return False

# Application logic
if __name__ == '__main__':
    from werkzeug.serving import run_simple
    app = create_app()
    run_simple('127.0.0.1', 5000, app, use_debugger=True, use_reloader=True)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
#@created: 09.10.2013
#@author: Aleksey Komissarov
#@contact: ad3002@gmail.com

import sys
sys.path.append("c:/Users/Master/Dropbox/workspace")
sys.path.append("/Users/akomissarov/Dropbox/workspace")
import Image, ImageDraw, ImageFont
import yaml

# fix for Windows-specific bug with png extension registering
from PIL.PngImagePlugin import _save, PngImageFile, _accept
Image.register_open("PNG", PngImageFile, _accept)
Image.register_save("PNG", _save)
Image.register_extension("PNG", ".png")
Image.register_mime("PNG", "image/png")

from PyChrDraw.draw_chromosome import *

if __name__ == '__main__':

        with open("human_chrs.yaml") as fh:
                settings = yaml.load(fh)

        chrs = settings["chromosome_sizes"]
        width = settings["canvas"]["width"]
        height = settings["canvas"]["height"]
        im = Image.new("RGBA", (width, height), "#ffffff")
        draw = ImageDraw.Draw(im)

        files = settings["orphans"]
        name2chr = settings["ncbi_names"]
        for file_name in files:
                bands2chr, max_value = load_bands_from_bed(file_name, last_column_color=True)
                print max_value
                new_bands2chr = {}
                for key in bands2chr:
                    new_key = key.split("|")[3]
                    if new_key in name2chr:
                        new_key = name2chr[new_key]
                        print key, "-->", new_key, "Size:", len(bands2chr[key])
                    else:
                        print "No key", new_key, "Size", len(bands2chr[key])
                new_bands2chr[new_key] = bands2chr[key]
                bands2chr = new_bands2chr
                exit()

        # x = settings["canvas"]["left_corner"]
        # y = settings["canvas"]["top_corner"]
        # keys = settings["chromosome_order"]
        # real_chr_size = settings["canvas"]["real_chr_size"]
        # space_between_chr = settings["canvas"]["space_between_chr"]
        # chr_width = settings["canvas"]["chr_width"]

        # mouse_chrs, bands2chr = normalize_chromosome_sizes(mouse_chrs, real_chr_size, bands=bands2chr)

        # for key in keys:
        #         length = mouse_chrs[key]
        #         draw_horizontal_chromosome(draw, x, y, length, bands=bands2chr[key], chr_width=chr_width, name=key, scale=1, stars=None)
        #         y += space_between_chr
        # im.save("output_file.png")<commit_msg>compute corrected coordinates<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
#@created: 09.10.2013
#@author: Aleksey Komissarov
#@contact: ad3002@gmail.com

import sys
sys.path.append("c:/Users/Master/Dropbox/workspace")
sys.path.append("/Users/akomissarov/Dropbox/workspace")
import Image, ImageDraw, ImageFont
import yaml

# fix for Windows-specific bug with png extension registering
from PIL.PngImagePlugin import _save, PngImageFile, _accept
Image.register_open("PNG", PngImageFile, _accept)
Image.register_save("PNG", _save)
Image.register_extension("PNG", ".png")
Image.register_mime("PNG", "image/png")

from PyChrDraw.draw_chromosome import *
from PyChrDraw.colors import *

if __name__ == '__main__':

    with open("human_chrs.yaml") as fh:
        settings = yaml.load(fh)

    chrs = settings["chromosome_sizes"]
    width = settings["canvas"]["width"]
    height = settings["canvas"]["height"]
    im = Image.new("RGBA", (width, height), "#ffffff")
    draw = ImageDraw.Draw(im)

    files = settings["orphans"]
    name2chr = settings["ncbi_names"]
    data_chrs = []
    max_value = 0.
    print "Load files..."
    for file_name in files:
        print "Load", file_name
        bands2chr, _max_value = load_bands_from_bed(file_name, last_column_color=True)
        if _max_value > max_value:
            max_value = float(_max_value)
        new_bands2chr = {}
        for key in bands2chr:
            new_key = key.split("|")[3]
            if new_key in name2chr:
                new_key = name2chr[new_key]
                # print key, "-->", new_key, "Size:", len(bands2chr[key])
                new_bands2chr[new_key] = bands2chr[key]
            else:
                pass
                # print "No key", new_key, "Size", len(bands2chr[key])
        data_chrs.append(new_bands2chr)

    x = settings["canvas"]["left_corner"]
    y = settings["canvas"]["top_corner"]
    keys = settings["chromosome_order"]
    real_chr_size = settings["canvas"]["real_chr_size"]
    space_between_chr = settings["canvas"]["space_between_chr"]
    chr_width = settings["canvas"]["chr_width"]

    print "Compute colors..."
    draw_data = []
    for i, bands2chr in enumerate(data_chrs):
        for key in bands2chr:
            for (start, length, color, border_color) in bands2chr[key]:
                c = get_color_by_map(color/max_value)
                bands2chr[key] = (start, length, c, c)
        ready_chrs, bands2chr = normalize_chromosome_sizes(chrs, real_chr_size, bands=bands2chr)
        draw_data.append(ready_chrs, bands2chr)
    print data_chrs[0]

    #

    # for key in keys:
    #         length = mouse_chrs[key]
    #         draw_horizontal_chromosome(draw, x, y, length, bands=bands2chr[key], chr_width=chr_width, name=key, scale=1, stars=None)
    #         y += space_between_chr
    # im.save("output_file.png")<|endoftext|>


<commit_before>from __future__ import print_function
from keras.models import Sequential, load_model
from keras.layers import Dense, Activation
from keras.layers import LSTM
from keras.optimizers import RMSprop
from keras.utils.data_utils import get_file
import numpy as np
import random
import sys
import string

import luigi

class TG_LSTM():
    # cut the text in semi-redundant sequences of maxlen characters
    maxlen = 40

    def __init__(self):
        # List of allowed charachters
        self.chars = sorted(list(set(string.ascii_lowercase + " '.,!\"")))
        self.char_indices = dict((c, i) for i, c in enumerate(self.chars))
        self.indices_char = dict((i, c) for i, c in enumerate(self.chars))

    def sample(self, preds, temperature=1.0):
        # helper function to sample an index from a probability array
        preds = np.asarray(preds).astype('float64')
        preds = np.log(preds) / temperature
        exp_preds = np.exp(preds)
        preds = exp_preds / np.sum(exp_preds)
        probas = np.random.multinomial(1, preds, 1)
        return np.argmax(probas)

    def train(self, txt_file_target):  # path='../data/pg/pg32040.txt'
        text = txt_file_target.open().read().lower()
        text = "".join([x if x in self.chars else " " for x in text])
        print('corpus length:', len(text))

        step = 3
        sentences = []
        next_chars = []
        for i in range(0, len(text) - self.maxlen, step):
            sentences.append(text[i: i + self.maxlen])
            next_chars.append(text[i + self.maxlen])
        print('nb sequences:', len(sentences))

        print('Vectorization...')
        X = np.zeros((len(sentences),
                      self.maxlen,
                      len(self.chars)),
                     dtype=np.bool)
        y = np.zeros((len(sentences), len(self.chars)), dtype=np.bool)
        for i, sentence in enumerate(sentences):
            for t, char in enumerate(sentence):
                X[i, t, self.char_indices[char]] = 1
            y[i, self.char_indices[next_chars[i]]] = 1

        # build the model: a single LSTM
        print('Build model...')
        self.model = Sequential()
        self.model.add(LSTM(128, input_shape=(self.maxlen, len(self.chars))))
        self.model.add(Dense(len(self.chars)))
        self.model.add(Activation('softmax'))

        optimizer = RMSprop(lr=0.01)
        self.model.compile(loss='categorical_crossentropy',
                           optimizer=optimizer)

        self.model.fit(X, y,
                       batch_size=128,
                       epochs=30)

    def generate(self, sentence, diversity=1.0):
        generated = ''
        generated += sentence
        print('----- Generating with seed: "' + sentence + '"')

        for i in range(400):
            x = np.zeros((1, self.maxlen, len(self.chars)))
            for t, char in enumerate(sentence):
                x[0, t, self.char_indices[char]] = 1.

            preds = self.model.predict(x, verbose=0)[0]
            next_index = self.sample(preds, diversity)
            next_char = self.indices_char[next_index]

            generated += next_char
            sentence = sentence + next_char
            if len(sentence) >= self.maxlen:
                sentence = sentence[-self.maxlen:]

        return generated

    def save(self, path='../data/models/lstm.model'):
        self.model.save(path)

    def load(self, path='../data/models/lstm.model'):
        self.model = load_model(path)

class InputFileOnLocal(luigi.ExternalTask):
    """
    This class represents file stored on Google Storage in advance.
    """
    path = luigi.Parameter()

    def output(self):
        """
        Returns the target output for this task.
        In this case, it expects a file to be present on Local disk.
        :return: the target output for this task.
        :rtype: object (:py:class:`luigi.target.Target`)
        """
        return luigi.LocalTarget(self.path)

class TrainLSTM(luigi.Task):

    # task_namespace = 'detect'
    path_txt = luigi.Parameter()
    epochs = luigi.IntParameter()
    epochs_step = luigi.IntParameter(10)

    def requires(self):
        """
        This task's dependencies:
        * :py:class:`~.InputFileOnStorage`
        We don't read this file, just make sure it exists.
        :return: list of object (:py:class:`luigi.task.Task`)
        """
        if self.epochs > self.epochs_step:
            return [TrainLSTM(path_txt=self.path_txt,
                              epochs=self.epochs - self.epochs_step,
                              epochs_step=self.epochs_step)]

    def output(self):
        """
        Returns the target output for this task.
        In this case, a successful execution of this task will create a file on
        the local filesystem.
        :return: the target output for this task.
        :rtype: object (:py:class:`luigi.target.Target`)
        """
        return luigi.LocalTarget(self.path_txt +
                                 "_" + str(self.epochs) +
                                 ".model")

    def run(self):
        """
        1. generate LSTM for given epoch_steps
        2. write the result into the :py:meth:`*(epoch).model` local target
        """
        lstm = TG_LSTM()

#lstm = TG_LSTM()
# lstm.train()
# print(lstm.generate('immunity'))

# lstm.save()

#lstm2 = TG_LSTM()
# lstm2.load()
# print(lstm2.generate('immunity'))

#print(lstm2.generate('immunity ' * 2))

#print(lstm2.generate('immunity ' * 4))
<commit_msg>LSTM woriking from command line<commit_after>from __future__ import print_function
from keras.models import Sequential, load_model
from keras.layers import Dense, Activation
from keras.layers import LSTM
from keras.optimizers import RMSprop
from keras.utils.data_utils import get_file
import numpy as np
import random
import sys
import string

import luigi

class TG_LSTM():
    # cut the text in semi-redundant sequences of maxlen characters
    maxlen = 40

    def __init__(self):
        # List of allowed charachters
        self.chars = sorted(list(set(string.ascii_lowercase + " '.,!\"")))
        self.char_indices = dict((c, i) for i, c in enumerate(self.chars))
        self.indices_char = dict((i, c) for i, c in enumerate(self.chars))

        self.model = Sequential()
        self.model.add(LSTM(128, input_shape=(self.maxlen, len(self.chars))))
        self.model.add(Dense(len(self.chars)))
        self.model.add(Activation('softmax'))

        optimizer = RMSprop(lr=0.01)
        self.model.compile(loss='categorical_crossentropy',
                           optimizer=optimizer)

    def sample(self, preds, temperature=1.0):
        # helper function to sample an index from a probability array
        preds = np.asarray(preds).astype('float64')
        preds = np.log(preds) / temperature
        exp_preds = np.exp(preds)
        preds = exp_preds / np.sum(exp_preds)
        probas = np.random.multinomial(1, preds, 1)
        return np.argmax(probas)

    def train(self, txt_file_target, epochs_num):
        text = open(txt_file_target).read().lower()
        text = "".join([x if x in self.chars else " " for x in text])
        print('corpus length:', len(text))

        step = 3
        sentences = []
        next_chars = []
        for i in range(0, len(text) - self.maxlen, step):
            sentences.append(text[i: i + self.maxlen])
            next_chars.append(text[i + self.maxlen])
        print('nb sequences:', len(sentences))

        print('Vectorization...')
        X = np.zeros((len(sentences),
                      self.maxlen,
                      len(self.chars)),
                     dtype=np.bool)
        y = np.zeros((len(sentences), len(self.chars)), dtype=np.bool)
        for i, sentence in enumerate(sentences):
            for t, char in enumerate(sentence):
                X[i, t, self.char_indices[char]] = 1
            y[i, self.char_indices[next_chars[i]]] = 1

        self.model.fit(X, y,
                       batch_size=128,
                       epochs=epochs_num)

    def generate(self, sentence, diversity=1.0):
        generated = ''
        generated += sentence
        print('----- Generating with seed: "' + sentence + '"')

        for i in range(400):
            x = np.zeros((1, self.maxlen, len(self.chars)))
            for t, char in enumerate(sentence):
                x[0, t, self.char_indices[char]] = 1.

            preds = self.model.predict(x, verbose=0)[0]
            next_index = self.sample(preds, diversity)
            next_char = self.indices_char[next_index]

            generated += next_char
            sentence = sentence + next_char
            if len(sentence) >= self.maxlen:
                sentence = sentence[-self.maxlen:]

        return generated

    def save(self, path='../data/models/lstm.model'):
        self.model.save(path)

    def load(self, path='../data/models/lstm.model'):
        self.model = load_model(path)

class InputFileOnLocal(luigi.ExternalTask):
    """
    This class represents file stored on Google Storage in advance.
    """
    path = luigi.Parameter()

    def output(self):
        """
        Returns the target output for this task.
        In this case, it expects a file to be present on Local disk.
        :return: the target output for this task.
        :rtype: object (:py:class:`luigi.target.Target`)
        """
        return luigi.LocalTarget(self.path)

class TrainLSTM(luigi.Task):

    # task_namespace = 'detect'
    path_txt = luigi.Parameter()
    epochs = luigi.IntParameter()
    epochs_step = luigi.IntParameter(1)

    def requires(self):
        """
        This task's dependencies:
        * :py:class:`~.InputFileOnStorage`
        We don't read this file, just make sure it exists.
        :return: list of object (:py:class:`luigi.task.Task`)
        """

        requirements = {"text": InputFileOnLocal(self.path_txt)}

        print
        if self.epochs > self.epochs_step:
            requirements["pre_model"] = TrainLSTM(path_txt=self.path_txt,
                                                  epochs=self.epochs -
                                                  self.epochs_step,
                                                  epochs_step=self.epochs_step)
        return requirements

    def output(self):
        """
        Returns the target output for this task.
        In this case, a successful execution of this task will create a file on
        the local filesystem.
        :return: the target output for this task.
        :rtype: object (:py:class:`luigi.target.Target`)
        """
        return luigi.LocalTarget(self.path_txt +
                                 "_" + str(self.epochs) +
                                 ".model")

    def run(self):
        """
        1. generate LSTM for given epoch_steps
        2. write the result into the :py:meth:`*(epoch).model` local target
        """
        lstm = TG_LSTM()

        if "pre_model" in self.input():
            lstm.load(self.input()["pre_model"].fn)

        lstm.train(txt_file_target=self.input()["text"].fn,
                   epochs_num=min(self.epochs, self.epochs_step))

        print(">>>>>>>>>>>>")
        print(lstm.generate('immunity '*2))
        print(">>>>>>>>>>>>")

        lstm.save(self.output().fn)

# lstm = TG_LSTM()
# lstm.train()
# print(lstm.generate('immunity'))

# lstm.save()

# lstm2 = TG_LSTM()
# lstm2.load()
# print(lstm2.generate('immunity'))

# print(lstm2.generate('immunity ' * 2))

# print(lstm2.generate('immunity ' * 4))

if __name__ == '__main__':
    luigi.run(['TrainLSTM',
               '--path-txt', '../data/pg/pg32040.txt',
               '--epochs', '3',
               '--workers', '1', '--local-scheduler'])
<|endoftext|>


<commit_before>#!/usr/bin/env python

__all__ = ['netease_download']

from ..common import *
from ..common import print_more_compatible as print
from ..util import fs
from json import loads
import hashlib
import base64
import os

def netease_hymn():
    return """
    player's Game Over,
    u can abandon.
    u get pissed,
    get pissed,
    Hallelujah my King!
    errr oh! fuck ohhh!!!!
    """

def netease_cloud_music_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
    rid = match1(url, r'\Wid=(.*)')
    if rid is None:
        rid = match1(url, r'/(\d+)/?')
    if "album" in url:
        j = loads(get_content("http://music.163.com/api/album/%s?id=%s&csrf_token=" % (rid, rid), headers={"Referer": "http://music.163.com/"}))

        artist_name = j['album']['artists'][0]['name']
        album_name = j['album']['name'].strip()
        new_dir = output_dir + '/' + fs.legitimize("%s - %s" % (artist_name, album_name))
        if not info_only:
            if not os.path.exists(new_dir):
                os.mkdir(new_dir)
            cover_url = j['album']['picUrl']
            download_urls([cover_url], "cover", "jpg", 0, new_dir)

        for i in j['album']['songs']:
            netease_song_download(i, output_dir=new_dir, info_only=info_only)
            try: # download lyrics
                assert kwargs['caption']
                l = loads(get_content("http://music.163.com/api/song/lyric/?id=%s&lv=-1&csrf_token=" % i['id'], headers={"Referer": "http://music.163.com/"}))
                netease_lyric_download(i, l["lrc"]["lyric"], output_dir=new_dir, info_only=info_only)
            except: pass

    elif "playlist" in url:
        j = loads(get_content("http://music.163.com/api/playlist/detail?id=%s&csrf_token=" % rid, headers={"Referer": "http://music.163.com/"}))

        new_dir = output_dir + '/' + fs.legitimize(j['result']['name'])
        if not info_only:
            if not os.path.exists(new_dir):
                os.mkdir(new_dir)
            cover_url = j['result']['coverImgUrl']
            download_urls([cover_url], "cover", "jpg", 0, new_dir)

        prefix_width = len(str(len(j['result']['tracks'])))
        for n, i in enumerate(j['result']['tracks']):
            playlist_prefix = '%%.%dd_' % prefix_width % n
            netease_song_download(i, output_dir=new_dir, info_only=info_only, playlist_prefix=playlist_prefix)
            try: # download lyrics
                assert kwargs['caption']
                l = loads(get_content("http://music.163.com/api/song/lyric/?id=%s&lv=-1&csrf_token=" % i['id'], headers={"Referer": "http://music.163.com/"}))
                netease_lyric_download(i, l["lrc"]["lyric"], output_dir=new_dir, info_only=info_only, playlist_prefix=playlist_prefix)
            except: pass

    elif "song" in url:
        j = loads(get_content("http://music.163.com/api/song/detail/?id=%s&ids=[%s]&csrf_token=" % (rid, rid), headers={"Referer": "http://music.163.com/"}))
        netease_song_download(j["songs"][0], output_dir=output_dir, info_only=info_only)
        try: # download lyrics
            assert kwargs['caption']
            l = loads(get_content("http://music.163.com/api/song/lyric/?id=%s&lv=-1&csrf_token=" % rid, headers={"Referer": "http://music.163.com/"}))
            netease_lyric_download(j["songs"][0], l["lrc"]["lyric"], output_dir=output_dir, info_only=info_only)
        except: pass

    elif "program" in url:
        j = loads(get_content("http://music.163.com/api/dj/program/detail/?id=%s&ids=[%s]&csrf_token=" % (rid, rid), headers={"Referer": "http://music.163.com/"}))
        netease_song_download(j["program"]["mainSong"], output_dir=output_dir, info_only=info_only)

    elif "radio" in url:
        j = loads(get_content("http://music.163.com/api/dj/program/byradio/?radioId=%s&ids=[%s]&csrf_token=" % (rid, rid), headers={"Referer": "http://music.163.com/"}))
        for i in j['programs']:
            netease_song_download(i["mainSong"],output_dir=output_dir, info_only=info_only)

    elif "mv" in url:
        j = loads(get_content("http://music.163.com/api/mv/detail/?id=%s&ids=[%s]&csrf_token=" % (rid, rid), headers={"Referer": "http://music.163.com/"}))
        netease_video_download(j['data'], output_dir=output_dir, info_only=info_only)

def netease_lyric_download(song, lyric, output_dir='.', info_only=False, playlist_prefix=""):
    if info_only: return

    title = "%s%s. %s" % (playlist_prefix, song['position'], song['name'])
    filename = '%s.lrc' % get_filename(title)
    print('Saving %s ...' % filename, end="", flush=True)
    with open(os.path.join(output_dir, filename),
              'w', encoding='utf-8') as x:
        x.write(lyric)
        print('Done.')

def netease_video_download(vinfo, output_dir='.', info_only=False):
    title = "%s - %s" % (vinfo['name'], vinfo['artistName'])
    url_best = sorted(vinfo["brs"].items(), reverse=True,
                      key=lambda x: int(x[0]))[0][1]
    netease_download_common(title, url_best,
                            output_dir=output_dir, info_only=info_only)

def netease_song_download(song, output_dir='.', info_only=False, playlist_prefix=""):
    title = "%s%s. %s" % (playlist_prefix, song['position'], song['name'])
    url_best = "http://music.163.com/song/media/outer/url?id=" + \
        str(song['id']) + ".mp3"
    '''
    songNet = 'p' + song['mp3Url'].split('/')[2][1:]

    if 'hMusic' in song and song['hMusic'] != None:
        url_best = make_url(songNet, song['hMusic']['dfsId'])
    elif 'mp3Url' in song:
        url_best = song['mp3Url']
    elif 'bMusic' in song:
        url_best = make_url(songNet, song['bMusic']['dfsId'])
    '''
    netease_download_common(title, url_best,
                            output_dir=output_dir, info_only=info_only)

def netease_download_common(title, url_best, output_dir, info_only):
    songtype, ext, size = url_info(url_best)
    print_info(site_info, title, songtype, size)
    if not info_only:
        download_urls([url_best], title, ext, size, output_dir)

def netease_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
    if "163.fm" in url:
        url = get_location(url)
    if "music.163.com" in url:
        netease_cloud_music_download(url, output_dir, merge, info_only, **kwargs)
    else:
        html = get_decoded_html(url)

        title = r1('movieDescription=\'([^\']+)\'', html) or r1('<title>(.+)</title>', html)

        if title[0] == ' ':
            title = title[1:]

        src = r1(r'<source src="([^"]+)"', html) or r1(r'<source type="[^"]+" src="([^"]+)"', html)

        if src:
            url = src
            _, ext, size = url_info(src)
            #sd_url = r1(r'(.+)-mobile.mp4', src) + ".flv"
            #hd_url = re.sub('/SD/', '/HD/', sd_url)

        else:
            url = (r1(r'["\'](.+)-list.m3u8["\']', html) or r1(r'["\'](.+).m3u8["\']', html)) + ".mp4"
            _, _, size = url_info(url)
            ext = 'mp4'

        print_info(site_info, title, ext, size)
        if not info_only:
            download_urls([url], title, ext, size, output_dir = output_dir, merge = merge)

def encrypted_id(dfsId):
    x = [ord(i[0]) for i in netease_hymn().split()]
    y = ''.join([chr(i - 61) if i > 96 else chr(i + 32) for i in x])
    byte1 = bytearray(y, encoding='ascii')
    byte2 = bytearray(str(dfsId), encoding='ascii')
    for i in range(len(byte2)):
        byte2[i] ^= byte1[i % len(byte1)]
    m = hashlib.md5()
    m.update(byte2)
    result = base64.b64encode(m.digest()).decode('ascii')
    result = result.replace('/', '_')
    result = result.replace('+', '-')
    return result

def make_url(songNet, dfsId):
    encId = encrypted_id(dfsId)
    mp3_url = "http://%s/%s/%s.mp3" % (songNet, encId, dfsId)
    return mp3_url

site_info = "163.com"
download = netease_download
download_playlist = playlist_not_supported('netease')
<commit_msg>add fake header<commit_after>#!/usr/bin/env python

__all__ = ['netease_download']

from ..common import *
from ..common import print_more_compatible as print
from ..util import fs
from json import loads
import hashlib
import base64
import os

def netease_hymn():
    return """
    player's Game Over,
    u can abandon.
    u get pissed,
    get pissed,
    Hallelujah my King!
    errr oh! fuck ohhh!!!!
    """

def netease_cloud_music_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
    rid = match1(url, r'\Wid=(.*)')
    if rid is None:
        rid = match1(url, r'/(\d+)/?')
    if "album" in url:
        j = loads(get_content("http://music.163.com/api/album/%s?id=%s&csrf_token=" % (rid, rid), headers={"Referer": "http://music.163.com/"}))

        artist_name = j['album']['artists'][0]['name']
        album_name = j['album']['name'].strip()
        new_dir = output_dir + '/' + fs.legitimize("%s - %s" % (artist_name, album_name))
        if not info_only:
            if not os.path.exists(new_dir):
                os.mkdir(new_dir)
            cover_url = j['album']['picUrl']
            download_urls([cover_url], "cover", "jpg", 0, new_dir)

        for i in j['album']['songs']:
            netease_song_download(i, output_dir=new_dir, info_only=info_only)
            try: # download lyrics
                assert kwargs['caption']
                l = loads(get_content("http://music.163.com/api/song/lyric/?id=%s&lv=-1&csrf_token=" % i['id'], headers={"Referer": "http://music.163.com/"}))
                netease_lyric_download(i, l["lrc"]["lyric"], output_dir=new_dir, info_only=info_only)
            except: pass

    elif "playlist" in url:
        j = loads(get_content("http://music.163.com/api/playlist/detail?id=%s&csrf_token=" % rid, headers={"Referer": "http://music.163.com/"}))

        new_dir = output_dir + '/' + fs.legitimize(j['result']['name'])
        if not info_only:
            if not os.path.exists(new_dir):
                os.mkdir(new_dir)
            cover_url = j['result']['coverImgUrl']
            download_urls([cover_url], "cover", "jpg", 0, new_dir)

        prefix_width = len(str(len(j['result']['tracks'])))
        for n, i in enumerate(j['result']['tracks']):
            playlist_prefix = '%%.%dd_' % prefix_width % n
            netease_song_download(i, output_dir=new_dir, info_only=info_only, playlist_prefix=playlist_prefix)
            try: # download lyrics
                assert kwargs['caption']
                l = loads(get_content("http://music.163.com/api/song/lyric/?id=%s&lv=-1&csrf_token=" % i['id'], headers={"Referer": "http://music.163.com/"}))
                netease_lyric_download(i, l["lrc"]["lyric"], output_dir=new_dir, info_only=info_only, playlist_prefix=playlist_prefix)
            except: pass

    elif "song" in url:
        j = loads(get_content("http://music.163.com/api/song/detail/?id=%s&ids=[%s]&csrf_token=" % (rid, rid), headers={"Referer": "http://music.163.com/"}))
        netease_song_download(j["songs"][0], output_dir=output_dir, info_only=info_only)
        try: # download lyrics
            assert kwargs['caption']
            l = loads(get_content("http://music.163.com/api/song/lyric/?id=%s&lv=-1&csrf_token=" % rid, headers={"Referer": "http://music.163.com/"}))
            netease_lyric_download(j["songs"][0], l["lrc"]["lyric"], output_dir=output_dir, info_only=info_only)
        except: pass

    elif "program" in url:
        j = loads(get_content("http://music.163.com/api/dj/program/detail/?id=%s&ids=[%s]&csrf_token=" % (rid, rid), headers={"Referer": "http://music.163.com/"}))
        netease_song_download(j["program"]["mainSong"], output_dir=output_dir, info_only=info_only)

    elif "radio" in url:
        j = loads(get_content("http://music.163.com/api/dj/program/byradio/?radioId=%s&ids=[%s]&csrf_token=" % (rid, rid), headers={"Referer": "http://music.163.com/"}))
        for i in j['programs']:
            netease_song_download(i["mainSong"],output_dir=output_dir, info_only=info_only)

    elif "mv" in url:
        j = loads(get_content("http://music.163.com/api/mv/detail/?id=%s&ids=[%s]&csrf_token=" % (rid, rid), headers={"Referer": "http://music.163.com/"}))
        netease_video_download(j['data'], output_dir=output_dir, info_only=info_only)

def netease_lyric_download(song, lyric, output_dir='.', info_only=False, playlist_prefix=""):
    if info_only: return

    title = "%s%s. %s" % (playlist_prefix, song['position'], song['name'])
    filename = '%s.lrc' % get_filename(title)
    print('Saving %s ...' % filename, end="", flush=True)
    with open(os.path.join(output_dir, filename),
              'w', encoding='utf-8') as x:
        x.write(lyric)
        print('Done.')

def netease_video_download(vinfo, output_dir='.', info_only=False):
    title = "%s - %s" % (vinfo['name'], vinfo['artistName'])
    url_best = sorted(vinfo["brs"].items(), reverse=True,
                      key=lambda x: int(x[0]))[0][1]
    netease_download_common(title, url_best,
                            output_dir=output_dir, info_only=info_only)

def netease_song_download(song, output_dir='.', info_only=False, playlist_prefix=""):
    title = "%s%s. %s" % (playlist_prefix, song['position'], song['name'])
    url_best = "http://music.163.com/song/media/outer/url?id=" + \
        str(song['id']) + ".mp3"
    '''
    songNet = 'p' + song['mp3Url'].split('/')[2][1:]

    if 'hMusic' in song and song['hMusic'] != None:
        url_best = make_url(songNet, song['hMusic']['dfsId'])
    elif 'mp3Url' in song:
        url_best = song['mp3Url']
    elif 'bMusic' in song:
        url_best = make_url(songNet, song['bMusic']['dfsId'])
    '''
    netease_download_common(title, url_best,
                            output_dir=output_dir, info_only=info_only)

def netease_download_common(title, url_best, output_dir, info_only):
    songtype, ext, size = url_info(url_best, faker=True)
    print_info(site_info, title, songtype, size)
    if not info_only:
        download_urls([url_best], title, ext, size, output_dir, faker=True)

def netease_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
    if "163.fm" in url:
        url = get_location(url)
    if "music.163.com" in url:
        netease_cloud_music_download(url, output_dir, merge, info_only, **kwargs)
    else:
        html = get_decoded_html(url)

        title = r1('movieDescription=\'([^\']+)\'', html) or r1('<title>(.+)</title>', html)

        if title[0] == ' ':
            title = title[1:]

        src = r1(r'<source src="([^"]+)"', html) or r1(r'<source type="[^"]+" src="([^"]+)"', html)

        if src:
            url = src
            _, ext, size = url_info(src)
            #sd_url = r1(r'(.+)-mobile.mp4', src) + ".flv"
            #hd_url = re.sub('/SD/', '/HD/', sd_url)

        else:
            url = (r1(r'["\'](.+)-list.m3u8["\']', html) or r1(r'["\'](.+).m3u8["\']', html)) + ".mp4"
            _, _, size = url_info(url)
            ext = 'mp4'

        print_info(site_info, title, ext, size)
        if not info_only:
            download_urls([url], title, ext, size, output_dir = output_dir, merge = merge)

def encrypted_id(dfsId):
    x = [ord(i[0]) for i in netease_hymn().split()]
    y = ''.join([chr(i - 61) if i > 96 else chr(i + 32) for i in x])
    byte1 = bytearray(y, encoding='ascii')
    byte2 = bytearray(str(dfsId), encoding='ascii')
    for i in range(len(byte2)):
        byte2[i] ^= byte1[i % len(byte1)]
    m = hashlib.md5()
    m.update(byte2)
    result = base64.b64encode(m.digest()).decode('ascii')
    result = result.replace('/', '_')
    result = result.replace('+', '-')
    return result

def make_url(songNet, dfsId):
    encId = encrypted_id(dfsId)
    mp3_url = "http://%s/%s/%s.mp3" % (songNet, encId, dfsId)
    return mp3_url

site_info = "163.com"
download = netease_download
download_playlist = playlist_not_supported('netease')
<|endoftext|>


<commit_before># Copyright (C) 2005-2006 Jelmer Vernooij <jelmer@samba.org>

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

from bzrlib.workingtree import WorkingTree
import bzrlib

import svn.core, svn.wc
from libsvn._core import SubversionException

class SvnWorkingTree(WorkingTree):
    """ Implementation of WorkingTree that uses a Subversion
    Working Copy for storage."""
    def __init__(self,path,branch):
        WorkingTree.__init__(self,path,branch)
        self.path = path
        self.wc = svn.wc.open_adm3(self.path)

    def revert(self,filenames,old_tree=None,backups=True):
        # FIXME: Respect old_tree and backups
        svn.wc.revert(filenames,True,self.wc,self.pool)

    def move(self, from_paths, to_name):
        revt = svn.core.svn_opt_revision_t()
        revt.kind = svn.core.svn_opt_revision_unspecified
        for entry in from_paths:
            svn.wc.move(entry, revt, to_name, False, self.wc, self.pool)

    def rename_one(self, from_rel, to_rel):
        # There is no difference between rename and move in SVN
        self.move([from_rel], to_rel)

    def add(self, files, ids=None):
        for f in files:
            svn.wc.add2(f, False, self.wc, self.pool)
            if ids:
                id = ids.pop()
                if id:
                    svn.wc.prop_set2('bzr:id', id, f, False, self.pool)
<commit_msg>Add WorkingTreeFormat implementation<commit_after># Copyright (C) 2005-2006 Jelmer Vernooij <jelmer@samba.org>

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

from bzrlib.workingtree import WorkingTree, WorkingTreeFormat
import bzrlib

import svn.core, svn.wc
from libsvn._core import SubversionException

class SvnWorkingTree(WorkingTree):
    """ Implementation of WorkingTree that uses a Subversion
    Working Copy for storage."""
    def __init__(self,path,branch):
        WorkingTree.__init__(self,path,branch)
        self.path = path
        self.wc = svn.wc.open_adm3(self.path)

    def revert(self,filenames,old_tree=None,backups=True):
        # FIXME: Respect old_tree and backups
        svn.wc.revert(filenames,True,self.wc,self.pool)

    def move(self, from_paths, to_name):
        revt = svn.core.svn_opt_revision_t()
        revt.kind = svn.core.svn_opt_revision_unspecified
        for entry in from_paths:
            svn.wc.move(entry, revt, to_name, False, self.wc, self.pool)

    def rename_one(self, from_rel, to_rel):
        # There is no difference between rename and move in SVN
        self.move([from_rel], to_rel)

    def add(self, files, ids=None):
        for f in files:
            svn.wc.add2(f, False, self.wc, self.pool)
            if ids:
                id = ids.pop()
                if id:
                    svn.wc.prop_set2('bzr:id', id, f, False, self.pool)

class SvnWorkingTreeFormat(WorkingTreeFormat):
    def get_format_description(self):
        return "Subversion Working Copy"

    def initialize(self, a_bzrdir, revision_id=None):
        # FIXME
        raise NotImplementedError(self.initialize)

    def open(self, a_bzrdir):
        # FIXME
        raise NotImplementedError(self.initialize)
<|endoftext|>


<commit_before># Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import collections
import datetime
import json
import operator
import time

import flask

from stackalytics.dashboard import decorators
from stackalytics.dashboard import helpers
from stackalytics.dashboard import parameters
from stackalytics.dashboard import vault
from stackalytics.processor import utils

DEFAULT_DAYS_COUNT = 7
FIRST_MEMBER_DATE = "2012-Jul-18"

blueprint = flask.Blueprint('reports', __name__, url_prefix='/report')

@blueprint.route('/blueprint/<module>/<blueprint_name>')
@decorators.templated()
@decorators.exception_handler()
def blueprint_summary(module, blueprint_name):
    blueprint_id = utils.get_blueprint_id(module, blueprint_name)
    bpd = vault.get_memory_storage().get_record_by_primary_key(
        'bpd:' + blueprint_id)
    if not bpd:
        flask.abort(404)
        return

    bpd = helpers.extend_record(bpd)
    record_ids = vault.get_memory_storage().get_record_ids_by_blueprint_ids(
        [blueprint_id])
    activity = [helpers.extend_record(record) for record in
                vault.get_memory_storage().get_records(record_ids)]
    activity.sort(key=lambda x: x['date'], reverse=True)

    return {'blueprint': bpd, 'activity': activity}

def _get_day(timestamp, time_now):
    return int((time_now - timestamp) / 60 / 60 / 24)

def _process_stat(data, key, time_now):
    if not data:
        return None

    data = sorted(data, key=operator.itemgetter(key))

    days = _get_day(data[0][key], time_now)
    chart_data = [0] * (days + 1)
    sum_ages = 0
    for review in data:
        age = time_now - review[key]
        sum_ages += age
        review[key + '_age'] = utils.make_age_string(age)
        chart_data[_get_day(review[key], time_now)] += 1

    return {
        'reviews': data,
        'average': utils.make_age_string(sum_ages / len(data)),
        'max': data[0][key + '_age'],
        'chart_data': json.dumps(chart_data),
    }

@blueprint.route('/reviews/<module>/open')
@decorators.templated()
@decorators.exception_handler()
def open_reviews(module):
    memory_storage_inst = vault.get_memory_storage()
    time_now = int(time.time())

    module_id_index = vault.get_vault()['module_id_index']
    module = module.lower()
    if module not in module_id_index:
        flask.abort(404)

    modules = module_id_index[module]['modules']

    review_ids = (memory_storage_inst.get_record_ids_by_modules(modules) &
                  memory_storage_inst.get_record_ids_by_types(['review']))

    waiting_on_reviewer = []
    waiting_on_submitter = []
    total_open = 0

    for review in memory_storage_inst.get_records(review_ids):
        if review.status == 'NEW':
            total_open += 1

            # review.value is minimum from votes made for the latest patch
            if review.value in [1, 2]:
                # CI or engineer liked this change request, waiting for someone
                # to merge or to put dislike
                waiting_on_reviewer.append(helpers.extend_record(review))
            elif review.value in [-1, -2]:
                # CI or reviewer do not like this, waiting for submitter to fix
                waiting_on_submitter.append(helpers.extend_record(review))
            else:
                # new requests without votes, waiting for CI
                pass

    return {
        'module': module,
        'total_open': total_open,
        'waiting_on_reviewer': len(waiting_on_reviewer),
        'waiting_on_submitter': len(waiting_on_submitter),
        'waiting_on_ci': (total_open - len(waiting_on_reviewer) -
                          len(waiting_on_submitter)),
        'reviewer_latest_revision': _process_stat(
            waiting_on_reviewer, 'updated_on', time_now),
        'reviewer_first_revision': _process_stat(
            waiting_on_reviewer, 'date', time_now),
        'submitter_latest_revision': _process_stat(
            waiting_on_submitter, 'updated_on', time_now),
        'submitter_first_revision': _process_stat(
            waiting_on_submitter, 'date', time_now),
    }

@blueprint.route('/contribution/<module>/<days>')
@decorators.templated()
@decorators.exception_handler()
def contribution(module, days):
    return {
        'module': module,
        'days': days,
        'start_date': int(time.time()) - int(days) * 24 * 60 * 60
    }

@blueprint.route('/ci/<module>/<days>')
@decorators.templated()
@decorators.exception_handler()
def external_ci(module, days):
    if int(days) > 100:
        days = 100

    return {
        'module': module,
        'days': days,
        'start_date': int(time.time()) - int(days) * 24 * 60 * 60
    }

@blueprint.route('/members')
@decorators.exception_handler()
@decorators.templated()
def members():
    days = int(flask.request.args.get('days') or DEFAULT_DAYS_COUNT)
    all_days = int(time.time() - utils.date_to_timestamp_ext(
        FIRST_MEMBER_DATE)) / (24 * 60 * 60) + 1

    return {
        'days': days,
        'all_days': all_days
    }

@blueprint.route('/affiliation_changes')
@decorators.exception_handler()
@decorators.templated()
def affiliation_changes():
    start_days = str(flask.request.args.get('start_days') or
                     utils.timestamp_to_date(int(time.time()) -
                                             365 * 24 * 60 * 60))
    end_days = str(flask.request.args.get('end_days') or
                   utils.timestamp_to_date(int(time.time())))

    return {
        'start_days': start_days,
        'end_days': end_days,
    }

@blueprint.route('/cores')
@decorators.exception_handler()
@decorators.templated()
def cores():
    project_type = parameters.get_single_parameter({}, 'project_type')
    return {
        'project_type': project_type,
    }

def _get_punch_card_data(records):
    punch_card_raw = []  # matrix days x hours
    for wday in range(7):
        punch_card_raw.append([0] * 24)
    for record in records:
        tt = datetime.datetime.fromtimestamp(record.date).timetuple()
        punch_card_raw[tt.tm_wday][tt.tm_hour] += 1

    punch_card_data = []  # format for jqplot bubble renderer
    for wday in range(7):
        for hour in range(24):
            v = punch_card_raw[wday][hour]
            if v:
                punch_card_data.append([hour, 6 - wday, v, v])  # upside down

    # add corner point, otherwise chart doesn't know the bounds
    if punch_card_raw[0][0] == 0:
        punch_card_data.append([0, 0, 0, 0])
    if punch_card_raw[6][23] == 0:
        punch_card_data.append([23, 6, 0, 0])

    return json.dumps(punch_card_data)

def _get_activity_summary(record_ids):
    memory_storage_inst = vault.get_memory_storage()

    record_ids_by_type = memory_storage_inst.get_record_ids_by_types(
        ['mark', 'patch', 'email', 'bpd', 'bpc', 'ci'])

    record_ids &= record_ids_by_type
    punch_card_data = _get_punch_card_data(
        memory_storage_inst.get_records(record_ids))

    return {
        'punch_card_data': punch_card_data,
    }

@blueprint.route('/users/<user_id>')
@decorators.templated()
@decorators.exception_handler()
def user_activity(user_id):
    user = vault.get_user_from_runtime_storage(user_id)
    if not user:
        flask.abort(404)
    user = helpers.extend_user(user)

    memory_storage_inst = vault.get_memory_storage()
    result = _get_activity_summary(
        memory_storage_inst.get_record_ids_by_user_ids([user_id]))
    result['user'] = user

    return result

@blueprint.route('/companies/<company>')
@decorators.templated()
@decorators.exception_handler()
def company_activity(company):
    memory_storage_inst = vault.get_memory_storage()
    original_name = memory_storage_inst.get_original_company_name(company)

    result = _get_activity_summary(
        memory_storage_inst.get_record_ids_by_companies([original_name]))
    result['company_name'] = original_name

    return result

@blueprint.route('/record/<path:record_id>')
@decorators.templated()
@decorators.exception_handler()
def record(record_id):
    memory_storage_inst = vault.get_memory_storage()
    record_obj = memory_storage_inst.get_record_by_primary_key(record_id)
    if not record_obj:
        flask.abort(404)

    result = dict(record=helpers.get_activity([record_obj], 0, 1)[0])
    return result

@blueprint.route('/activity')
@decorators.templated()
@decorators.exception_handler()
def activity():
    pass

@blueprint.route('/large_commits')
@decorators.response()
@decorators.jsonify('commits')
@decorators.exception_handler()
@decorators.record_filter()
def get_commit_report(records, **kwargs):
    loc_threshold = int(flask.request.args.get('loc_threshold') or 1000)
    response = []
    for record in records:
        if record.record_type == 'commit' and record.loc > loc_threshold:
            ext_record = vault.extend_record(record)
            nr = dict([(k, ext_record[k])
                       for k in ['loc', 'subject', 'module', 'primary_key',
                                 'change_id']
                       if k in ext_record])
            response.append(nr)
    return response

@blueprint.route('/single_plus_two_reviews')
@decorators.response()
@decorators.jsonify()
@decorators.exception_handler()
@decorators.record_filter(ignore='metric')
def get_single_plus_two_reviews_report(records, **kwargs):
    memory_storage_inst = vault.get_memory_storage()
    plus_twos = collections.defaultdict(list)
    for record in records:
        if record['record_type'] != 'mark':
            continue

        if (record['branch'] == 'master' and
                record['type'] == 'Code-Review' and record['value'] == +2):
            review_id = record['review_id']
            review = memory_storage_inst.get_record_by_primary_key(review_id)
            if review and review['status'] == 'MERGED':
                plus_twos[review_id].append(record)

    response = []
    for review_id in plus_twos.keys():
        if len(plus_twos[review_id]) < 2:
            mark = plus_twos[review_id][0]
            review = memory_storage_inst.get_record_by_primary_key(
                mark['review_id'])
            response.append({'review_by': review['user_id'],
                             'mark_by': mark['user_id'],
                             'subject': review['subject'],
                             'url': review['url'],
                             'project': review['project']})

    return response

@blueprint.route('/driverlog')
@decorators.templated()
@decorators.exception_handler()
def driverlog():
    pass
<commit_msg>Py3 compat: convert division result<commit_after># Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import collections
import datetime
import json
import operator
import time

import flask

from stackalytics.dashboard import decorators
from stackalytics.dashboard import helpers
from stackalytics.dashboard import parameters
from stackalytics.dashboard import vault
from stackalytics.processor import utils

DEFAULT_DAYS_COUNT = 7
FIRST_MEMBER_DATE = "2012-Jul-18"

blueprint = flask.Blueprint('reports', __name__, url_prefix='/report')

@blueprint.route('/blueprint/<module>/<blueprint_name>')
@decorators.templated()
@decorators.exception_handler()
def blueprint_summary(module, blueprint_name):
    blueprint_id = utils.get_blueprint_id(module, blueprint_name)
    bpd = vault.get_memory_storage().get_record_by_primary_key(
        'bpd:' + blueprint_id)
    if not bpd:
        flask.abort(404)
        return

    bpd = helpers.extend_record(bpd)
    record_ids = vault.get_memory_storage().get_record_ids_by_blueprint_ids(
        [blueprint_id])
    activity = [helpers.extend_record(record) for record in
                vault.get_memory_storage().get_records(record_ids)]
    activity.sort(key=lambda x: x['date'], reverse=True)

    return {'blueprint': bpd, 'activity': activity}

def _get_day(timestamp, time_now):
    return int((time_now - timestamp) / 60 / 60 / 24)

def _process_stat(data, key, time_now):
    if not data:
        return None

    data = sorted(data, key=operator.itemgetter(key))

    days = _get_day(data[0][key], time_now)
    chart_data = [0] * (days + 1)
    sum_ages = 0
    for review in data:
        age = time_now - review[key]
        sum_ages += age
        review[key + '_age'] = utils.make_age_string(age)
        chart_data[_get_day(review[key], time_now)] += 1

    return {
        'reviews': data,
        'average': utils.make_age_string(sum_ages / len(data)),
        'max': data[0][key + '_age'],
        'chart_data': json.dumps(chart_data),
    }

@blueprint.route('/reviews/<module>/open')
@decorators.templated()
@decorators.exception_handler()
def open_reviews(module):
    memory_storage_inst = vault.get_memory_storage()
    time_now = int(time.time())

    module_id_index = vault.get_vault()['module_id_index']
    module = module.lower()
    if module not in module_id_index:
        flask.abort(404)

    modules = module_id_index[module]['modules']

    review_ids = (memory_storage_inst.get_record_ids_by_modules(modules) &
                  memory_storage_inst.get_record_ids_by_types(['review']))

    waiting_on_reviewer = []
    waiting_on_submitter = []
    total_open = 0

    for review in memory_storage_inst.get_records(review_ids):
        if review.status == 'NEW':
            total_open += 1

            # review.value is minimum from votes made for the latest patch
            if review.value in [1, 2]:
                # CI or engineer liked this change request, waiting for someone
                # to merge or to put dislike
                waiting_on_reviewer.append(helpers.extend_record(review))
            elif review.value in [-1, -2]:
                # CI or reviewer do not like this, waiting for submitter to fix
                waiting_on_submitter.append(helpers.extend_record(review))
            else:
                # new requests without votes, waiting for CI
                pass

    return {
        'module': module,
        'total_open': total_open,
        'waiting_on_reviewer': len(waiting_on_reviewer),
        'waiting_on_submitter': len(waiting_on_submitter),
        'waiting_on_ci': (total_open - len(waiting_on_reviewer) -
                          len(waiting_on_submitter)),
        'reviewer_latest_revision': _process_stat(
            waiting_on_reviewer, 'updated_on', time_now),
        'reviewer_first_revision': _process_stat(
            waiting_on_reviewer, 'date', time_now),
        'submitter_latest_revision': _process_stat(
            waiting_on_submitter, 'updated_on', time_now),
        'submitter_first_revision': _process_stat(
            waiting_on_submitter, 'date', time_now),
    }

@blueprint.route('/contribution/<module>/<days>')
@decorators.templated()
@decorators.exception_handler()
def contribution(module, days):
    return {
        'module': module,
        'days': days,
        'start_date': int(time.time()) - int(days) * 24 * 60 * 60
    }

@blueprint.route('/ci/<module>/<days>')
@decorators.templated()
@decorators.exception_handler()
def external_ci(module, days):
    if int(days) > 100:
        days = 100

    return {
        'module': module,
        'days': days,
        'start_date': int(time.time()) - int(days) * 24 * 60 * 60
    }

@blueprint.route('/members')
@decorators.exception_handler()
@decorators.templated()
def members():
    days = int(flask.request.args.get('days') or DEFAULT_DAYS_COUNT)
    all_days = int((time.time() - utils.date_to_timestamp_ext(
        FIRST_MEMBER_DATE)) / (24 * 60 * 60)) + 1

    return {
        'days': days,
        'all_days': all_days
    }

@blueprint.route('/affiliation_changes')
@decorators.exception_handler()
@decorators.templated()
def affiliation_changes():
    start_days = str(flask.request.args.get('start_days') or
                     utils.timestamp_to_date(int(time.time()) -
                                             365 * 24 * 60 * 60))
    end_days = str(flask.request.args.get('end_days') or
                   utils.timestamp_to_date(int(time.time())))

    return {
        'start_days': start_days,
        'end_days': end_days,
    }

@blueprint.route('/cores')
@decorators.exception_handler()
@decorators.templated()
def cores():
    project_type = parameters.get_single_parameter({}, 'project_type')
    return {
        'project_type': project_type,
    }

def _get_punch_card_data(records):
    punch_card_raw = []  # matrix days x hours
    for wday in range(7):
        punch_card_raw.append([0] * 24)
    for record in records:
        tt = datetime.datetime.fromtimestamp(record.date).timetuple()
        punch_card_raw[tt.tm_wday][tt.tm_hour] += 1

    punch_card_data = []  # format for jqplot bubble renderer
    for wday in range(7):
        for hour in range(24):
            v = punch_card_raw[wday][hour]
            if v:
                punch_card_data.append([hour, 6 - wday, v, v])  # upside down

    # add corner point, otherwise chart doesn't know the bounds
    if punch_card_raw[0][0] == 0:
        punch_card_data.append([0, 0, 0, 0])
    if punch_card_raw[6][23] == 0:
        punch_card_data.append([23, 6, 0, 0])

    return json.dumps(punch_card_data)

def _get_activity_summary(record_ids):
    memory_storage_inst = vault.get_memory_storage()

    record_ids_by_type = memory_storage_inst.get_record_ids_by_types(
        ['mark', 'patch', 'email', 'bpd', 'bpc', 'ci'])

    record_ids &= record_ids_by_type
    punch_card_data = _get_punch_card_data(
        memory_storage_inst.get_records(record_ids))

    return {
        'punch_card_data': punch_card_data,
    }

@blueprint.route('/users/<user_id>')
@decorators.templated()
@decorators.exception_handler()
def user_activity(user_id):
    user = vault.get_user_from_runtime_storage(user_id)
    if not user:
        flask.abort(404)
    user = helpers.extend_user(user)

    memory_storage_inst = vault.get_memory_storage()
    result = _get_activity_summary(
        memory_storage_inst.get_record_ids_by_user_ids([user_id]))
    result['user'] = user

    return result

@blueprint.route('/companies/<company>')
@decorators.templated()
@decorators.exception_handler()
def company_activity(company):
    memory_storage_inst = vault.get_memory_storage()
    original_name = memory_storage_inst.get_original_company_name(company)

    result = _get_activity_summary(
        memory_storage_inst.get_record_ids_by_companies([original_name]))
    result['company_name'] = original_name

    return result

@blueprint.route('/record/<path:record_id>')
@decorators.templated()
@decorators.exception_handler()
def record(record_id):
    memory_storage_inst = vault.get_memory_storage()
    record_obj = memory_storage_inst.get_record_by_primary_key(record_id)
    if not record_obj:
        flask.abort(404)

    result = dict(record=helpers.get_activity([record_obj], 0, 1)[0])
    return result

@blueprint.route('/activity')
@decorators.templated()
@decorators.exception_handler()
def activity():
    pass

@blueprint.route('/large_commits')
@decorators.response()
@decorators.jsonify('commits')
@decorators.exception_handler()
@decorators.record_filter()
def get_commit_report(records, **kwargs):
    loc_threshold = int(flask.request.args.get('loc_threshold') or 1000)
    response = []
    for record in records:
        if record.record_type == 'commit' and record.loc > loc_threshold:
            ext_record = vault.extend_record(record)
            nr = dict([(k, ext_record[k])
                       for k in ['loc', 'subject', 'module', 'primary_key',
                                 'change_id']
                       if k in ext_record])
            response.append(nr)
    return response

@blueprint.route('/single_plus_two_reviews')
@decorators.response()
@decorators.jsonify()
@decorators.exception_handler()
@decorators.record_filter(ignore='metric')
def get_single_plus_two_reviews_report(records, **kwargs):
    memory_storage_inst = vault.get_memory_storage()
    plus_twos = collections.defaultdict(list)
    for record in records:
        if record['record_type'] != 'mark':
            continue

        if (record['branch'] == 'master' and
                record['type'] == 'Code-Review' and record['value'] == +2):
            review_id = record['review_id']
            review = memory_storage_inst.get_record_by_primary_key(review_id)
            if review and review['status'] == 'MERGED':
                plus_twos[review_id].append(record)

    response = []
    for review_id in plus_twos.keys():
        if len(plus_twos[review_id]) < 2:
            mark = plus_twos[review_id][0]
            review = memory_storage_inst.get_record_by_primary_key(
                mark['review_id'])
            response.append({'review_by': review['user_id'],
                             'mark_by': mark['user_id'],
                             'subject': review['subject'],
                             'url': review['url'],
                             'project': review['project']})

    return response

@blueprint.route('/driverlog')
@decorators.templated()
@decorators.exception_handler()
def driverlog():
    pass
<|endoftext|>