<commit_before>import socket
import mpd
import ssl
import os
import urllib2
import thread
import time

MPD_HOST = "127.0.0.1"
MPD_PORT = 6600

HOST = "irc.rizon.net"
HOME_CHANNEL = "#JiyuuRadio"
NICK = "JiyuuRadio"
PORT = 9999
SSL = True

MUSIC_PATH = "/media/music/Music"

mpc = mpd.MPDClient()
mpc.connect(MPD_HOST, MPD_PORT)

if(SSL):
    s = ssl.wrap_socket(socket.socket( ))
else:
    s = socket.socket( )
s.connect((HOST, PORT))
s.send("USER " + NICK + " " + NICK + " " + NICK + " :" + NICK + "\n")
s.send("NICK " + NICK + "\r\n")
s.send("JOIN " + HOME_CHANNEL + "\r\n")

if not os.path.exists(os.path.join(MUSIC_PATH, NICK+"_downloaded_music")):
    os.mkdir(os.path.join(MUSIC_PATH, NICK+"_downloaded_music"))

def parse_command(command, what):
    try:
        mapped = command[:command.index(" ")]
    except ValueError:
        mapped = command
    COMMAND_MAPPING[mapped](command)

def cmd_play(command):
    mpc.play()

def cmd_next(command):
    mpc.next()

def cmd_download(command):
    try:
        args = command[command.index(" ") + 1 : ]
        return_output("Fetching track " + args)
        dl = urllib2.urlopen(args)
        fnameline=""
        for line in str(dl.info()).split("\r\n"):
            if "filename=" in line:
                fnameline = line
                break
        if dl.info().maintype == "audio":
            fname = args[args.rindex("/") + 1 : ]
            if not fnameline == "":
                fname = fnameline[fnameline.index("filename=") + 10 : fnameline.rindex("\"")]
            track = open(os.path.join(os.path.join(MUSIC_PATH, NICK + "_downloaded_music"), fname), "wb")
            track.write(dl.read())
            track.close()
            mpc.update()
            time.sleep(5)
            mpc.searchadd("file", fname)
            return_output(fname + " fetched and queued")
        else:
            return_output("Mime type " + dl.info().type + " not allowed.")
    except Exception, e:
        return_output(e)
        return_output("I dun goofed, soz aye m80.")

def cmd_add(command):
    args = command[4 : ]
    mpc.searchadd("any", args)

def cmd_current(command):
    return_output(mpc.currentsong())

def cmd_queue(command):
    queue = "Next 4 tracks:\n"
    for track in mpc.playlist()[ : 4]:
        queue += str(track)+"\n"
    return_output(queue)

def cmd_stats(command):
    return_output(mpc.stats())

def cmd_help(command):
    return_output("Available commands: .add .play .next .current .queue .stats .download")

    try:
        args = command[5:]
        if args.startswith("."):
            args = args[1:]

        if args == "add":
            return_output(".add [search term] - queues tracks for playback")
        elif args == "play":
            return_output(".play - starts playing (useful for when the stream dies due to lack of queued tracks")
        elif args == "next":
            return_output(".next - next track")
        elif args == "current":
            return_output(".current - prints infor about current track")
        elif args == "queue":
            return_output(".queue - shows next 4 songs")
        elif args == "stats":
            return_output(".stats - shows stats")
        elif args == "download" or args == "dl":
            return_output("<.download | .dl> [url] - downloads track from URL and queues it")
    except:
        pass

def return_output(text):
    #hopefully fixes issue where the first line of a message is sometimes dropped
    s.send("PRIVMSG " + HOME_CHANNEL + " :\r\n")
    #can only send 5 lines at a time on Rizon before being kicked for flood
    text=str(text)
    for msg in text.split("\n"):
        s.send("PRIVMSG " + HOME_CHANNEL + " :" + str(msg) + "\r\n")

COMMAND_MAPPING = {"dl": cmd_download, "download": cmd_download, "add": cmd_add, "play": cmd_play, "next": cmd_next, "current": cmd_current, "queue": cmd_queue, "stats": cmd_stats, "help": cmd_help}

print "*** Connecting... ***"
while 1:
    line = s.recv(2048)
    line = line.strip("\r\n")
    if("End of /NAMES list." in line):
	print "\n*** Connected! ***\n"
	break
    else:
	print line

while 1:
    line = s.recv(2048)
    line = line.strip("\r\n")
    print line
    if "PING" in line:
        s.send("PONG :" + line[6 : ])
    elif "PRIVMSG" in line and not "NOTICE" in line and HOME_CHANNEL in line:
        command = line[line.rindex(HOME_CHANNEL + " :") + len(HOME_CHANNEL) + 2 : ]
        if "*" in command:
            return_output("NOPE")
        elif command.startswith("."):
            thread.start_new_thread(parse_command, (command[1 : ], ""))
<commit_msg>Added dict mapping for help<commit_after>import socket
import mpd
import ssl
import os
import urllib2
import threading
import time
import sys

MPD_HOST = "127.0.0.1"
MPD_PORT = 6600

HOST = "irc.rizon.net"
HOME_CHANNEL = "#JiyuuRadio"
NICK = "JiyuuRadio"
PORT = 9999
SSL = True

MUSIC_PATH = "/media/music/Music"

HELP_DICT = {"add": ".add [search term] - queues tracks for playback", "play": ".play - starts playing (useful for when the stream dies due to lack of queued tracks", "next": ".next - next track", "current": ".current - prints infor about current track", "queue": ".queue - shows next 4 songs", "stats": ".stats - shows stats", "download": "<.download | .dl> [url] - downloads track from URL and queues it", "dl": "<.download | .dl> [url] - downloads track from URL and queues it"}

mpc = mpd.MPDClient()
mpc.connect(MPD_HOST, MPD_PORT)

if(SSL):
    s = ssl.wrap_socket(socket.socket( ))
else:
    s = socket.socket( )
s.connect((HOST, PORT))
s.send("USER " + NICK + " " + NICK + " " + NICK + " :" + NICK + "\n")
s.send("NICK " + NICK + "\r\n")
s.send("JOIN " + HOME_CHANNEL + "\r\n")

if not os.path.exists(os.path.join(MUSIC_PATH, NICK+"_downloaded_music")):
    os.mkdir(os.path.join(MUSIC_PATH, NICK+"_downloaded_music"))

def reconnect_mpd():
    try:
        mpc.disconnect()
    except mpd.ConnectionError:
        pass
    mpc.connect(MPD_HOST, MPD_PORT)

def cmd_play(command):
        mpc.play()

def cmd_next(command):
        mpc.next()

def cmd_download(command):
    try:
        args = command[command.index(" ") + 1 : ]
        return_output("Fetching track " + args)
        dl = urllib2.urlopen(args)
        fnameline=""
        for line in str(dl.info()).split("\r\n"):
            if "filename=" in line:
                fnameline = line
                break
        if dl.info().maintype == "audio":
            fname = args[args.rindex("/") + 1 : ]
            if not fnameline == "":
                fname = fnameline[fnameline.index("filename=") + 10 : fnameline.rindex("\"")]
            track = open(os.path.join(os.path.join(MUSIC_PATH, NICK + "_downloaded_music"), fname), "wb")
            track.write(dl.read())
            track.close()
            mpc.update()
            time.sleep(5)
            mpc.searchadd("file", fname)
            return_output(fname + " fetched and queued")
        else:
            return_output("Mime type " + dl.info().type + " not allowed.")
    except Exception, e:
        return_output(e)
        return_output("I dun goofed, soz aye m80.")

def cmd_add(command):
    args = command[4 : ]
    mpc.searchadd("any", args)

def cmd_current(command):
    song_info = ""
    return_output(mpc.currentsong())

def cmd_queue(command):
    queue = mpc.playlist()
    queuestr = "Next 4 of %s tracks:\n" % len(queue)
    for track in queue[ : 4]:
        queuestr += str(track)+"\n"
    return_output(queuestr)

def cmd_stats(command):
    return_output(mpc.stats())

def cmd_help(command):
    return_output("Available commands: .add .play .next .current .queue .stats .download")
    try:
        args = command[5:]
        if args.startswith("."):
            args = args[1:]
        return_output(HELP_DICT[args])
    except:
        pass

def return_output(text):
    #hopefully fixes issue where the first line of a message is sometimes dropped
    s.send("PRIVMSG " + HOME_CHANNEL + " :\r\n")
    #can only send 5 lines at a time on Rizon before being kicked for flood
    text=str(text)
    for msg in text.split("\n"):
        s.send("PRIVMSG " + HOME_CHANNEL + " :" + str(msg) + "\r\n")

def parse_command(command):
    try:
        mapped = command[:command.index(" ")]
    except ValueError:
        mapped = command
    try:
        COMMAND_MAPPING[mapped](command)
    except (KeyError, mpd.ConnectionError) as e:
        if type(e) == mpd.ConnectionError:
            reconnect_mpd()
            COMMAND_MAPPING[mapped](command)
        else:
            pass

COMMAND_MAPPING = {"dl": cmd_download, "download": cmd_download, "add": cmd_add, "play": cmd_play, "next": cmd_next, "current": cmd_current, "queue": cmd_queue, "stats": cmd_stats, "help": cmd_help}

print "*** Connecting... ***"
while 1:
    line = s.recv(2048)
    line = line.strip("\r\n")
    if("End of /NAMES list." in line):
	print "\n*** Connected! ***\n"
	break
    else:
	print line

while 1:
    try:
        line = s.recv(2048)
        line = line.strip("\r\n")
        print line
        if "PING" in line:
            s.send("PONG :" + line[6 : ])
        elif "PRIVMSG" in line and not "NOTICE" in line and HOME_CHANNEL in line:
            command = line[line.rindex(HOME_CHANNEL + " :") + len(HOME_CHANNEL) + 2 : ]
            if "*" in command:
                return_output("NOPE")
            elif command.startswith("."):
                t = threading.Thread(target = parse_command, args=(command[1 : ],))
                t.daemon = 1
                t.start()
    except KeyboardInterrupt:
        sys.exit()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import numpy as np
import torch

import matplotlib
import matplotlib.pyplot as plt
from torch.autograd import Variable

from itertools import chain
import copy
import atexit
import sys
import argparse

class PhnSpkGenerator():
    def __init__(self, mu, cov, phn, spk):
        self._mu = mu
        self._cov = cov
        self._phn = phn
        self._spk = spk

    def generate(self, N):
        X = np.random.multivariate_normal(self._mu, self._cov, size=N)
        phn = torch.ones(N) * self._phn
        spk = torch.ones(N) * self._spk
        return torch.from_numpy(X).float(), phn, spk

class Plotter():
    def __init__(self):
        atexit.register(plt.show)
        self._cmap = matplotlib.colors.ListedColormap([
            (1, 0, 0),
            (0, 1, 0),
            (0, 0, 1)
        ])

    def plot(self, X, phn, spk, name="fig", transform = lambda x:x):
        plt.figure(name)

        for i, m in enumerate(['o', '+', 'x']):
            mask = (spk.numpy() == i)
            spk_set = X.numpy()[mask]
            spk_set = Variable(torch.from_numpy(spk_set).float())
            spk_set = transform(spk_set).data.numpy()
            plt.scatter(spk_set[:,0], spk_set[:,1],
                        c=t_phn.numpy()[mask], cmap=self._cmap, marker=m)
        plt.show(block=False)

def epoch(fwd, params, X, target, batch_size=16, shuffle=True, train=True):
    N = X.size()[0]
    assert target.size()[0] == N

    train_X = X
    train_t = target
    if shuffle:
        p = np.random.permutation(N)
        train_X = X.numpy()[p]
        train_t = target.numpy()[p]

    train_X = torch.from_numpy(train_X)
    train_t = torch.from_numpy(train_t)

    nb_batches = N // batch_size
    total = nb_batches * batch_size
    total_loss = 0.0
    correct = 0

    criterion = torch.nn.NLLLoss()
    optim = torch.optim.SGD(params, lr=1e-3)

    for i in range(nb_batches):
        batch_X = Variable(train_X[i*batch_size:(i+1)*batch_size])
        batch_t = Variable(train_t[i*batch_size:(i+1)*batch_size])

        y = fwd(batch_X)
        loss = criterion(y, batch_t)
        _, preds = y.max(dim=1)
        correct += (preds == batch_t).sum().data[0]

        total_loss += loss.data[0]

        if train:
            optim.zero_grad()
            loss.backward()
            optim.step()

    return total_loss/nb_batches, correct/total

def multi_target_epoch(common, decoders, params, X, targets, batch_size=16, shuffle=True, train=True):
    assert len(decoders) == len(targets)
    N = X.size()[0]
    for t in targets:
        assert t.size()[0] == N

    # TODO: clone?
    train_X = X
    train_targets = targets
    if shuffle:
        p = np.random.permutation(N)
        train_X = X.numpy()[p]
        train_X = torch.from_numpy(train_X)
        for i, t in enumerate(train_targets):
            train_targets[i] = torch.from_numpy(t.numpy()[p])

    nb_batches = N // batch_size
    total = nb_batches * batch_size
    cum_losses = []
    nb_correct = []
    for i in range(len(decoders)):
        cum_losses.append(0.0)
        nb_correct.append(0)

    criterion = torch.nn.NLLLoss()
    optim = torch.optim.SGD(params, lr=1e-3)

    for i in range(nb_batches):
        batch_X = Variable(train_X[i*batch_size:(i+1)*batch_size])
        batch_ts = []
        for t in train_targets:
            batch_ts.append(Variable(t[i*batch_size:(i+1)*batch_size]))

        repre = common(batch_X)

        losses = []
        for i, (dec, t) in enumerate(zip(decoders, batch_ts)):
            y = dec(repre)
            loss = criterion(y, t)
            losses.append(loss)
            _, preds = y.max(dim=1)
            nb_correct[i] += (preds == t).sum().data[0]
            cum_losses[i] += loss.data[0]

        if train:
            complete_loss = sum(losses)
            optim.zero_grad()
            complete_loss.backward()
            optim.step()

    return [cl/nb_batches for cl in cum_losses], [c/total for c in nb_correct]

def instantiate_generators():
    phn_mus = []
    phn_mus.append(np.asarray([1,1]))
    phn_mus.append(np.asarray([3.5,2]))
    phn_mus.append(np.asarray([2,4]))

    phn_covs = []
    phn_covs.append(np.asarray([[0.2,0], [0,0.2]]))
    phn_covs.append(np.asarray([[0.2,0], [0,0.2]]))
    phn_covs.append(np.asarray([[0.2,0], [0,0.2]]))

    spk_mus = []
    spk_mus.append(np.asarray([-0.7, 0.7]))
    spk_mus.append(np.asarray([1, 0]))
    spk_mus.append(np.asarray([-0.7, -0.7]))

    gens = []
    for phn, (phn_mu, phn_cov) in enumerate(zip(phn_mus, phn_covs)):
        for spk, spk_mu in enumerate(spk_mus):
            gens.append(PhnSpkGenerator(phn_mu+spk_mu, phn_cov, phn, spk))

    return gens

def generate(gens, N_per_cluster):
    X = torch.zeros((0, 2))
    t_phn = torch.zeros((0,))
    t_spk = torch.zeros((0,))

    for g in gens:
        X_g, phn_g, spk_g = g.generate(100)
        X = torch.cat([X, X_g], 0)
        t_phn = torch.cat([t_phn, phn_g], 0)
        t_spk = torch.cat([t_spk, spk_g], 0)
    t_phn = t_phn.long()
    t_spk = t_spk.long()

    return X, t_phn, t_spk

def train(common, decoders, params, X, ts, nb_epochs, report_interval=25):
    params = list(params)
    for i in range(nb_epochs):
        ce, acc = multi_target_epoch(common, decoders, params, X, ts)
        val_ce, val_acc = multi_target_epoch(common, decoders, params, X, ts, train=False)

        if i % report_interval == report_interval - 1:
            string = "{:>3} phn CE: {:.3f}, phn Acc: {:2.2f} | Valid: CE {:.3f} Acc {:.3f}".format(
                i, ce[0], 100.0*acc[0], val_ce[0], 100.0*val_acc[0]
            )
            print(string)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("--nb-epochs", type=int, default=200,
                        help="number of training epochs")
    args = parser.parse_args()

    gens = instantiate_generators()

    X, t_phn, t_spk = generate(gens, 100)
    X_val, t_phn_val, t_spk_val = generate(gens, 100)

    plotter = Plotter()
    #plotter.plot(X, t_phn, t_spk, name="Raw data")

    bn_extractor = torch.nn.Sequential(
        torch.nn.Linear(2, 10),
        torch.nn.ReLU(),
        torch.nn.Linear(10, 10),
        torch.nn.ReLU(),
        torch.nn.Linear(10, 2),
    )

    bn_backup = copy.deepcopy(bn_extractor)

    plotter.plot(X, t_phn, t_spk, name="BN features, random init", transform=bn_extractor)

    phn_decoder = torch.nn.Sequential(
        torch.nn.Linear(2,10),
        torch.nn.ReLU(),
        torch.nn.Linear(10,3),
        torch.nn.LogSoftmax()
    )
    phn_backup = copy.deepcopy(phn_decoder)

    print("Training PHN network")
    for i in range(args.nb_epochs):
        ce, acc = multi_target_epoch(bn_extractor, [phn_decoder],
            chain(bn_extractor.parameters(), phn_decoder.parameters()), X, [t_phn])
        val_ce, val_acc = multi_target_epoch(bn_extractor, [phn_decoder],
            chain(bn_extractor.parameters(), phn_decoder.parameters()), X, [t_phn], train=False)

        if i % 25 == 25 - 1:
            string = "{:>3} phn CE: {:.3f}, phn Acc: {:2.2f} | Valid: CE {:.3f} Acc {:.3f}".format(
                i, ce[0], 100.0*acc[0], val_ce[0], 100.0*val_acc[0]
            )
            print(string)
    # train(bn_extractor, [phn_decoder],
    #       chain(bn_extractor.parameters(), phn_decoder.parameters()),
    #       X, [t_phn], args.nb_epochs)

    plotter.plot(X, t_phn, t_spk, name="BN features, PHN optimized", transform=bn_extractor)

    spk_decoder = torch.nn.Sequential(
        torch.nn.Linear(2,10),
        torch.nn.ReLU(),
        torch.nn.Linear(10,3),
        torch.nn.LogSoftmax()
    )
    spk_backup = copy.deepcopy(spk_decoder)

    print("Training SPK decoder")
    for i in range(args.nb_epochs):
        ce, acc = multi_target_epoch(bn_extractor, [spk_decoder],
            spk_decoder.parameters(), X, [t_spk])
        val_ce, val_acc = multi_target_epoch(bn_extractor, [spk_decoder],
            spk_decoder.parameters(), X, [t_spk], train=False)

        if i % 25 == 25 - 1:
            string = "{:>3} phn CE: {:.3f}, phn Acc: {:2.2f} | Valid: CE {:.3f} Acc {:.3f}".format(
                i, ce[0], 100.0*acc[0], val_ce[0], 100.0*val_acc[0]
            )
            print(string)
    # train(bn_extractor, [spk_decoder],
    #       spk_decoder.parameters(), X, [t_spk], args.nb_epochs)

    print("Training jointly, from same init:")
    for i in range(args.nb_epochs):
        ces, accs= multi_target_epoch(
            bn_backup, [phn_backup, spk_backup],
            chain(bn_backup.parameters(), phn_backup.parameters(), spk_backup.parameters()),
            X, [t_phn, t_spk]
        )
        if i % 25 == 24:
            string = "{:>3} phn CE: {:.3f}, phn Acc: {:2.2f}, spk CE: {:.3f}, spk Acc: {:2.2f}".format(
                i, ces[0], 100.0*accs[0], ces[1], 100.0*accs[1]
            )
            print(string)
    plotter.plot(X, t_phn, t_spk, name="BN features, PHN+SPK optimized", transform=bn_backup)
<commit_msg>multi_target_epoch(): pass optimizer instead of parameters<commit_after>#!/usr/bin/env python3

import numpy as np
import torch

import matplotlib
import matplotlib.pyplot as plt
from torch.autograd import Variable

from itertools import chain
import copy
import atexit
import sys
import argparse

class PhnSpkGenerator():
    def __init__(self, mu, cov, phn, spk):
        self._mu = mu
        self._cov = cov
        self._phn = phn
        self._spk = spk

    def generate(self, N):
        X = np.random.multivariate_normal(self._mu, self._cov, size=N)
        phn = torch.ones(N) * self._phn
        spk = torch.ones(N) * self._spk
        return torch.from_numpy(X).float(), phn, spk

class Plotter():
    def __init__(self):
        atexit.register(plt.show)
        self._cmap = matplotlib.colors.ListedColormap([
            (1, 0, 0),
            (0, 1, 0),
            (0, 0, 1)
        ])

    def plot(self, X, phn, spk, name="fig", transform = lambda x:x):
        plt.figure(name)

        for i, m in enumerate(['o', '+', 'x']):
            mask = (spk.numpy() == i)
            spk_set = X.numpy()[mask]
            spk_set = Variable(torch.from_numpy(spk_set).float())
            spk_set = transform(spk_set).data.numpy()
            plt.scatter(spk_set[:,0], spk_set[:,1],
                        c=t_phn.numpy()[mask], cmap=self._cmap, marker=m)
        plt.show(block=False)

def epoch(fwd, optim, X, target, batch_size=16, shuffle=True, train=True):
    N = X.size()[0]
    assert target.size()[0] == N

    train_X = X
    train_t = target
    if shuffle:
        p = np.random.permutation(N)
        train_X = X.numpy()[p]
        train_t = target.numpy()[p]

    train_X = torch.from_numpy(train_X)
    train_t = torch.from_numpy(train_t)

    nb_batches = N // batch_size
    total = nb_batches * batch_size
    total_loss = 0.0
    correct = 0

    criterion = torch.nn.NLLLoss()

    for i in range(nb_batches):
        batch_X = Variable(train_X[i*batch_size:(i+1)*batch_size])
        batch_t = Variable(train_t[i*batch_size:(i+1)*batch_size])

        y = fwd(batch_X)
        loss = criterion(y, batch_t)
        _, preds = y.max(dim=1)
        correct += (preds == batch_t).sum().data[0]

        total_loss += loss.data[0]

        if train:
            optim.zero_grad()
            loss.backward()
            optim.step()

    return total_loss/nb_batches, correct/total

def multi_target_epoch(common, decoders, params, X, targets, batch_size=16, shuffle=True, train=True):
    assert len(decoders) == len(targets)
    N = X.size()[0]
    for t in targets:
        assert t.size()[0] == N

    # TODO: clone?
    train_X = X
    train_targets = targets
    if shuffle:
        p = np.random.permutation(N)
        train_X = X.numpy()[p]
        train_X = torch.from_numpy(train_X)
        for i, t in enumerate(train_targets):
            train_targets[i] = torch.from_numpy(t.numpy()[p])

    nb_batches = N // batch_size
    total = nb_batches * batch_size
    cum_losses = []
    nb_correct = []
    for i in range(len(decoders)):
        cum_losses.append(0.0)
        nb_correct.append(0)

    criterion = torch.nn.NLLLoss()

    for i in range(nb_batches):
        batch_X = Variable(train_X[i*batch_size:(i+1)*batch_size])
        batch_ts = []
        for t in train_targets:
            batch_ts.append(Variable(t[i*batch_size:(i+1)*batch_size]))

        repre = common(batch_X)

        losses = []
        for i, (dec, t) in enumerate(zip(decoders, batch_ts)):
            y = dec(repre)
            loss = criterion(y, t)
            losses.append(loss)
            _, preds = y.max(dim=1)
            nb_correct[i] += (preds == t).sum().data[0]
            cum_losses[i] += loss.data[0]

        if train:
            complete_loss = sum(losses)
            optim.zero_grad()
            complete_loss.backward()
            optim.step()

    return [cl/nb_batches for cl in cum_losses], [c/total for c in nb_correct]

def instantiate_generators():
    phn_mus = []
    phn_mus.append(np.asarray([1,1]))
    phn_mus.append(np.asarray([3.5,2]))
    phn_mus.append(np.asarray([2,4]))

    phn_covs = []
    phn_covs.append(np.asarray([[0.2,0], [0,0.2]]))
    phn_covs.append(np.asarray([[0.2,0], [0,0.2]]))
    phn_covs.append(np.asarray([[0.2,0], [0,0.2]]))

    spk_mus = []
    spk_mus.append(np.asarray([-0.7, 0.7]))
    spk_mus.append(np.asarray([1, 0]))
    spk_mus.append(np.asarray([-0.7, -0.7]))

    gens = []
    for phn, (phn_mu, phn_cov) in enumerate(zip(phn_mus, phn_covs)):
        for spk, spk_mu in enumerate(spk_mus):
            gens.append(PhnSpkGenerator(phn_mu+spk_mu, phn_cov, phn, spk))

    return gens

def generate(gens, N_per_cluster):
    X = torch.zeros((0, 2))
    t_phn = torch.zeros((0,))
    t_spk = torch.zeros((0,))

    for g in gens:
        X_g, phn_g, spk_g = g.generate(100)
        X = torch.cat([X, X_g], 0)
        t_phn = torch.cat([t_phn, phn_g], 0)
        t_spk = torch.cat([t_spk, spk_g], 0)
    t_phn = t_phn.long()
    t_spk = t_spk.long()

    return X, t_phn, t_spk

def train(common, decoders, params, X, ts, nb_epochs, report_interval=25):
    params = list(params)
    for i in range(nb_epochs):
        ce, acc = multi_target_epoch(common, decoders, params, X, ts)
        val_ce, val_acc = multi_target_epoch(common, decoders, params, X, ts, train=False)

        if i % report_interval == report_interval - 1:
            string = "{:>3} phn CE: {:.3f}, phn Acc: {:2.2f} | Valid: CE {:.3f} Acc {:.3f}".format(
                i, ce[0], 100.0*acc[0], val_ce[0], 100.0*val_acc[0]
            )
            print(string)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("--nb-epochs", type=int, default=200,
                        help="number of training epochs")
    args = parser.parse_args()

    gens = instantiate_generators()

    X, t_phn, t_spk = generate(gens, 100)
    X_val, t_phn_val, t_spk_val = generate(gens, 100)

    plotter = Plotter()
    #plotter.plot(X, t_phn, t_spk, name="Raw data")

    bn_extractor = torch.nn.Sequential(
        torch.nn.Linear(2, 10),
        torch.nn.ReLU(),
        torch.nn.Linear(10, 10),
        torch.nn.ReLU(),
        torch.nn.Linear(10, 2),
    )

    bn_backup = copy.deepcopy(bn_extractor)

    plotter.plot(X, t_phn, t_spk, name="BN features, random init", transform=bn_extractor)

    phn_decoder = torch.nn.Sequential(
        torch.nn.Linear(2,10),
        torch.nn.ReLU(),
        torch.nn.Linear(10,3),
        torch.nn.LogSoftmax()
    )
    phn_backup = copy.deepcopy(phn_decoder)

    print("Training PHN network")
    optim = torch.optim.SGD(chain(bn_extractor.parameters(), phn_decoder.parameters()), lr=1e-3)
    for i in range(args.nb_epochs):
        ce, acc = multi_target_epoch(bn_extractor, [phn_decoder], optim , X, [t_phn])
        val_ce, val_acc = multi_target_epoch(bn_extractor, [phn_decoder], optim, X, [t_phn], train=False)

        if i % 25 == 25 - 1:
            string = "{:>3} phn CE: {:.3f}, phn Acc: {:2.2f} | Valid: CE {:.3f} Acc {:.3f}".format(
                i, ce[0], 100.0*acc[0], val_ce[0], 100.0*val_acc[0]
            )
            print(string)
    # train(bn_extractor, [phn_decoder],
    #       chain(bn_extractor.parameters(), phn_decoder.parameters()),
    #       X, [t_phn], args.nb_epochs)

    plotter.plot(X, t_phn, t_spk, name="BN features, PHN optimized", transform=bn_extractor)

    spk_decoder = torch.nn.Sequential(
        torch.nn.Linear(2,10),
        torch.nn.ReLU(),
        torch.nn.Linear(10,3),
        torch.nn.LogSoftmax()
    )
    spk_backup = copy.deepcopy(spk_decoder)

    print("Training SPK decoder")
    optim = torch.optim.SGD(spk_decoder.parameters(), lr=1e-3)
    for i in range(args.nb_epochs):
        ce, acc = multi_target_epoch(bn_extractor, [spk_decoder], optim, X, [t_spk])
        val_ce, val_acc = multi_target_epoch(bn_extractor, [spk_decoder], optim, X, [t_spk], train=False)

        if i % 25 == 25 - 1:
            string = "{:>3} phn CE: {:.3f}, phn Acc: {:2.2f} | Valid: CE {:.3f} Acc {:.3f}".format(
                i, ce[0], 100.0*acc[0], val_ce[0], 100.0*val_acc[0]
            )
            print(string)
    # train(bn_extractor, [spk_decoder],
    #       spk_decoder.parameters(), X, [t_spk], args.nb_epochs)

    print("Training jointly, from same init:")
    optim = torch.optim.SGD(chain(bn_backup.parameters(), phn_backup.parameters(), spk_backup.parameters()), lr=1e-3)
    for i in range(args.nb_epochs):
        ces, accs= multi_target_epoch(bn_backup, [phn_backup, spk_backup], optim, X, [t_phn, t_spk])
        if i % 25 == 24:
            string = "{:>3} phn CE: {:.3f}, phn Acc: {:2.2f}, spk CE: {:.3f}, spk Acc: {:2.2f}".format(
                i, ces[0], 100.0*accs[0], ces[1], 100.0*accs[1]
            )
            print(string)
    plotter.plot(X, t_phn, t_spk, name="BN features, PHN+SPK optimized", transform=bn_backup)
<|endoftext|>


<commit_before>import datetime
import os
from flask import Flask, g, request, render_template, redirect, url_for, abort
from sqlalchemy import and_
from sqlalchemy.orm.exc import NoResultFound
from webhelpers import paginate

from lib.messages import parse_line
from lib.model import Log, LogPage
from lib.requests import connect_mysql, disconnect_mysql

from werkzeug.contrib.fixers import ProxyFix

app = Flask(__name__)

# Export config
app.config['EXPORT_URL'] = os.environ.get("EXPORT_URL", "http://unsupportedlogs.msparp.com")

app.wsgi_app = ProxyFix(app.wsgi_app, 2)

# Pre and post request stuff
app.before_request(connect_mysql)
app.teardown_request(disconnect_mysql)

# Chat

@app.route('/chat')
@app.route('/chat/<chat>')
def chat(chat=None):
    if chat is None:
        return redirect(url_for("home"))
    return redirect(url_for("view_log", chat=chat))

@app.route('/logs/group/<chat>')
def old_view_log(chat):
    return redirect(url_for('view_log', chat=chat))

@app.route('/logs/<log_id>')
def view_log_by_id(log_id=None):
    try:
        log_id = int(log_id)
    except ValueError:
        abort(400)
    try:
        log = g.mysql.query(Log).filter(Log.id==log_id).one()
    except NoResultFound:
        abort(404)
    if log.url is not None:
        return redirect(url_for('view_log', chat=log.url))
    abort(404)

@app.route('/chat/<chat>/log')
def view_log(chat=None):

    try:
        log = g.mysql.query(Log).filter(Log.url==chat).one()
    except NoResultFound:
        abort(404)

    current_page = request.args.get('page') or log.page_count
    mode = request.args.get('mode') or 'normal'

    try:
        log_page = g.mysql.query(LogPage).filter(and_(LogPage.log_id==log.id, LogPage.number==current_page)).one()
    except NoResultFound:
        abort(404)

    url_generator = paginate.PageURL(url_for('view_log', chat=chat), {'page': current_page})

    # It's only one row per page and we want to fetch them via both log id and
    # page number rather than slicing, so we'll just give it an empty list and
    # override the count.
    paginator = paginate.Page([], page=current_page, items_per_page=1, item_count=log.page_count, url=url_generator)

    # Pages end with a line break, so the last line is blank.
    lines = log_page.content.split('\n')[0:-1]
    lines = filter(lambda x: x is not None, map(lambda _: parse_line(_, 0), lines))

    for line in lines:
        line['datetime'] = datetime.datetime.fromtimestamp(line['timestamp'])

    return render_template('log.html',
        chat=chat,
        lines=lines,
        current_page=current_page,
        mode=mode,
        paginator=paginator,
    )

@app.route('/health', methods=['GET'])
def doHealthCheck():
    # should probably actually DO a health check here
    return 'ok'

# Redirects

@app.route("/faq")
def faq():
    return render_template("pages/msparpfaq.html")

@app.route("/bbcode")
def bbcode():
    return render_template("pages/bbcode.html")

@app.route("/userguide")
def userguide():
    return render_template("pages/userguide.html")

# Home

@app.route("/")
def configure():
    return render_template("frontpage.html")

# Exporting

@app.route('/chat/<chat>/export')
def export_log(chat=None):
    if g.redis.exists('chat.' + chat + '.exported'):
        return render_template('export_complete.html', chat=chat)

    # Add to queue if chat log exists.
    if g.mysql.query(Log).filter(Log.url == chat).scalar():
        g.redis.sadd('export-queue', chat)

    return render_template('export_progress.html', chat=chat)

if __name__ == "__main__":
    app.run(port=8000, debug=True)
<commit_msg>home -> configure<commit_after>import datetime
import os
from flask import Flask, g, request, render_template, redirect, url_for, abort
from sqlalchemy import and_
from sqlalchemy.orm.exc import NoResultFound
from webhelpers import paginate

from lib.messages import parse_line
from lib.model import Log, LogPage
from lib.requests import connect_mysql, disconnect_mysql

from werkzeug.contrib.fixers import ProxyFix

app = Flask(__name__)

# Export config
app.config['EXPORT_URL'] = os.environ.get("EXPORT_URL", "http://unsupportedlogs.msparp.com")

app.wsgi_app = ProxyFix(app.wsgi_app, 2)

# Pre and post request stuff
app.before_request(connect_mysql)
app.teardown_request(disconnect_mysql)

# Chat

@app.route('/chat')
@app.route('/chat/<chat>')
def chat(chat=None):
    if chat is None:
        return redirect(url_for("configure"))
    return redirect(url_for("view_log", chat=chat))

@app.route('/logs/group/<chat>')
def old_view_log(chat):
    return redirect(url_for('view_log', chat=chat))

@app.route('/logs/<log_id>')
def view_log_by_id(log_id=None):
    try:
        log_id = int(log_id)
    except ValueError:
        abort(400)
    try:
        log = g.mysql.query(Log).filter(Log.id==log_id).one()
    except NoResultFound:
        abort(404)
    if log.url is not None:
        return redirect(url_for('view_log', chat=log.url))
    abort(404)

@app.route('/chat/<chat>/log')
def view_log(chat=None):

    try:
        log = g.mysql.query(Log).filter(Log.url==chat).one()
    except NoResultFound:
        abort(404)

    current_page = request.args.get('page') or log.page_count
    mode = request.args.get('mode') or 'normal'

    try:
        log_page = g.mysql.query(LogPage).filter(and_(LogPage.log_id==log.id, LogPage.number==current_page)).one()
    except NoResultFound:
        abort(404)

    url_generator = paginate.PageURL(url_for('view_log', chat=chat), {'page': current_page})

    # It's only one row per page and we want to fetch them via both log id and
    # page number rather than slicing, so we'll just give it an empty list and
    # override the count.
    paginator = paginate.Page([], page=current_page, items_per_page=1, item_count=log.page_count, url=url_generator)

    # Pages end with a line break, so the last line is blank.
    lines = log_page.content.split('\n')[0:-1]
    lines = filter(lambda x: x is not None, map(lambda _: parse_line(_, 0), lines))

    for line in lines:
        line['datetime'] = datetime.datetime.fromtimestamp(line['timestamp'])

    return render_template('log.html',
        chat=chat,
        lines=lines,
        current_page=current_page,
        mode=mode,
        paginator=paginator,
    )

@app.route('/health', methods=['GET'])
def doHealthCheck():
    # should probably actually DO a health check here
    return 'ok'

# Redirects

@app.route("/faq")
def faq():
    return render_template("pages/msparpfaq.html")

@app.route("/bbcode")
def bbcode():
    return render_template("pages/bbcode.html")

@app.route("/userguide")
def userguide():
    return render_template("pages/userguide.html")

# Home

@app.route("/")
def configure():
    return render_template("frontpage.html")

# Exporting

@app.route('/chat/<chat>/export')
def export_log(chat=None):
    if g.redis.exists('chat.' + chat + '.exported'):
        return render_template('export_complete.html', chat=chat)

    # Add to queue if chat log exists.
    if g.mysql.query(Log).filter(Log.url == chat).scalar():
        g.redis.sadd('export-queue', chat)

    return render_template('export_progress.html', chat=chat)

if __name__ == "__main__":
    app.run(port=8000, debug=True)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import urllib
import urllib2
import json

import aqt.editor

# import the main window object (mw) from ankiqt
from aqt import mw, browser, editor
# import the "show info" tool from utils.py
from aqt.utils import showInfo
from anki.utils import stripHTML
# import all of the Qt GUI library
from aqt.qt import *
from aqt.editor import Editor

#more imports
from anki.hooks import addHook, runHook, wrap
from anki.consts import MODEL_STD

#Debugging
#import code
# code.interact(local=locals())

def setupSearchBrowserButton(self):
    open_button = self._addButton("imagesDialog", lambda self=self: review_entries(),
                text=u"img\u0336", tip="ImageDialog (Ctrl+i)", key="Ctrl+i")

def setupDrawingCanvasButton(self):
    canvas_button = self._addButton("Canvas", lambda self=self: initialize_canvas(),
            text=u"d\u0336", tip="Canvas (Ctrl+d)", key="Ctrl+d")

Editor.setupButtons = wrap(Editor.setupButtons, setupSearchBrowserButton)
Editor.setupButtons = wrap(Editor.setupButtons, setupDrawingCanvasButton)
addHook("editFocusGained", gainFocus)<commit_msg>Added local folder to system path<commit_after># -*- coding: utf-8 -*-
import urllib
import urllib2
import json

import aqt.editor

import sys
print(sys.path)
sys.path.append(".")
print(sys.path)

# import the main window object (mw) from ankiqt
from aqt import mw, browser, editor
# import the "show info" tool from utils.py
from aqt.utils import showInfo
from anki.utils import stripHTML
# import all of the Qt GUI library
from aqt.qt import *
from aqt.editor import Editor

#more imports
from anki.hooks import addHook, runHook, wrap
from anki.consts import MODEL_STD

# import web_search.py

from web_search import review_entries
from paint_tool import initialize_canvas
#

#Debugging
#import code
# code.interact(local=locals())

def setupSearchBrowserButton(self):
    open_button = self._addButton("imagesDialog", lambda self=self: review_entries(),
                text=u"img\u0336", tip="ImageDialog (Ctrl+i)", key="Ctrl+i")

def setupDrawingCanvasButton(self):
    canvas_button = self._addButton("Canvas", lambda self=self: initialize_canvas(),
            text=u"d\u0336", tip="Canvas (Ctrl+d)", key="Ctrl+d")

def gainFocus(note, field):
    global search_term
    global currentNote
    global mediaField
    # Sets the search term to the field you're on
    # Note: does not work with newly added fields until restart
    search_term = note.fields[field]
    currentNote = note
    mediaField = field

Editor.setupButtons = wrap(Editor.setupButtons, setupSearchBrowserButton)
Editor.setupButtons = wrap(Editor.setupButtons, setupDrawingCanvasButton)
addHook("editFocusGained", gainFocus)
<|endoftext|>


<commit_before><commit_msg>Fix chi2 parsing<commit_after><|endoftext|>


<commit_before>import tensorflow as tf
import sys
import os
import dataqueue
from train import train

file_name = "../src/build/data/guanhanw/combined.data"

FLAGS = tf.app.flags.FLAGS
tf.app.flags.DEFINE_string('num_gpus',1,'''the number of gpu''')
tf.app.flags.DEFINE_string('batch_size',20,'''the batch size''')
tf.app.flags.DEFINE_string('feature_dim',10,'''the feature dimension''')
tf.app.flags.DEFINE_string('label_dim',9,'''the label dimension''')

tf.app.flags.DEFINE_string('train_log_dir','visual_audio_logs',
        '''directory wherer to write event logs''')
tf.app.flags.DEFINE_integer('max_training_iter', 10000,
        '''the max number of training iteration''')
tf.app.flags.DEFINE_float('init_learning_rate',0.1,
        '''initial learning rate''')
tf.app.flags.DEFINE_string('model_dir', 'models','''directory where to save the model''')
tf.app.flags.DEFINE_string('txt_log', 'train_log.txt','''directory where to save the display log''')

def main(argv = sys.argv):
    if not os.path.exists(FLAGS.model_dir):
        os.makedirs(FLAGS.model_dir)
    if not os.path.exists(FLAGS.train_log_dir):
        os.makedirs(FLAGS.train_log_dir)

    dataqueue = dataqueue.dataqueue(file_name)
    train(FLAGS, dataqueue)

if __name__ == '__main__':
    tf.app.run()

<commit_msg>rmmain<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""\
This is a python port of "Goose" orignialy licensed to Gravity.com
under one or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.

Python port was written by Xavier Grangier for Recrutae

Gravity.com licenses this file
to you under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from goose.utils import ReplaceSequence

class DocumentCleaner(object):

    def __init__(self, config):
        self.config = config
        # parser
        self.parser = self.config.get_parser()
        self.remove_nodes_re = (
        "^side$|combx|retweet|mediaarticlerelated|menucontainer|navbar"
        "|comment|PopularQuestions|contact|foot|footer|Footer|footnote"
        "|cnn_strycaptiontxt|cnn_html_slideshow|cnn_strylftcntnt|links|meta$|scroll|shoutbox|sponsor"
        "|tags|socialnetworking|socialNetworking|cnnStryHghLght"
        "|cnn_stryspcvbx|^inset$|pagetools|post-attributes"
        "|welcome_form|contentTools2|the_answers"
        "|communitypromo|runaroundLeft|subscribe|vcard|articleheadings"
        "|date|^print$|popup|author-dropdown|tools|socialtools|byline"
        "|konafilter|KonaFilter|breadcrumbs|^fn$|wp-caption-text"
        "|legende|ajoutVideo|timestamp|js_replies"
        )
        self.regexp_namespace = "http://exslt.org/regular-expressions"
        self.nauthy_ids_re = "//*[re:test(@id, '%s', 'i')]" % self.remove_nodes_re
        self.nauthy_classes_re = "//*[re:test(@class, '%s', 'i')]" % self.remove_nodes_re
        self.nauthy_names_re = "//*[re:test(@name, '%s', 'i')]" % self.remove_nodes_re
        self.div_to_p_re = r"<(a|blockquote|dl|div|img|ol|p|pre|table|ul)"
        self.caption_re = "^caption$"
        self.google_re = " google "
        self.entries_re = "^[^entry-]more.*$"
        self.facebook_re = "[^-]facebook"
        self.facebook_braodcasting_re = "facebook-broadcasting"
        self.twitter_re = "[^-]twitter"
        self.tablines_replacements = ReplaceSequence()\
                                            .create("\n", "\n\n")\
                                            .append("\t")\
                                            .append("^\\s+$")

    def clean(self, article):

        doc_to_clean = article.doc
        doc_to_clean = self.clean_article_tags(doc_to_clean)
        doc_to_clean = self.clean_em_tags(doc_to_clean)
        doc_to_clean = self.remove_drop_caps(doc_to_clean)
        doc_to_clean = self.remove_scripts_styles(doc_to_clean)
        doc_to_clean = self.clean_bad_tags(doc_to_clean)
        doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.caption_re)
        doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.google_re)
        doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.entries_re)
        doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.facebook_re)
        doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.facebook_braodcasting_re)
        doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.twitter_re)
        doc_to_clean = self.clean_para_spans(doc_to_clean)
        doc_to_clean = self.div_to_para(doc_to_clean, 'div')
        doc_to_clean = self.div_to_para(doc_to_clean, 'span')
        return doc_to_clean

    def clean_article_tags(self, doc):
        articles = self.parser.getElementsByTag(doc, tag='article')
        for article in articles:
            for attr in ['id', 'name', 'class']:
                self.parser.delAttribute(article, attr=attr)
        return doc

    def clean_em_tags(self, doc):
        ems = self.parser.getElementsByTag(doc, tag='em')
        for node in ems:
            images = self.parser.getElementsByTag(node, tag='img')
            if len(images) == 0:
                self.parser.drop_tag(node)
        return doc

    def remove_drop_caps(self, doc):
        items = self.parser.css_select(doc, "span[class~=dropcap], span[class~=drop_cap]")
        for item in items:
            self.parser.drop_tag(item)

        return doc

    def remove_scripts_styles(self, doc):
        # remove scripts
        scripts = self.parser.getElementsByTag(doc, tag='script')
        for item in scripts:
            self.parser.remove(item)

        # remove styles
        styles = self.parser.getElementsByTag(doc, tag='style')
        for item in styles:
            self.parser.remove(item)

        # remove comments
        comments = self.parser.getComments(doc)
        for item in comments:
            self.parser.remove(item)

        return doc

    def clean_bad_tags(self, doc):
        # ids
        naughty_list = self.parser.xpath_re(doc, self.nauthy_ids_re)
        for node in naughty_list:
            self.parser.remove(node)

        # class
        naughty_classes = self.parser.xpath_re(doc, self.nauthy_classes_re)
        for node in naughty_classes:
            self.parser.remove(node)

        # name
        naughty_names = self.parser.xpath_re(doc, self.nauthy_names_re)
        for node in naughty_names:
            self.parser.remove(node)

        return doc

    def remove_nodes_regex(self, doc, pattern):
        for selector in ['id', 'class']:
            reg = "//*[re:test(@%s, '%s', 'i')]" % (selector, pattern)
            naughty_list = self.parser.xpath_re(doc, reg)
            for node in naughty_list:
                self.parser.remove(node)
        return doc

    def clean_para_spans(self, doc):
        spans = self.parser.css_select(doc, 'p > span')
        for item in spans:
            self.parser.drop_tag(item)
        return doc

    def get_flushed_buffer(self, replacement_text, doc):
        return self.parser.textToPara(replacement_text)

    def get_replacement_nodes(self, doc, div):
        replacement_text = []
        nodes_to_return = []
        nodes_to_remove = []
        childs = self.parser.childNodesWithText(div)

        for kid in childs:
            # node is a p
            # and already have some replacement text
            if self.parser.getTag(kid) == 'p' and len(replacement_text) > 0:
                newNode = self.get_flushed_buffer(''.join(replacement_text), doc)
                nodes_to_return.append(newNode)
                replacement_text = []
                nodes_to_return.append(kid)
            # node is a text node
            elif self.parser.isTextNode(kid):
                kid_text_node = kid
                kid_text = self.parser.getText(kid)
                replace_text = self.tablines_replacements.replaceAll(kid_text)
                if(len(replace_text)) > 1:
                    previous_sibling_node = self.parser.previousSibling(kid_text_node)
                    while previous_sibling_node is not None \
                        and self.parser.getTag(previous_sibling_node) == "a" \
                        and self.parser.getAttribute(previous_sibling_node, 'grv-usedalready') != 'yes':
                        outer = " " + self.parser.outerHtml(previous_sibling_node) + " "
                        replacement_text.append(outer)
                        nodes_to_remove.append(previous_sibling_node)
                        self.parser.setAttribute(previous_sibling_node,
                                    attr='grv-usedalready', value='yes')
                        prev = self.parser.previousSibling(previous_sibling_node)
                        previous_sibling_node = prev if prev is not None else None
                    # append replace_text
                    replacement_text.append(replace_text)
                    #
                    next_sibling_node = self.parser.nextSibling(kid_text_node)
                    while next_sibling_node is not None \
                        and self.parser.getTag(next_sibling_node) == "a" \
                        and self.parser.getAttribute(next_sibling_node, 'grv-usedalready') != 'yes':
                        outer = " " + self.parser.outerHtml(next_sibling_node) + " "
                        replacement_text.append(outer)
                        nodes_to_remove.append(next_sibling_node)
                        self.parser.setAttribute(next_sibling_node,
                                    attr='grv-usedalready', value='yes')
                        next = self.parser.nextSibling(next_sibling_node)
                        previous_sibling_node = next if next is not None else None

            # otherwise
            else:
                nodes_to_return.append(kid)

        # flush out anything still remaining
        if(len(replacement_text) > 0):
            new_node = self.get_flushed_buffer(''.join(replacement_text), doc)
            nodes_to_return.append(new_node)
            replacement_text = []

        for n in nodes_to_remove:
            self.parser.remove(n)

        return nodes_to_return

    def replace_with_para(self, doc, div):
        self.parser.replaceTag(div, 'p')

    def div_to_para(self, doc, dom_type):
        bad_divs = 0
        else_divs = 0
        divs = self.parser.getElementsByTag(doc, tag=dom_type)
        tags = ['a', 'blockquote', 'dl', 'div', 'img', 'ol', 'p', 'pre', 'table', 'ul']

        for div in divs:
            items = self.parser.getElementsByTags(div, tags)
            if div is not None and len(items) == 0:
                self.replace_with_para(doc, div)
                bad_divs += 1
            elif div is not None:
                replaceNodes = self.get_replacement_nodes(doc, div)
                div.clear()

                for c, n in enumerate(replaceNodes):
                    div.insert(c, n)

                else_divs += 1

        return doc

class StandardDocumentCleaner(DocumentCleaner):
    pass
<commit_msg>#74 remove classes for body tag<commit_after># -*- coding: utf-8 -*-
"""\
This is a python port of "Goose" orignialy licensed to Gravity.com
under one or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.

Python port was written by Xavier Grangier for Recrutae

Gravity.com licenses this file
to you under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from goose.utils import ReplaceSequence

class DocumentCleaner(object):

    def __init__(self, config):
        self.config = config
        # parser
        self.parser = self.config.get_parser()
        self.remove_nodes_re = (
        "^side$|combx|retweet|mediaarticlerelated|menucontainer|"
        "navbar|storytopbar-bucket|utility-bar|inline-share-tools"
        "|comment|PopularQuestions|contact|foot|footer|Footer|footnote"
        "|cnn_strycaptiontxt|cnn_html_slideshow|cnn_strylftcntnt"
        "|links|meta$|shoutbox|sponsor"
        "|tags|socialnetworking|socialNetworking|cnnStryHghLght"
        "|cnn_stryspcvbx|^inset$|pagetools|post-attributes"
        "|welcome_form|contentTools2|the_answers"
        "|communitypromo|runaroundLeft|subscribe|vcard|articleheadings"
        "|date|^print$|popup|author-dropdown|tools|socialtools|byline"
        "|konafilter|KonaFilter|breadcrumbs|^fn$|wp-caption-text"
        "|legende|ajoutVideo|timestamp|js_replies"
        )
        self.regexp_namespace = "http://exslt.org/regular-expressions"
        self.nauthy_ids_re = "//*[re:test(@id, '%s', 'i')]" % self.remove_nodes_re
        self.nauthy_classes_re = "//*[re:test(@class, '%s', 'i')]" % self.remove_nodes_re
        self.nauthy_names_re = "//*[re:test(@name, '%s', 'i')]" % self.remove_nodes_re
        self.div_to_p_re = r"<(a|blockquote|dl|div|img|ol|p|pre|table|ul)"
        self.caption_re = "^caption$"
        self.google_re = " google "
        self.entries_re = "^[^entry-]more.*$"
        self.facebook_re = "[^-]facebook"
        self.facebook_braodcasting_re = "facebook-broadcasting"
        self.twitter_re = "[^-]twitter"
        self.tablines_replacements = ReplaceSequence()\
                                            .create("\n", "\n\n")\
                                            .append("\t")\
                                            .append("^\\s+$")

    def clean(self, article):

        doc_to_clean = article.doc
        doc_to_clean = self.clean_body_classes(doc_to_clean)
        doc_to_clean = self.clean_article_tags(doc_to_clean)
        doc_to_clean = self.clean_em_tags(doc_to_clean)
        doc_to_clean = self.remove_drop_caps(doc_to_clean)
        doc_to_clean = self.remove_scripts_styles(doc_to_clean)
        doc_to_clean = self.clean_bad_tags(doc_to_clean)
        doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.caption_re)
        doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.google_re)
        doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.entries_re)
        doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.facebook_re)
        doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.facebook_braodcasting_re)
        doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.twitter_re)
        doc_to_clean = self.clean_para_spans(doc_to_clean)
        doc_to_clean = self.div_to_para(doc_to_clean, 'div')
        doc_to_clean = self.div_to_para(doc_to_clean, 'span')
        return doc_to_clean

    def clean_body_classes(self, doc):
        # we don't need body classes
        # in case it matches an unwanted class all the document
        # will be empty
        elements = self.parser.getElementsByTag(doc, tag="body")
        self.parser.delAttribute(elements[0], attr="class")
        return doc

    def clean_article_tags(self, doc):
        articles = self.parser.getElementsByTag(doc, tag='article')
        for article in articles:
            for attr in ['id', 'name', 'class']:
                self.parser.delAttribute(article, attr=attr)
        return doc

    def clean_em_tags(self, doc):
        ems = self.parser.getElementsByTag(doc, tag='em')
        for node in ems:
            images = self.parser.getElementsByTag(node, tag='img')
            if len(images) == 0:
                self.parser.drop_tag(node)
        return doc

    def remove_drop_caps(self, doc):
        items = self.parser.css_select(doc, "span[class~=dropcap], span[class~=drop_cap]")
        for item in items:
            self.parser.drop_tag(item)

        return doc

    def remove_scripts_styles(self, doc):
        # remove scripts
        scripts = self.parser.getElementsByTag(doc, tag='script')
        for item in scripts:
            self.parser.remove(item)

        # remove styles
        styles = self.parser.getElementsByTag(doc, tag='style')
        for item in styles:
            self.parser.remove(item)

        # remove comments
        comments = self.parser.getComments(doc)
        for item in comments:
            self.parser.remove(item)

        return doc

    def clean_bad_tags(self, doc):
        # ids
        naughty_list = self.parser.xpath_re(doc, self.nauthy_ids_re)
        for node in naughty_list:
            self.parser.remove(node)

        # class
        naughty_classes = self.parser.xpath_re(doc, self.nauthy_classes_re)
        for node in naughty_classes:
            self.parser.remove(node)

        # name
        naughty_names = self.parser.xpath_re(doc, self.nauthy_names_re)
        for node in naughty_names:
            self.parser.remove(node)

        return doc

    def remove_nodes_regex(self, doc, pattern):
        for selector in ['id', 'class']:
            reg = "//*[re:test(@%s, '%s', 'i')]" % (selector, pattern)
            naughty_list = self.parser.xpath_re(doc, reg)
            for node in naughty_list:
                self.parser.remove(node)
        return doc

    def clean_para_spans(self, doc):
        spans = self.parser.css_select(doc, 'p > span')
        for item in spans:
            self.parser.drop_tag(item)
        return doc

    def get_flushed_buffer(self, replacement_text, doc):
        return self.parser.textToPara(replacement_text)

    def get_replacement_nodes(self, doc, div):
        replacement_text = []
        nodes_to_return = []
        nodes_to_remove = []
        childs = self.parser.childNodesWithText(div)

        for kid in childs:
            # node is a p
            # and already have some replacement text
            if self.parser.getTag(kid) == 'p' and len(replacement_text) > 0:
                newNode = self.get_flushed_buffer(''.join(replacement_text), doc)
                nodes_to_return.append(newNode)
                replacement_text = []
                nodes_to_return.append(kid)
            # node is a text node
            elif self.parser.isTextNode(kid):
                kid_text_node = kid
                kid_text = self.parser.getText(kid)
                replace_text = self.tablines_replacements.replaceAll(kid_text)
                if(len(replace_text)) > 1:
                    previous_sibling_node = self.parser.previousSibling(kid_text_node)
                    while previous_sibling_node is not None \
                        and self.parser.getTag(previous_sibling_node) == "a" \
                        and self.parser.getAttribute(previous_sibling_node, 'grv-usedalready') != 'yes':
                        outer = " " + self.parser.outerHtml(previous_sibling_node) + " "
                        replacement_text.append(outer)
                        nodes_to_remove.append(previous_sibling_node)
                        self.parser.setAttribute(previous_sibling_node,
                                    attr='grv-usedalready', value='yes')
                        prev = self.parser.previousSibling(previous_sibling_node)
                        previous_sibling_node = prev if prev is not None else None
                    # append replace_text
                    replacement_text.append(replace_text)
                    #
                    next_sibling_node = self.parser.nextSibling(kid_text_node)
                    while next_sibling_node is not None \
                        and self.parser.getTag(next_sibling_node) == "a" \
                        and self.parser.getAttribute(next_sibling_node, 'grv-usedalready') != 'yes':
                        outer = " " + self.parser.outerHtml(next_sibling_node) + " "
                        replacement_text.append(outer)
                        nodes_to_remove.append(next_sibling_node)
                        self.parser.setAttribute(next_sibling_node,
                                    attr='grv-usedalready', value='yes')
                        next = self.parser.nextSibling(next_sibling_node)
                        previous_sibling_node = next if next is not None else None

            # otherwise
            else:
                nodes_to_return.append(kid)

        # flush out anything still remaining
        if(len(replacement_text) > 0):
            new_node = self.get_flushed_buffer(''.join(replacement_text), doc)
            nodes_to_return.append(new_node)
            replacement_text = []

        for n in nodes_to_remove:
            self.parser.remove(n)

        return nodes_to_return

    def replace_with_para(self, doc, div):
        self.parser.replaceTag(div, 'p')

    def div_to_para(self, doc, dom_type):
        bad_divs = 0
        else_divs = 0
        divs = self.parser.getElementsByTag(doc, tag=dom_type)
        tags = ['a', 'blockquote', 'dl', 'div', 'img', 'ol', 'p', 'pre', 'table', 'ul']

        for div in divs:
            items = self.parser.getElementsByTags(div, tags)
            if div is not None and len(items) == 0:
                self.replace_with_para(doc, div)
                bad_divs += 1
            elif div is not None:
                replaceNodes = self.get_replacement_nodes(doc, div)
                div.clear()

                for c, n in enumerate(replaceNodes):
                    div.insert(c, n)

                else_divs += 1

        return doc

class StandardDocumentCleaner(DocumentCleaner):
    pass
<|endoftext|>


<commit_before>#!/usr/bin/env python3
"""Write a sample config for a given directory containing fastq files
following SRA naming conventions
"""

#--- standard library imports
#
import glob
import re
import os
import sys
import argparse
import logging

#--- third-party imports
#
import yaml

# --- project specific imports
#
# add lib dir for this pipeline installation to PYTHONPATH
LIB_PATH = os.path.abspath(os.path.join(os.path.dirname(
    os.path.realpath(__file__)), "..", "lib"))
if LIB_PATH not in sys.path:
    sys.path.insert(0, LIB_PATH)
from readunits import ReadUnit
from readunits import create_rg_id_from_ru
from readunits import key_for_readunit

__author__ = "Andreas WILM"
__email__ = "wilma@gis.a-star.edu.sg"
__copyright__ = "2016 Genome Institute of Singapore"
__license__ = "The MIT License (MIT)"

# global logger
logger = logging.getLogger(__name__)
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter(
    '[{asctime}] {levelname:8s} {filename} {message}', style='{'))
logger.addHandler(handler)

yaml.Dumper.ignore_aliases = lambda *args: True

def scheme_for_fastq(fastq):
    """
    Returns regexp matching auto determined fastq naming scheme

    From Chih Chuan 2016-10-26:
    We have 2 schemes, the Old H5 and the current CRAM.
    H5 Format :
    <runid>_<flowcell>_<barcode>.<library_id>_<laneid>_R[1|2].fastq.gz
    (AW: example actually showing a third also valid name:)
    HS003-PE-R00047_BC0HBTACXX.WSB100_TTAGGC_L003_R1.fastq.gz
    CRAM:
    <library_id>_<runid>_<laneid>_R[1|2].fastq
    WHH530_HS006-PE-R00021_L001_R1.fastq.gz

    """

    # sra naming schemes
    schemes = dict()
    schemes['h5old'] = re.compile(
        r'(?P<run_id>\w+)_(?P<flowcell>\w+).(?P<library_id>\w+)_(?P<barcode>\w+)_(?P<lane_id>\w+)_R(?P<read_no>[12]).fastq.gz')
    # schemes['h5new'] = re.compile(# FIXME MISSING
    schemes['cram'] = re.compile(# FIXME untested
        r'(?P<library_id>\w+)_(?P<run_id>\w+)_(?P<lane_id>\w+)_R(?P<read_no>[12]).fastq.gz')

    scheme_re = None#pylint
    for scheme_name, scheme_re in schemes.items():
        match = scheme_re.search(os.path.basename(fastq))
        if match:
            logger.info("Matching scheme %s", scheme_name)
            break
    assert match, ("No matching scheme found for {}".format(fastq))
    return scheme_re

def readunits_for_sampledir(sampledir):
    """Turns fastq files in sampledir to readunits assuming they follow a
    valid SRA naming scheme

    """

    # determine naming scheme and loop through fastqs assuming fixed scheme
    fq1s = glob.glob(os.path.join(sampledir, "*R1.fastq.gz"))
    scheme = scheme_for_fastq(fq1s[0])
    readunits = dict()
    for fq1 in fq1s:
        match = scheme.search(os.path.basename(fq1))
        mgroups = match.groupdict()
        fq2 = fq1.replace("R1.fastq.gz", "R2.fastq.gz")
        if not os.path.exists(fq2):
            fq2 = None
        rg = None
        ru = ReadUnit(mgroups.get('run_id'),
                      mgroups.get('flowcell'),
                      mgroups.get('library_id'),
                      mgroups.get('lane_id'),
                      rg, fq1, fq2)
        ru = ru._replace(rg_id=create_rg_id_from_ru(ru))
        readunits[key_for_readunit(ru)] = dict(ru._asdict())
    return readunits

def sampledir_to_cfg(sampledir, samplecfg):
    """FIXME:add-doc
    """

    readunits = readunits_for_sampledir(sampledir)

    # in theory we could support multi sample in one dir. here we're being strict
    lib_ids = [ru['library_id'] for ru in readunits.values()]
    assert len(set(lib_ids)) == 1
    sample_name = lib_ids[0]
    samples = dict()
    samples[sample_name] = list(readunits.keys())

    # make fastq paths relativ to output
    for ru_key, ru in readunits.items():
        # read units are dicts here (not namedtuple)
        fq1 = ru['fq1']
        ru['fq1'] = os.path.relpath(fq1, start=os.path.dirname(samplecfg))
        fq2 = ru['fq2']
        if fq2:
            fq2 = os.path.relpath(fq2, start=os.path.dirname(samplecfg))
            ru['fq2'] = fq2
        # no need: readunits[ru_key] = ru

    with open(samplecfg, 'w') as fh:
        yaml.dump(dict(samples=samples), fh, default_flow_style=False)
        yaml.dump(dict(readunits=readunits), fh, default_flow_style=False)

def main():
    """main function"""

    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('-s', "--sampledir", required=True,
                        help="Sample directory containing fastq files, following any SRA naming convention")
    parser.add_argument('-o', "--samplecfg", required=True,
                        help="Output YAML files. FastQ file names will be relative to this file")
    parser.add_argument('-v', '--verbose', action='count', default=0,
                        help="Increase verbosity")
    parser.add_argument('-q', '--quiet', action='count', default=0,
                        help="Decrease verbosity")
    args = parser.parse_args()

    # Repeateable -v and -q for setting logging level.
    # See https://www.reddit.com/r/Python/comments/3nctlm/what_python_tools_should_i_be_using_on_every/
    logger.setLevel(logging.WARN + 10*args.quiet - 10*args.verbose)

    if not os.path.exists(args.sampledir):
        logger.fatal("Non existing sample directory %s", args.sampledir)
        sys.exit(1)
    if os.path.exists(args.samplecfg):
        logger.fatal("Cowardly refusing to overwrite existing %s", args.samplecfg)
        sys.exit(1)

    sampledir_to_cfg(args.sampledir, args.samplecfg)

if __name__ == "__main__":
    main()
<commit_msg>Fixed regexp handling in tools/sra_dir_to_sample_cfg.py<commit_after>#!/usr/bin/env python3
"""Write a sample config for a given directory containing fastq files
following SRA naming conventions
"""

#--- standard library imports
#
import glob
import re
import os
import sys
import argparse
import logging

#--- third-party imports
#
import yaml

# --- project specific imports
#
# add lib dir for this pipeline installation to PYTHONPATH
LIB_PATH = os.path.abspath(os.path.join(os.path.dirname(
    os.path.realpath(__file__)), "..", "lib"))
if LIB_PATH not in sys.path:
    sys.path.insert(0, LIB_PATH)
from readunits import ReadUnit
from readunits import create_rg_id_from_ru
from readunits import key_for_readunit

__author__ = "Andreas WILM"
__email__ = "wilma@gis.a-star.edu.sg"
__copyright__ = "2016 Genome Institute of Singapore"
__license__ = "The MIT License (MIT)"

# global logger
logger = logging.getLogger(__name__)
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter(
    '[{asctime}] {levelname:8s} {filename} {message}', style='{'))
logger.addHandler(handler)

yaml.Dumper.ignore_aliases = lambda *args: True

def scheme_for_fastq(fastq):
    """
    Returns regexp matching auto determined fastq naming scheme

    From Chih Chuan 2016-10-26:
    We have 2 schemes, the Old H5 and the current CRAM.
    H5 Format :
    <runid>_<flowcell>_<barcode>.<library_id>_<laneid>_R[1|2].fastq.gz
    (AW: example actually showing a third also valid name:)
    HS003-PE-R00047_BC0HBTACXX.WSB100_TTAGGC_L003_R1.fastq.gz
    CRAM:
    <library_id>_<runid>_<laneid>_R[1|2].fastq
    WHH530_HS006-PE-R00021_L001_R1.fastq.gz

    """

    # sra naming schemes (use basename as input)
    schemes = dict()
    schemes['h5old'] = re.compile(
        r'(?P<run_id>[A-Za-z0-9-]+)_(?P<flowcell>[A-Za-z0-9-]+)\.(?P<library_id>[A-Za-z0-9-]+)_(?P<barcode>[A-Za-z0-9-]+)_L0*(?P<lane_id>[A-Za-z0-9-]+)_R(?P<read_no>[12]).fastq.gz')
    # schemes['h5new'] = re.compile(# FIXME MISSING
    schemes['cram'] = re.compile(# FIXME untested
        r'(?P<library_id>[A-Za-z0-9-]+)_(?P<run_id>[A-Za-z0-9-]+)_L0*(?P<lane_id>[A-Za-z0-9-]+)_R(?P<read_no>[12]).fastq.gz')

    scheme_re = None#pylint
    for scheme_name, scheme_re in schemes.items():
        match = scheme_re.match(os.path.basename(fastq))
        if match:
            logger.info("Matching scheme %s", scheme_name)
            break
    assert match, ("No matching scheme found for {}".format(fastq))
    return scheme_re

def readunits_for_sampledir(sampledir):
    """Turns fastq files in sampledir to readunits assuming they follow a
    valid SRA naming scheme

    """

    # determine naming scheme and loop through fastqs assuming fixed scheme
    fq1s = glob.glob(os.path.join(sampledir, "*R1.fastq.gz"))
    scheme = scheme_for_fastq(fq1s[0])
    readunits = dict()
    for fq1 in fq1s:
        match = scheme.search(os.path.basename(fq1))
        mgroups = match.groupdict()
        fq2 = fq1.replace("R1.fastq.gz", "R2.fastq.gz")
        if not os.path.exists(fq2):
            fq2 = None
        rg = None

        ru = ReadUnit(mgroups['run_id'],
                      mgroups.get('flowcell'),
                      mgroups['library_id'],
                      mgroups['lane_id'],
                      rg, fq1, fq2)
        ru = ru._replace(rg_id=create_rg_id_from_ru(ru))
        readunits[key_for_readunit(ru)] = dict(ru._asdict())
    return readunits

def sampledir_to_cfg(sampledir, samplecfg):
    """FIXME:add-doc
    """

    readunits = readunits_for_sampledir(sampledir)

    # in theory we could support multi sample in one dir. here we're being strict
    lib_ids = [ru['library_id'] for ru in readunits.values()]
    assert len(set(lib_ids)) == 1
    sample_name = lib_ids[0]
    samples = dict()
    samples[sample_name] = list(readunits.keys())

    # make fastq paths relativ to output
    for ru_key, ru in readunits.items():
        # read units are dicts here (not namedtuple)
        fq1 = ru['fq1']
        ru['fq1'] = os.path.relpath(fq1, start=os.path.dirname(samplecfg))
        fq2 = ru['fq2']
        if fq2:
            fq2 = os.path.relpath(fq2, start=os.path.dirname(samplecfg))
            ru['fq2'] = fq2
        # no need: readunits[ru_key] = ru

    with open(samplecfg, 'w') as fh:
        yaml.dump(dict(samples=samples), fh, default_flow_style=False)
        yaml.dump(dict(readunits=readunits), fh, default_flow_style=False)

def main():
    """main function"""

    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('-s', "--sampledir", required=True,
                        help="Sample directory containing fastq files, following any SRA naming convention")
    parser.add_argument('-o', "--samplecfg", required=True,
                        help="Output YAML files. FastQ file names will be relative to this file")
    parser.add_argument('-v', '--verbose', action='count', default=0,
                        help="Increase verbosity")
    parser.add_argument('-q', '--quiet', action='count', default=0,
                        help="Decrease verbosity")
    args = parser.parse_args()

    # Repeateable -v and -q for setting logging level.
    # See https://www.reddit.com/r/Python/comments/3nctlm/what_python_tools_should_i_be_using_on_every/
    logger.setLevel(logging.WARN + 10*args.quiet - 10*args.verbose)

    if not os.path.exists(args.sampledir):
        logger.fatal("Non existing sample directory %s", args.sampledir)
        sys.exit(1)
    if os.path.exists(args.samplecfg):
        logger.fatal("Cowardly refusing to overwrite existing %s", args.samplecfg)
        sys.exit(1)

    sampledir_to_cfg(args.sampledir, args.samplecfg)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>"""
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

This file serves two main purposes
- it serves up the main html page
- and it provides a simple set of apis to the javascript
"""

# If this is set to true, the client_secrets.json must be valid and users will
# be required to grant OAuth access to this app before continuing.
# This enables the Google API to work
REQUIRE_OAUTH = False

# If this is set to true, then this file will assume that app engine is
# being used to run the server.
USE_APPENGINE = False

import httplib2
import jinja2
import json
import logging
import os
import re
import socket
import webapp2

if USE_APPENGINE:
  from oauth2client import appengine
  from google.appengine.api import users
  from google.appengine.api import memcache

socket.setdefaulttimeout(60)
http = httplib2.Http(cache=memcache if USE_APPENGINE else None)

JINJA_ENVIRONMENT = jinja2.Environment(
    loader=jinja2.FileSystemLoader(os.path.dirname(__file__)),
    autoescape=True,
    extensions=['jinja2.ext.autoescape'])

client_secrets = os.path.join(os.path.dirname(__file__), 'client_secrets.json')

if USE_APPENGINE and REQUIRE_OAUTH:
  decorator = appengine.oauth2decorator_from_clientsecrets(
      client_secrets,
      scope=[
        'https://www.googleapis.com/auth/genomics',
        'https://www.googleapis.com/auth/devstorage.read_write'
      ])
else:
  class FakeOauthDecorator():
    def http(self):
      return http
    def oauth_aware(self, method):
      return method
    @property
    def callback_path(self):
      return '/unused'
    def callback_handler(self):
      pass
  decorator = FakeOauthDecorator()

# TODO: Dataset information should come from the list datasets api call
SUPPORTED_BACKENDS = {
  'NCBI' : {'name': 'NCBI',
            'url': 'http://trace.ncbi.nlm.nih.gov/Traces/gg',
            'datasets': {'SRP034507': 'SRP034507', 'SRP029392': 'SRP029392'}},
  'LOCAL' : {'name': 'Local',
             'url': 'http://localhost:5000',
             'datasets': {'All': ''}},
}
if REQUIRE_OAUTH:
  # Google temporarily requires OAuth on all calls
  SUPPORTED_BACKENDS['GOOGLE'] = {
    'name': 'Google',
    'url': 'https://www.googleapis.com/genomics/v1beta',
    'datasets': {'1000 Genomes': '376902546192', 'PGP': '383928317087'}
  }

class ApiException(Exception):
  pass

# Request handlers
class BaseRequestHandler(webapp2.RequestHandler):
  def handle_exception(self, exception, debug_mode):
    if isinstance(exception, ApiException):
      # ApiExceptions are expected, and will return nice error
      # messages to the client
      self.response.write(exception.message)
      self.response.set_status(400)
    else:
      # All other exceptions are unexpected and should be logged
      logging.exception('Unexpected exception')
      self.response.write('Unexpected internal exception')
      self.response.set_status(500)

  def get_backend(self):
    backend = self.request.get('backend')
    if not backend:
      raise ApiException('Backend parameter must be set')
    return backend

  def get_base_api_url(self):
    return SUPPORTED_BACKENDS[self.get_backend()]['url']

  def get_content(self, path, method='POST', body=None):
    http = decorator.http()
    response, content = http.request(
      uri="%s/%s" % (self.get_base_api_url(), path),
      method=method, body=json.dumps(body) if body else None,
      headers={'Content-Type': 'application/json; charset=UTF-8'})

    try:
      content = json.loads(content)
    except ValueError:
      logging.error("non-json api content %s" % content)
      raise ApiException('The API returned invalid JSON')

    if response.status >= 300:
      logging.error("error api response %s" % response)
      logging.error("error api content %s" % content)
      if 'error' in content:
        raise ApiException(content['error']['message'])
      else:
        raise ApiException('Something went wrong with the API call!')
    return content

  def write_content(self, path, method='POST', body=None):
    self.response.write(json.dumps(self.get_content(path, method, body)))

class ReadsetSearchHandler(BaseRequestHandler):

  @decorator.oauth_aware
  def get(self):
    readset_id = self.request.get('readsetId')
    if not readset_id:
      dataset_id = self.request.get('datasetId')
      name = self.request.get('name')
      if dataset_id:
        body = {'datasetIds' : [dataset_id]}
      else:
        # This is needed for the local readstore
        body = {'datasetIds' : []}
      response = self.get_content("readsets/search?fields=readsets(id,name)",
                                 body=body)

      # TODO: Use the api once name filtering is supported
      if name:
        name = name.lower()
        response['readsets'] = [r for r in response['readsets']
                                if name in r['name'].lower()]
      self.response.write(json.dumps(response))
      return

    # Single readset response
    self.write_content("readsets/%s" % readset_id, method='GET')

class ReadSearchHandler(BaseRequestHandler):

  @decorator.oauth_aware
  def get(self):
    body = {
      'readsetIds': self.request.get('readsetIds').split(','),
      'sequenceName': self.request.get('sequenceName'),
      'sequenceStart': max(0, int(self.request.get('sequenceStart'))),
      'sequenceEnd': int(self.request.get('sequenceEnd')),
    }
    pageToken = self.request.get('pageToken')
    if pageToken:
      body['pageToken'] = pageToken
    self.write_content("reads/search", body=body)

class BaseSnpediaHandler(webapp2.RequestHandler):
  def getSnppediaPageContent(self, snp):
    uri = "http://bots.snpedia.com/api.php?action=query&prop=revisions&" \
          "format=json&rvprop=content&titles=%s" % snp
    response, content = http.request(uri=uri)

    page_id, page = json.loads(content)['query']['pages'].popitem()
    return page['revisions'][0]['*']

  def getContentValue(self, content, key):
    try:
      matcher = '%s=(.*)\n' % key
      return re.search(matcher, content, re.I).group(1)
    except (KeyError, AttributeError):
      return ''

  def complement(self, base):
    return {'A': 'T', 'T': 'A', 'G': 'C', 'C': 'G'}[base]

class SnpSearchHandler(BaseSnpediaHandler):

  def getSnpResponse(self, name, content):
    return {
      'name': name,
      'link': 'http://www.snpedia.com/index.php/%s' % name,
      'position': self.getContentValue(content, 'position'),
      'chr': self.getContentValue(content, 'chromosome')
    }

  def get(self):
    snp = self.request.get('snp')

    try:
      content = self.getSnppediaPageContent(snp)
      if snp[:2].lower() == 'rs':
        snps = [self.getSnpResponse(snp, content)]
      else:
        # Try a gene format
        snps = re.findall('\[\[(rs\d+?)\]\]', content, re.I)
        snps = [self.getSnpResponse(s, self.getSnppediaPageContent(s))
                for s in set(snps)]

    except (ValueError, KeyError, AttributeError):
      snps = []
    self.response.write(json.dumps({'snps' : snps}))

class AlleleSearchHandler(BaseSnpediaHandler):

  def getAlleleResponse(self, name, content):
    return {
      'name': name,
      'link': 'http://www.snpedia.com/index.php/%s' % name,
      'repute': self.getContentValue(content, 'repute'),
      'summary': self.getContentValue(content, 'summary') or 'Unknown',
      'magnitude': self.getContentValue(content, 'magnitude')
    }

  def get(self):
    snp = self.request.get('snp')
    a1 = self.request.get('a1')
    a2 = self.request.get('a2')

    a1c = self.complement(a1)
    a2c = self.complement(a2)
    possible_names = [(snp, a1, a2), (snp, a2, a1),
                      (snp, a1c, a2c), (snp, a2c, a1c)]
    for name in possible_names:
      try:
        page = "%s(%s;%s)" % name
        content = self.getSnppediaPageContent(page)
        self.response.write(json.dumps(self.getAlleleResponse(page, content)))
        return
      except (ValueError, KeyError, AttributeError):
        pass # Continue trying the next allele name

    self.response.write(json.dumps({}))

class MainHandler(webapp2.RequestHandler):

  @decorator.oauth_aware
  def get(self):
    if not REQUIRE_OAUTH or decorator.has_credentials():
      template = JINJA_ENVIRONMENT.get_template('main.html')
      self.response.write(template.render({
        'username': users.User().nickname() if USE_APPENGINE else '',
        'logout_url': users.create_logout_url('/') if USE_APPENGINE else '',
        'backends': SUPPORTED_BACKENDS,
      }))
    else:
      # TODO: What kind of access do the non-google backends need?
      template = JINJA_ENVIRONMENT.get_template('grantaccess.html')
      self.response.write(template.render({
        'url': decorator.authorize_url()
      }))

web_app = webapp2.WSGIApplication(
    [
     ('/', MainHandler),
     ('/api/reads', ReadSearchHandler),
     ('/api/readsets', ReadsetSearchHandler),
     ('/api/snps', SnpSearchHandler),
     ('/api/alleles', AlleleSearchHandler),
     (decorator.callback_path, decorator.callback_handler()),
    ],
    debug=True)
<commit_msg>Remove unneeded devstorage oauth scope<commit_after>"""
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

This file serves two main purposes
- it serves up the main html page
- and it provides a simple set of apis to the javascript
"""

# If this is set to true, the client_secrets.json must be valid and users will
# be required to grant OAuth access to this app before continuing.
# This enables the Google API to work
REQUIRE_OAUTH = False

# If this is set to true, then this file will assume that app engine is
# being used to run the server.
USE_APPENGINE = False

import httplib2
import jinja2
import json
import logging
import os
import re
import socket
import webapp2

if USE_APPENGINE:
  from oauth2client import appengine
  from google.appengine.api import users
  from google.appengine.api import memcache

socket.setdefaulttimeout(60)
http = httplib2.Http(cache=memcache if USE_APPENGINE else None)

JINJA_ENVIRONMENT = jinja2.Environment(
    loader=jinja2.FileSystemLoader(os.path.dirname(__file__)),
    autoescape=True,
    extensions=['jinja2.ext.autoescape'])

client_secrets = os.path.join(os.path.dirname(__file__), 'client_secrets.json')

if USE_APPENGINE and REQUIRE_OAUTH:
  decorator = appengine.oauth2decorator_from_clientsecrets(
      client_secrets,
      scope=[
        'https://www.googleapis.com/auth/genomics',
      ])
else:
  class FakeOauthDecorator():
    def http(self):
      return http
    def oauth_aware(self, method):
      return method
    @property
    def callback_path(self):
      return '/unused'
    def callback_handler(self):
      pass
  decorator = FakeOauthDecorator()

# TODO: Dataset information should come from the list datasets api call
SUPPORTED_BACKENDS = {
  'NCBI' : {'name': 'NCBI',
            'url': 'http://trace.ncbi.nlm.nih.gov/Traces/gg',
            'datasets': {'SRP034507': 'SRP034507', 'SRP029392': 'SRP029392'}},
  'LOCAL' : {'name': 'Local',
             'url': 'http://localhost:5000',
             'datasets': {'All': ''}},
}
if REQUIRE_OAUTH:
  # Google temporarily requires OAuth on all calls
  SUPPORTED_BACKENDS['GOOGLE'] = {
    'name': 'Google',
    'url': 'https://www.googleapis.com/genomics/v1beta',
    'datasets': {'1000 Genomes': '376902546192', 'PGP': '383928317087'}
  }

class ApiException(Exception):
  pass

# Request handlers
class BaseRequestHandler(webapp2.RequestHandler):
  def handle_exception(self, exception, debug_mode):
    if isinstance(exception, ApiException):
      # ApiExceptions are expected, and will return nice error
      # messages to the client
      self.response.write(exception.message)
      self.response.set_status(400)
    else:
      # All other exceptions are unexpected and should be logged
      logging.exception('Unexpected exception')
      self.response.write('Unexpected internal exception')
      self.response.set_status(500)

  def get_backend(self):
    backend = self.request.get('backend')
    if not backend:
      raise ApiException('Backend parameter must be set')
    return backend

  def get_base_api_url(self):
    return SUPPORTED_BACKENDS[self.get_backend()]['url']

  def get_content(self, path, method='POST', body=None):
    http = decorator.http()
    response, content = http.request(
      uri="%s/%s" % (self.get_base_api_url(), path),
      method=method, body=json.dumps(body) if body else None,
      headers={'Content-Type': 'application/json; charset=UTF-8'})

    try:
      content = json.loads(content)
    except ValueError:
      logging.error("non-json api content %s" % content)
      raise ApiException('The API returned invalid JSON')

    if response.status >= 300:
      logging.error("error api response %s" % response)
      logging.error("error api content %s" % content)
      if 'error' in content:
        raise ApiException(content['error']['message'])
      else:
        raise ApiException('Something went wrong with the API call!')
    return content

  def write_content(self, path, method='POST', body=None):
    self.response.write(json.dumps(self.get_content(path, method, body)))

class ReadsetSearchHandler(BaseRequestHandler):

  @decorator.oauth_aware
  def get(self):
    readset_id = self.request.get('readsetId')
    if not readset_id:
      dataset_id = self.request.get('datasetId')
      name = self.request.get('name')
      if dataset_id:
        body = {'datasetIds' : [dataset_id]}
      else:
        # This is needed for the local readstore
        body = {'datasetIds' : []}
      response = self.get_content("readsets/search?fields=readsets(id,name)",
                                 body=body)

      # TODO: Use the api once name filtering is supported
      if name:
        name = name.lower()
        response['readsets'] = [r for r in response['readsets']
                                if name in r['name'].lower()]
      self.response.write(json.dumps(response))
      return

    # Single readset response
    self.write_content("readsets/%s" % readset_id, method='GET')

class ReadSearchHandler(BaseRequestHandler):

  @decorator.oauth_aware
  def get(self):
    body = {
      'readsetIds': self.request.get('readsetIds').split(','),
      'sequenceName': self.request.get('sequenceName'),
      'sequenceStart': max(0, int(self.request.get('sequenceStart'))),
      'sequenceEnd': int(self.request.get('sequenceEnd')),
    }
    pageToken = self.request.get('pageToken')
    if pageToken:
      body['pageToken'] = pageToken
    self.write_content("reads/search", body=body)

class BaseSnpediaHandler(webapp2.RequestHandler):
  def getSnppediaPageContent(self, snp):
    uri = "http://bots.snpedia.com/api.php?action=query&prop=revisions&" \
          "format=json&rvprop=content&titles=%s" % snp
    response, content = http.request(uri=uri)

    page_id, page = json.loads(content)['query']['pages'].popitem()
    return page['revisions'][0]['*']

  def getContentValue(self, content, key):
    try:
      matcher = '%s=(.*)\n' % key
      return re.search(matcher, content, re.I).group(1)
    except (KeyError, AttributeError):
      return ''

  def complement(self, base):
    return {'A': 'T', 'T': 'A', 'G': 'C', 'C': 'G'}[base]

class SnpSearchHandler(BaseSnpediaHandler):

  def getSnpResponse(self, name, content):
    return {
      'name': name,
      'link': 'http://www.snpedia.com/index.php/%s' % name,
      'position': self.getContentValue(content, 'position'),
      'chr': self.getContentValue(content, 'chromosome')
    }

  def get(self):
    snp = self.request.get('snp')

    try:
      content = self.getSnppediaPageContent(snp)
      if snp[:2].lower() == 'rs':
        snps = [self.getSnpResponse(snp, content)]
      else:
        # Try a gene format
        snps = re.findall('\[\[(rs\d+?)\]\]', content, re.I)
        snps = [self.getSnpResponse(s, self.getSnppediaPageContent(s))
                for s in set(snps)]

    except (ValueError, KeyError, AttributeError):
      snps = []
    self.response.write(json.dumps({'snps' : snps}))

class AlleleSearchHandler(BaseSnpediaHandler):

  def getAlleleResponse(self, name, content):
    return {
      'name': name,
      'link': 'http://www.snpedia.com/index.php/%s' % name,
      'repute': self.getContentValue(content, 'repute'),
      'summary': self.getContentValue(content, 'summary') or 'Unknown',
      'magnitude': self.getContentValue(content, 'magnitude')
    }

  def get(self):
    snp = self.request.get('snp')
    a1 = self.request.get('a1')
    a2 = self.request.get('a2')

    a1c = self.complement(a1)
    a2c = self.complement(a2)
    possible_names = [(snp, a1, a2), (snp, a2, a1),
                      (snp, a1c, a2c), (snp, a2c, a1c)]
    for name in possible_names:
      try:
        page = "%s(%s;%s)" % name
        content = self.getSnppediaPageContent(page)
        self.response.write(json.dumps(self.getAlleleResponse(page, content)))
        return
      except (ValueError, KeyError, AttributeError):
        pass # Continue trying the next allele name

    self.response.write(json.dumps({}))

class MainHandler(webapp2.RequestHandler):

  @decorator.oauth_aware
  def get(self):
    if not REQUIRE_OAUTH or decorator.has_credentials():
      template = JINJA_ENVIRONMENT.get_template('main.html')
      self.response.write(template.render({
        'username': users.User().nickname() if USE_APPENGINE else '',
        'logout_url': users.create_logout_url('/') if USE_APPENGINE else '',
        'backends': SUPPORTED_BACKENDS,
      }))
    else:
      # TODO: What kind of access do the non-google backends need?
      template = JINJA_ENVIRONMENT.get_template('grantaccess.html')
      self.response.write(template.render({
        'url': decorator.authorize_url()
      }))

web_app = webapp2.WSGIApplication(
    [
     ('/', MainHandler),
     ('/api/reads', ReadSearchHandler),
     ('/api/readsets', ReadsetSearchHandler),
     ('/api/snps', SnpSearchHandler),
     ('/api/alleles', AlleleSearchHandler),
     (decorator.callback_path, decorator.callback_handler()),
    ],
    debug=True)
<|endoftext|>