<commit_before>#    Copyright 2012, SIL International
#    All rights reserved.
#
#    This library is free software; you can redistribute it and/or modify
#    it under the terms of the GNU Lesser General Public License as published
#    by the Free Software Foundation; either version 2.1 of License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#    Lesser General Public License for more details.
#
#    You should also have received a copy of the GNU Lesser General Public
#    License along with this library in the file named "LICENSE".
#    If not, write to the Free Software Foundation, 51 Franklin Street,
#    suite 500, Boston, MA 02110-1335, USA or visit their web page on the
#    internet at http://www.fsf.org/licenses/lgpl.html.

from xml.etree.cElementTree import iterparse
from graide.makegdl.glyph import isMakeGDLSpecialClass
import os, traceback

class Gdx(object) :

    def __init__(self) :
        self.passes = []
        self.passTypes = []
        self.collisionFix=[]
        self.flipDirs = []
        self.keepelements = False

    def readfile(self, fname, font, autoGdlFile = None, apFileName = None, ronly = False) :
        relbase = os.path.relpath(os.path.dirname(fname) or ".") or ""

        self.file = open(fname)
        if not apFileName :  # autoGdlFile??
            font.initGlyphs()
        for (event, e) in iterparse(self.file, events=('start', 'end')) :
            if event == 'start' :
                if e.tag == 'pass' :
                    #print "table=",e.get('table'),"index=",e.get('index'),"flipDir=",e.get('flipDir'),"collisionFix",e.get('collisionFix'),"autoKern",e.get('autoKern')
                    self.passes.append([])
                    self.passTypes.append(e.get('table'))
                    self.collisionFix.append(int(e.get('collisionFix')) if e.get('collisionFix') else 0)
                    # ignore autoKern for now
                    self.flipDirs.append(int(e.get('flipDir')) if e.get('flipDir') else 0)

                elif e.tag in ('rule', 'glyph', 'class') :
                    self.keepelements = True
            else :
                if e.tag == "rule" :
                    self.keepelements = False
                    self.passes[-1].append(Rule(e, relbase))
                if e.tag == 'glyph' :
                    self.keepelements = False
                    font.addGdxGlyph(e, apFileName)
                elif e.tag == 'class' :
                    self.keepelements = False
                    n = e.get('name')
                    c = e.findall('member')
                    if len(c) :
                        g = font[int(c[0].get('glyphid'))]
                        fn = c[0].get('inFile')
                        f = os.path.join(relbase, fn) if fn else None
                        l = int(c[0].get('atLine')) if f else 0
                        if len(c) == 1 and g and g.GDLName() == n :
                            pass
                        elif not isMakeGDLSpecialClass(n) :
                            # Note: subtract 1 from the line number because the GDX file 1-based,
                            # the file editor is 0-based.
                            if not autoGdlFile :
                                font.addClass(n, map(lambda x: int(x.get('glyphid')), c), f, l - 1)
                            elif n not in font.classes or ronly :
                                font.addClass(n, map(lambda x: int(x.get('glyphid')), c), f, l - 1, generated = True)
                if not self.keepelements :
                    e.clear()

class Rule(object) :

    def __init__(self, e, relbase) :
        self.srcfile = os.path.join(relbase, e.get('inFile'))
        self.srcline = int(e.get('atLine')) - 1
        self.pretty = e.get('prettyPrint')
        slots = map(lambda x: int(x.get('slotIndex')), e.findall('rhsSlot'))
        if any(slots) :
            d = next(slots)
            self.slots = map(lambda x : x - d, slots)
        else :
            self.slots = {}
<commit_msg>Undo a list->dict change<commit_after>#    Copyright 2012, SIL International
#    All rights reserved.
#
#    This library is free software; you can redistribute it and/or modify
#    it under the terms of the GNU Lesser General Public License as published
#    by the Free Software Foundation; either version 2.1 of License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#    Lesser General Public License for more details.
#
#    You should also have received a copy of the GNU Lesser General Public
#    License along with this library in the file named "LICENSE".
#    If not, write to the Free Software Foundation, 51 Franklin Street,
#    suite 500, Boston, MA 02110-1335, USA or visit their web page on the
#    internet at http://www.fsf.org/licenses/lgpl.html.

from xml.etree.cElementTree import iterparse
from graide.makegdl.glyph import isMakeGDLSpecialClass
import os, traceback

class Gdx(object) :

    def __init__(self) :
        self.passes = []
        self.passTypes = []
        self.collisionFix=[]
        self.flipDirs = []
        self.keepelements = False

    def readfile(self, fname, font, autoGdlFile = None, apFileName = None, ronly = False) :
        relbase = os.path.relpath(os.path.dirname(fname) or ".") or ""

        self.file = open(fname)
        if not apFileName :  # autoGdlFile??
            font.initGlyphs()
        for (event, e) in iterparse(self.file, events=('start', 'end')) :
            if event == 'start' :
                if e.tag == 'pass' :
                    #print "table=",e.get('table'),"index=",e.get('index'),"flipDir=",e.get('flipDir'),"collisionFix",e.get('collisionFix'),"autoKern",e.get('autoKern')
                    self.passes.append([])
                    self.passTypes.append(e.get('table'))
                    self.collisionFix.append(int(e.get('collisionFix')) if e.get('collisionFix') else 0)
                    # ignore autoKern for now
                    self.flipDirs.append(int(e.get('flipDir')) if e.get('flipDir') else 0)

                elif e.tag in ('rule', 'glyph', 'class') :
                    self.keepelements = True
            else :
                if e.tag == "rule" :
                    self.keepelements = False
                    self.passes[-1].append(Rule(e, relbase))
                if e.tag == 'glyph' :
                    self.keepelements = False
                    font.addGdxGlyph(e, apFileName)
                elif e.tag == 'class' :
                    self.keepelements = False
                    n = e.get('name')
                    c = e.findall('member')
                    if len(c) :
                        g = font[int(c[0].get('glyphid'))]
                        fn = c[0].get('inFile')
                        f = os.path.join(relbase, fn) if fn else None
                        l = int(c[0].get('atLine')) if f else 0
                        if len(c) == 1 and g and g.GDLName() == n :
                            pass
                        elif not isMakeGDLSpecialClass(n) :
                            # Note: subtract 1 from the line number because the GDX file 1-based,
                            # the file editor is 0-based.
                            if not autoGdlFile :
                                font.addClass(n, map(lambda x: int(x.get('glyphid')), c), f, l - 1)
                            elif n not in font.classes or ronly :
                                font.addClass(n, map(lambda x: int(x.get('glyphid')), c), f, l - 1, generated = True)
                if not self.keepelements :
                    e.clear()

class Rule(object) :

    def __init__(self, e, relbase) :
        self.srcfile = os.path.join(relbase, e.get('inFile'))
        self.srcline = int(e.get('atLine')) - 1
        self.pretty = e.get('prettyPrint')
        slots = map(lambda x: int(x.get('slotIndex')), e.findall('rhsSlot'))
        if any(slots) :
            d = next(slots)
            self.slots = map(lambda x : x - d, slots)
        else :
            self.slots = []
<|endoftext|>


<commit_before>#!/usr/bin/env python

import csv
import random

# Checkout the README.md file for some notes on the script.

if __name__ == "__main__":

    players = {}
    teams = {
        'Sharks': { 'players': [] },
        'Dragons': { 'players': [] },
        'Raptors': { 'players': [] }
    }

    def load_players():
        with open('soccer_players.csv', 'r') as csv_file:
            csv_data = csv.DictReader(csv_file)
            for row in csv_data:
                players[row['Name']] = {
                    'height': row['Height (inches)'],
                    'experience': row['Soccer Experience'],
                    'guardian': row['Guardian Name(s)']
                }

    def players_who_have_experience(do_they_have_it):
        return_list = []
        for player_name, player_data in players.items():
            if player_data['experience'] == do_they_have_it:
                return_list.append(player_name)
        return return_list

    def put_players_on_team_randomly(player_list):
        while len(player_list):
            for team_key, team_data in teams.items():
                if len(player_list):
                    team_data['players'].append(player_list.pop(random.randint(0,len(player_list) - 1)))

    def output_team_file():
        with open('output/teams.txt', 'w') as teams_file:
            for team_key, team_data in teams.items():
                teams_file.write(team_key + '\n')
                for player in team_data['players']:
                    teams_file.write('{}, {}, {}'.format(player, players[player]['experience'], players[player]['guardian']) + '\n')
                teams_file.write('\n')

    load_players()
    put_players_on_team_randomly(players_who_have_experience('YES'))
    put_players_on_team_randomly(players_who_have_experience('NO'))
    output_team_file()

<commit_msg>Setup proper output file names and stubbing individual letter output.<commit_after>#!/usr/bin/env python

import csv
import random

# Checkout the README.md file for some notes on the script.

if __name__ == "__main__":

    players = {}
    teams = {
        'Sharks': { 'players': [] },
        'Dragons': { 'players': [] },
        'Raptors': { 'players': [] }
    }

    def load_players():
        with open('soccer_players.csv', 'r') as csv_file:
            csv_data = csv.DictReader(csv_file)
            for row in csv_data:
                players[row['Name']] = {
                    'height': row['Height (inches)'],
                    'experience': row['Soccer Experience'],
                    'guardian': row['Guardian Name(s)']
                }

    def players_who_have_experience(do_they_have_it):
        return_list = []
        for player_name, player_data in players.items():
            if player_data['experience'] == do_they_have_it:
                return_list.append(player_name)
        return return_list

    def put_players_on_team_randomly(player_list):
        while len(player_list):
            for team_key, team_data in teams.items():
                if len(player_list):
                    team_data['players'].append(player_list.pop(random.randint(0,len(player_list) - 1)))

    def output_team_file():
        with open('output/teams.txt', 'w') as teams_file:
            for team_key, team_data in teams.items():
                teams_file.write(team_key + '\n')
                for player in team_data['players']:
                    teams_file.write('{}, {}, {}'.format(player, players[player]['experience'], players[player]['guardian']) + '\n')
                teams_file.write('\n')

    def output_welcome_letters():
        for team_name, team_data in teams.items():
        	for player_name in team_data['players']:
        		file_name = player_name.lower().replace(' ', '_') + '.txt'
        		with open('letters/' + file_name, 'w') as letter:
        		    letter.write('Dear {}'.format('x'))

    load_players()
    put_players_on_team_randomly(players_who_have_experience('YES'))
    put_players_on_team_randomly(players_who_have_experience('NO'))
    output_team_file()
    output_welcome_letters()

<|endoftext|>


<commit_before>import numpy as np
import tensorflow as tf
from nltk.tokenize import word_tokenize
from termcolor import colored
import regex as re

from utils import log

def load_tokenized(text_file, preprocess=None):
    """
    Loads a tokenized text file a list of list of tokens.

    Args:

        text_file: An opened file.

        preprocess: A function/callable that (linguistically) preprocesses the
            sentences

    """

    if not preprocess:
        preprocess = lambda x: x

    return [preprocess(re.split(ur"[ ]", l.rstrip())) for l in text_file]

def tokenize_char_seq(chars):
    return word_tokenize("".join(chars))

def feed_dicts(dataset, batch_size, coders, train=False):
    """

    This function ensures all encoder and decoder objects feed their the data
    they need from the dataset.
    """
    dicts = [{} for _ in range(len(dataset) / batch_size + int(len(dataset) % batch_size > 0))]

    for coder in coders:
        coder.feed_dict(dataset, batch_size, dicts=dicts, train=train)

    return dicts

# TODO postprocess, copynet, beamsearch will be hidden in runner

def training_loop(sess, epochs, trainer, all_coders, decoder, batch_size,
                  train_dataset, val_dataset,
                  postprocess, log_directory,
                  evaluation_functions,
                  use_copynet=False,
                  test_dataset=None,
                  use_beamsearch=False,
                  initial_variables=None,
                  test_run=False):

    """

    Performs the training loop for given graph and data.

    Args:

        sess: TF Session.

        epochs: Number of epochs for which the algoritm will learn.

        trainer: The trainer object containg the TensorFlow code for computing
            the loss and optimization operation.

        decoder: The decoder object.

        train_dataset:

        val_dataset:

        postprocess: Function that takes the output sentence as produced by the
            decoder and transforms into tokenized sentence.

        log_directory: Directory where the TensordBoard log will be generated.
            If None, nothing will be done.

        evaluation_functions: List of evaluation functions. The last function
            is used as the main. Each function accepts list of decoded sequences
            and list of reference sequences and returns a float.

        use_copynet: Flag whether the copying mechanism is used.

        use_beamsearch:

        initial_variables: Either None or file where the variables are stored.
            Training then starts from the point the loaded values.

    """

    evaluation_labels = [f.__name__ for f in evaluation_functions]
    if postprocess is None:
        postprocess = lambda s: s
    log("Starting training")
    step = 0
    seen_instances = 0

    saver = tf.train.Saver()

    if initial_variables:
        saver.restore(sess, initial_variables)

    variables_file = log_directory+'/variables.data'
    saver.save(sess, variables_file)

    if log_directory:
        tb_writer = tf.train.SummaryWriter(log_directory, sess.graph_def)

    max_score = 0.0
    max_score_epoch = 0
    max_score_batch_no = 0

    def expand(feed_dict, state, hypotheses):
        feed_dict[decoder.encoded] = state
        hyp_len = len(hypotheses[0][1])
        hyp_count = len(hypotheses)
        if hyp_len == 2:
            for key in feed_dict:
                shape = key.get_shape()
                if not shape == tf.TensorShape(None):
                    if len(shape) == 1:
                        feed_dict[key] = np.repeat(feed_dict[key], hyp_count)
                    elif len(shape) == 2:
                        feed_dict[key] = np.repeat(np.array(feed_dict[key]), hyp_count, axis=0)
                    else:
                        log("ERROR in expanding beamsearch \
                            hypothesis")
        elif hyp_len >= 3: # we still need to expand the state
            feed_dict[decoder.encoded] = np.repeat(np.array(state), hyp_count, axis=0)

        for decoder_in, i in zip(decoder.gt_inputs, range(hyp_len)):
            if decoder_in not in feed_dict:
                feed_dict[decoder_in] = np.zeros(feed_dict[decoder.go_symbols].shape)
            for k in range(hyp_count):
                feed_dict[decoder_in][k] = hypotheses[k][1][i]
        probs, prob_i = sess.run([decoder.top10_probs[hyp_len - 1][0],
                                  decoder.top10_probs[hyp_len - 1][1]],
                                 feed_dict=feed_dict)
        beam = []
        for i in range(hyp_count):
            for p, x in zip(probs[i], prob_i[i]):
                beam.append((hypotheses[i][0] + p, hypotheses[i][1] + [x]))
        return beam

    def beamsearch(fd):
        beam = [(1.0, [1])]
        state = sess.run(decoder.encoded, fd)
        for _ in range(len(decoder.decoded_probs)):
            new_beam = expand(fd, state, beam)
            new_beam.sort(reverse=True)
            beam = new_beam[:10]
        return beam[0][1]

    def copynet_substitute(decoded_sentences, copy_sentences, computation):
        """
        Substitutes the <unk> tokens with the tokens from the source encoder we are
        copying from.
        """
        copy_logits = computation[-(2*decoder.max_output_len)-2 : -decoder.max_output_len - 1]
        assert len(copy_logits) == decoder.max_output_len + 1 ## kdyby nahodou
        #assert len(computation) -(2*decoder.max_output_len)-2 == 5 ## neplati pri validaci
        assert len(decoded_sentences) == len(copy_sentences)

        for i, (s, copy_s) in enumerate(zip(decoded_sentences, copy_sentences)):
            for j, w in enumerate(s):
                if w == '<unk>':
                    selected = np.argmax(copy_logits[j][i])

                    ## Copynet can generate <pad> tokens from outside the sentence
                    if selected < len(copy_s) and selected != 0:
                        decoded_sentences[i][j] = copy_s[selected-1]

        return decoded_sentences

    val_feed_dicts = feed_dicts(val_dataset, batch_size, all_coders, train=False)
    val_tgt_sentences = val_dataset.series[decoder.data_id]

    try:
        for i in range(epochs):
            print ""
            log("Epoch {} starts".format(i + 1), color='red')

            train_dataset.shuffle()
            train_feed_dicts = feed_dicts(train_dataset, batch_size, all_coders, train=True)
            batched_targets = train_dataset.batch_serie(decoder.data_id, batch_size)

            for batch_n, (batch_feed_dict, batch_sentences) in \
                    enumerate(zip(train_feed_dicts, batched_targets)):

                step += 1
                seen_instances += len(batch_sentences)
                if step % 20 == 1:

                    computation = trainer.run(sess, batch_feed_dict, batch_sentences, verbose=True)

                    decoded_sentences = \
                            decoder.vocabulary.vectors_to_sentences(\
                            computation[-decoder.max_output_len - 1:])

                    decoded_sentences = [postprocess(s) for s in decoded_sentences]

                    evaluation_result = \
                            [f(decoded_sentences, batch_sentences) for f in evaluation_functions]

                    eval_string = "    ".join(["{}: {:.2f}".format(name, value) for name, value \
                        in zip(evaluation_labels, evaluation_result)])

                    log("opt. loss: {:.4f}    dec. loss: {:.4f}    ".\
                            format(computation[2], computation[1]) + eval_string)

                    if log_directory:
                        summary_str = computation[3]
                        tb_writer.add_summary(summary_str, seen_instances)
                        #histograms_str = computation[4]
                        #tb_writer.add_summary(histograms_str, seen_instances)
                        external_str = \
                                tf.Summary(value=[tf.Summary.Value(tag="train_"+name,
                                                                   simple_value=value) \
                                for name, value in zip(evaluation_labels, evaluation_result)])

                        tb_writer.add_summary(external_str, seen_instances)
                else:
                    trainer.run(sess, batch_feed_dict, batch_sentences, verbose=False)

                if step % 500 == (61 if test_run else 499):
                    decoded_val_sentences = []

                    for val_batch_n, (val_batch_feed_dict) in enumerate(val_feed_dicts):
                        if use_beamsearch and val_batch_n % 100 == 99:
                             decoded_val_sentences.append(beamsearch(val_batch_feed_dict))
                             log("Beamsearch: " + str(val_batch_n) + " sentences done.")
                        else:
                            computation = sess.run([decoder.loss_with_decoded_ins,
                                                    decoder.loss_with_gt_ins, trainer.summary_val] \
                                                     + decoder.copynet_logits + decoder.decoded_seq,
                                                   feed_dict=val_batch_feed_dict)
                            decoded_val_sentences_batch = decoder.vocabulary.vectors_to_sentences(computation[-decoder.max_output_len - 1:])

                            #if use_copynet: # TODO beamsearch nefunguje s copynetem
                            #    decoded_val_sentences_batch = \
                            #            copynet_substitute(decoded_val_sentences_batch, val_copy_sentences, computation)

                        decoded_val_sentences += decoded_val_sentences_batch

                    decoded_val_sentences = [postprocess(s) for s in decoded_val_sentences]
                    evaluation_result = \
                            [f(decoded_val_sentences, val_tgt_sentences) for f in evaluation_functions]

                    eval_string = "    ".join(["{}: {:.2f}".format(name, value) for name, value \
                        in zip(evaluation_labels, evaluation_result)])

                    if evaluation_result[-1] > max_score:
                        max_score = evaluation_result[-1]
                        max_score_epoch = i
                        max_score_batch_no = batch_n
                        saver.save(sess, variables_file)

                    print ""
                    log("Validation (epoch {}, batch number {}):".format(i + 1, batch_n), color='cyan')
                    log("opt. loss: {:.4f}    dec. loss: {:.4f}    "\
                            .format(computation[1], computation[0]) + eval_string, color='cyan')
                    log("max {} on validation: {:.2f} (in epoch {}, after batch number {})".\
                            format(evaluation_labels[-1], max_score,
                                   max_score_epoch, max_score_batch_no), color='cyan')

                    print ""
                    print "Examples:"
                    for sent, ref_sent in zip(decoded_val_sentences[:15], val_tgt_sentences):
                        print u"    {}".format(u" ".join(sent))
                        print colored(u"      ref.: {}".format(u" ".join(ref_sent)), color="magenta")
                    print ""

                    if log_directory:
                        summary_str = computation[2]
                        tb_writer.add_summary(summary_str, seen_instances)
                        tb_writer.add_summary(external_str, seen_instances)
                        external_str = \
                            tf.Summary(value=[tf.Summary.Value(tag="val_"+name, simple_value=value)\
                                              for name, value in zip(evaluation_labels,
                                                                     evaluation_result)])

                        tb_writer.add_summary(external_str, seen_instances)
    except KeyboardInterrupt:
        log("Training interrupted by user.")

    saver.restore(sess, variables_file)
    log("Training finished. Maximum {} on validation data: {:.2f}, epoch {}".format(evaluation_labels[-1], max_score, max_score_epoch))

#    if test_feed_dicts and batched_test_copy_sentences and test_output_file:
#        log("Translating test data and writing to {}".format(test_output_file))
#        decoded_test_sentences = []
#
#        for i, (test_feed_dict, test_copy_sentences) in enumerate(zip(test_feed_dicts, batched_test_copy_sentences)):
#            if use_beamsearch:
#                 decoded_test_sentence_indices = beamsearch(test_feed_dict)
#                 decoded_test_sentences_batch = [[decoder.decoder.vocabulary.index_to_word[i] \
#                         for i in decoded_test_sentence_indices][1:]]
#                 log(decoded_test_sentences_batch)
#            else:
#                computation = sess.run(decoder.copynet_logits + decoder.decoded_seq, feed_dict=test_feed_dict)
#                decoded_test_sentences_batch = decoder.vocabulary.vectors_to_sentences(computation[-decoder.max_output_len - 1:])
#
#                #if use_copynet: # TODO beamsearch (porad) nefunguje s copynetem
#                #    decoded_test_sentences_batch = \
#                #        copynet_substitute(decoded_test_sentences_batch, test_copy_sentences, computation)
#
#            decoded_test_sentences += [postprocess(s) for s in decoded_test_sentences_batch]
#
#            with open(test_output_file.name, test_output_file.mode) as fout:
#
#                for sent in decoded_test_sentences:
#                    fout.write("{}\n".format(" ".join(sent)))
#                fout.close()

    log("Finished.")

<commit_msg>refactor beamsearch<commit_after>import numpy as np
import tensorflow as tf
from nltk.tokenize import word_tokenize
from termcolor import colored
import regex as re

from utils import log

def load_tokenized(text_file, preprocess=None):
    """
    Loads a tokenized text file a list of list of tokens.

    Args:

        text_file: An opened file.

        preprocess: A function/callable that (linguistically) preprocesses the
            sentences

    """

    if not preprocess:
        preprocess = lambda x: x

    return [preprocess(re.split(ur"[ ]", l.rstrip())) for l in text_file]

def tokenize_char_seq(chars):
    return word_tokenize("".join(chars))

def feed_dicts(dataset, batch_size, coders, train=False):
    """

    This function ensures all encoder and decoder objects feed their the data
    they need from the dataset.
    """
    dicts = [{} for _ in range(len(dataset) / batch_size + int(len(dataset) % batch_size > 0))]

    for coder in coders:
        coder.feed_dict(dataset, batch_size, dicts=dicts, train=train)

    return dicts

# TODO postprocess, copynet, beamsearch will be hidden in runner

def expand(session, decoder, feed_dict, state, hypotheses):
    feed_dict[decoder.encoded] = state
    hyp_length = len(hypotheses[0][1])
    hyp_count = len(hypotheses)
    if hyp_length == 2:
        for k in feed_dict:
            sh = k.get_shape()
            if not sh == tf.TensorShape(None):
                if len(sh) == 1:
                    feed_dict[k] = np.repeat(feed_dict[k], hyp_count)
                elif len(sh) == 2:
                    feed_dict[k] = np.repeat(np.array(feed_dict[k]), hyp_count, axis=0)
                else:
                    log("ERROR in expanding beamsearch hypothesis")
    elif hyp_length > 2:
        feed_dict[decoder.encoded] = np.repeat(state, hyp_count, axis=0)

    for i, n in zip(decoder.gt_inputs, range(hyp_length)):
        for k in range(hyp_count):
            feed_dict[i][k] = hypotheses[k][1][n]
    probs, prob_i = session.run([decoder.top10_probs[hyp_length - 1][0],
                     decoder.top10_probs[hyp_length - 1][1]],
                     feed_dict=feed_dict)
    beam = []
    for i in range(hyp_count):
        for p, x in zip(probs[i], prob_i[i]):
            beam.append((hypotheses[i][0] + p, hypotheses[i][1] + [x]))
    return beam

def beamsearch(session, decoder, feed_dict):
    beam = [(1.0, [1])]
    state = session.run(decoder.encoded, feed_dict)
    for _ in range(len(decoder.decoded_probs)):
         new_beam = expand(session, decoder, feed_dict, state, beam)
         new_beam.sort(reverse=True)
         beam = new_beam[:10]
    return beam[0][1]

def training_loop(sess, epochs, trainer, all_coders, decoder, batch_size,
                  train_dataset, val_dataset,
                  postprocess, log_directory,
                  evaluation_functions,
                  use_copynet=False,
                  test_dataset=None,
                  use_beamsearch=False,
                  initial_variables=None,
                  test_run=False):

    """

    Performs the training loop for given graph and data.

    Args:

        sess: TF Session.

        epochs: Number of epochs for which the algoritm will learn.

        trainer: The trainer object containg the TensorFlow code for computing
            the loss and optimization operation.

        decoder: The decoder object.

        train_dataset:

        val_dataset:

        postprocess: Function that takes the output sentence as produced by the
            decoder and transforms into tokenized sentence.

        log_directory: Directory where the TensordBoard log will be generated.
            If None, nothing will be done.

        evaluation_functions: List of evaluation functions. The last function
            is used as the main. Each function accepts list of decoded sequences
            and list of reference sequences and returns a float.

        use_copynet: Flag whether the copying mechanism is used.

        use_beamsearch:

        initial_variables: Either None or file where the variables are stored.
            Training then starts from the point the loaded values.

    """

    evaluation_labels = [f.__name__ for f in evaluation_functions]
    if postprocess is None:
        postprocess = lambda s: s
    log("Starting training")
    step = 0
    seen_instances = 0

    saver = tf.train.Saver()

    if initial_variables:
        saver.restore(sess, initial_variables)

    variables_file = log_directory+'/variables.data'
    saver.save(sess, variables_file)

    if log_directory:
        tb_writer = tf.train.SummaryWriter(log_directory, sess.graph_def)

    max_score = 0.0
    max_score_epoch = 0
    max_score_batch_no = 0

    def expand(feed_dict, state, hypotheses):
        feed_dict[decoder.encoded] = state
        hyp_len = len(hypotheses[0][1])
        hyp_count = len(hypotheses)
        if hyp_len == 2:
            for key in feed_dict:
                shape = key.get_shape()
                if not shape == tf.TensorShape(None):
                    if len(shape) == 1:
                        feed_dict[key] = np.repeat(feed_dict[key], hyp_count)
                    elif len(shape) == 2:
                        feed_dict[key] = np.repeat(np.array(feed_dict[key]), hyp_count, axis=0)
                    else:
                        log("ERROR in expanding beamsearch \
                            hypothesis")
        elif hyp_len >= 3: # we still need to expand the state
            feed_dict[decoder.encoded] = np.repeat(np.array(state), hyp_count, axis=0)

        for decoder_in, i in zip(decoder.gt_inputs, range(hyp_len)):
            if decoder_in not in feed_dict:
                feed_dict[decoder_in] = np.zeros(feed_dict[decoder.go_symbols].shape)
            for k in range(hyp_count):
                feed_dict[decoder_in][k] = hypotheses[k][1][i]
        probs, prob_i = sess.run([decoder.top10_probs[hyp_len - 1][0],
                                  decoder.top10_probs[hyp_len - 1][1]],
                                 feed_dict=feed_dict)
        beam = []
        for i in range(hyp_count):
            for p, x in zip(probs[i], prob_i[i]):
                beam.append((hypotheses[i][0] + p, hypotheses[i][1] + [x]))
        return beam

    def beamsearch(fd):
        beam = [(1.0, [1])]
        state = sess.run(decoder.encoded, fd)
        for _ in range(len(decoder.decoded_probs)):
            new_beam = expand(fd, state, beam)
            new_beam.sort(reverse=True)
            beam = new_beam[:10]
        return beam[0][1]

    def copynet_substitute(decoded_sentences, copy_sentences, computation):
        """
        Substitutes the <unk> tokens with the tokens from the source encoder we are
        copying from.
        """
        copy_logits = computation[-(2*decoder.max_output_len)-2 : -decoder.max_output_len - 1]
        assert len(copy_logits) == decoder.max_output_len + 1 ## kdyby nahodou
        #assert len(computation) -(2*decoder.max_output_len)-2 == 5 ## neplati pri validaci
        assert len(decoded_sentences) == len(copy_sentences)

        for i, (s, copy_s) in enumerate(zip(decoded_sentences, copy_sentences)):
            for j, w in enumerate(s):
                if w == '<unk>':
                    selected = np.argmax(copy_logits[j][i])

                    ## Copynet can generate <pad> tokens from outside the sentence
                    if selected < len(copy_s) and selected != 0:
                        decoded_sentences[i][j] = copy_s[selected-1]

        return decoded_sentences

    val_feed_dicts = feed_dicts(val_dataset, batch_size, all_coders, train=False)
    val_tgt_sentences = val_dataset.series[decoder.data_id]

    try:
        for i in range(epochs):
            print ""
            log("Epoch {} starts".format(i + 1), color='red')

            train_dataset.shuffle()
            train_feed_dicts = feed_dicts(train_dataset, batch_size, all_coders, train=True)
            batched_targets = train_dataset.batch_serie(decoder.data_id, batch_size)

            for batch_n, (batch_feed_dict, batch_sentences) in \
                    enumerate(zip(train_feed_dicts, batched_targets)):

                step += 1
                seen_instances += len(batch_sentences)
                if step % 20 == 1:

                    computation = trainer.run(sess, batch_feed_dict, batch_sentences, verbose=True)

                    decoded_sentences = \
                            decoder.vocabulary.vectors_to_sentences(\
                            computation[-decoder.max_output_len - 1:])

                    decoded_sentences = [postprocess(s) for s in decoded_sentences]

                    evaluation_result = \
                            [f(decoded_sentences, batch_sentences) for f in evaluation_functions]

                    eval_string = "    ".join(["{}: {:.2f}".format(name, value) for name, value \
                        in zip(evaluation_labels, evaluation_result)])

                    log("opt. loss: {:.4f}    dec. loss: {:.4f}    ".\
                            format(computation[2], computation[1]) + eval_string)

                    if log_directory:
                        summary_str = computation[3]
                        tb_writer.add_summary(summary_str, seen_instances)
                        #histograms_str = computation[4]
                        #tb_writer.add_summary(histograms_str, seen_instances)
                        external_str = \
                                tf.Summary(value=[tf.Summary.Value(tag="train_"+name,
                                                                   simple_value=value) \
                                for name, value in zip(evaluation_labels, evaluation_result)])

                        tb_writer.add_summary(external_str, seen_instances)
                else:
                    trainer.run(sess, batch_feed_dict, batch_sentences, verbose=False)

                if step % 500 == (61 if test_run else 499):
                    decoded_val_sentences = []

                    for val_batch_n, (val_batch_feed_dict) in enumerate(val_feed_dicts):
                        if use_beamsearch and val_batch_n % 100 == 99:
                             decoded_val_sentences.append(beamsearch(val_batch_feed_dict))
                             log("Beamsearch: " + str(val_batch_n) + " sentences done.")
                        else:
                            computation = sess.run([decoder.loss_with_decoded_ins,
                                                    decoder.loss_with_gt_ins, trainer.summary_val] \
                                                     + decoder.copynet_logits + decoder.decoded_seq,
                                                   feed_dict=val_batch_feed_dict)
                            decoded_val_sentences_batch = decoder.vocabulary.vectors_to_sentences(computation[-decoder.max_output_len - 1:])

                            #if use_copynet: # TODO beamsearch nefunguje s copynetem
                            #    decoded_val_sentences_batch = \
                            #            copynet_substitute(decoded_val_sentences_batch, val_copy_sentences, computation)

                        decoded_val_sentences += decoded_val_sentences_batch

                    decoded_val_sentences = [postprocess(s) for s in decoded_val_sentences]
                    evaluation_result = \
                            [f(decoded_val_sentences, val_tgt_sentences) for f in evaluation_functions]

                    eval_string = "    ".join(["{}: {:.2f}".format(name, value) for name, value \
                        in zip(evaluation_labels, evaluation_result)])

                    if evaluation_result[-1] > max_score:
                        max_score = evaluation_result[-1]
                        max_score_epoch = i
                        max_score_batch_no = batch_n
                        saver.save(sess, variables_file)

                    print ""
                    log("Validation (epoch {}, batch number {}):".format(i + 1, batch_n), color='cyan')
                    log("opt. loss: {:.4f}    dec. loss: {:.4f}    "\
                            .format(computation[1], computation[0]) + eval_string, color='cyan')
                    log("max {} on validation: {:.2f} (in epoch {}, after batch number {})".\
                            format(evaluation_labels[-1], max_score,
                                   max_score_epoch, max_score_batch_no), color='cyan')

                    print ""
                    print "Examples:"
                    for sent, ref_sent in zip(decoded_val_sentences[:15], val_tgt_sentences):
                        print u"    {}".format(u" ".join(sent))
                        print colored(u"      ref.: {}".format(u" ".join(ref_sent)), color="magenta")
                    print ""

                    if log_directory:
                        summary_str = computation[2]
                        tb_writer.add_summary(summary_str, seen_instances)
                        tb_writer.add_summary(external_str, seen_instances)
                        external_str = \
                            tf.Summary(value=[tf.Summary.Value(tag="val_"+name, simple_value=value)\
                                              for name, value in zip(evaluation_labels,
                                                                     evaluation_result)])

                        tb_writer.add_summary(external_str, seen_instances)
    except KeyboardInterrupt:
        log("Training interrupted by user.")

    saver.restore(sess, variables_file)
    log("Training finished. Maximum {} on validation data: {:.2f}, epoch {}".format(evaluation_labels[-1], max_score, max_score_epoch))

#    if test_feed_dicts and batched_test_copy_sentences and test_output_file:
#        log("Translating test data and writing to {}".format(test_output_file))
#        decoded_test_sentences = []
#
#        for i, (test_feed_dict, test_copy_sentences) in enumerate(zip(test_feed_dicts, batched_test_copy_sentences)):
#            if use_beamsearch:
#                 decoded_test_sentence_indices = beamsearch(test_feed_dict)
#                 decoded_test_sentences_batch = [[decoder.decoder.vocabulary.index_to_word[i] \
#                         for i in decoded_test_sentence_indices][1:]]
#                 log(decoded_test_sentences_batch)
#            else:
#                computation = sess.run(decoder.copynet_logits + decoder.decoded_seq, feed_dict=test_feed_dict)
#                decoded_test_sentences_batch = decoder.vocabulary.vectors_to_sentences(computation[-decoder.max_output_len - 1:])
#
#                #if use_copynet: # TODO beamsearch (porad) nefunguje s copynetem
#                #    decoded_test_sentences_batch = \
#                #        copynet_substitute(decoded_test_sentences_batch, test_copy_sentences, computation)
#
#            decoded_test_sentences += [postprocess(s) for s in decoded_test_sentences_batch]
#
#            with open(test_output_file.name, test_output_file.mode) as fout:
#
#                for sent in decoded_test_sentences:
#                    fout.write("{}\n".format(" ".join(sent)))
#                fout.close()

    log("Finished.")

<|endoftext|>


<commit_before>import json

import requests
from tabulate import tabulate

from lecli import apiutils

def _url(endpoint):
    """
    Get rest query url of account resource id.
    """
    if endpoint == 'owner':
        return 'https://rest.logentries.com/management/accounts/' + str(
            apiutils.get_account_resource_id()) + '/owners'
    elif endpoint == 'user':
        return 'https://rest.logentries.com/management/accounts/' + str(
            apiutils.get_account_resource_id()) + '/users'

def response_error(response):
    """
    Check response if it has any errors.
    """
    if response.headers.get('X-RateLimit-Remaining') is not None:
        if int(response.headers['X-RateLimit-Remaining']) == 0:
            print 'Error: Rate Limit Reached, will reset in ' + response.headers.get(
                'X-RateLimit-Reset') + ' seconds'
            return True
    try:
        response.raise_for_status()
    except requests.exceptions.HTTPError as error:
        print "Request Error:", error.message
        if response.status_code == 500:
            print 'Your account may have no owner assigned. ' \
                  'Please visit www.logentries.com for information on assigning an account owner.'
        return True

    if response.status_code == 200:
        if response.headers['Content-Type'] != 'application/json':
            print 'Unexpected Content Type Received in Response: ' + response.headers[
                'Content-Type']
            return True
        else:
            return False
    return False

def handle_userlist_response(response):
    """
    Handle userlist response. Exit if it has any errors, print if status code is 200.
    """
    if response_error(response) is True:  # Check response has no errors
        exit(1)
    elif response.status_code == 200:
        print_users(response)

def handle_create_user_response(response):
    """
    Handle create user response. If it has any errors, print help.
    """
    if response_error(response) is True:  # Check response has no errors
        if response.status_code >= 400:
            print 'Failed to add user - User may have already been added this account or have a ' \
                  'Logentries account'
            print 'To add a new user: lecli useradd -f John -l Smyth -e john@smyth.com'
            print 'To add an existing user using their user ID: lecli useradd -u ' \
                  '12345678-aaaa-bbbb-1234-1234cb123456'
        exit(1)

    if response.status_code == 200:
        try:
            print 'Added user to account:\n' + \
              'Name: ' + response.json()['first_name'] + ' ' + response.json()['last_name'] + '\n' \
              'Login: ' + response.json()['login_name'] + '\n' \
              'Email: ' + response.json()['email'] + '\n' \
              'User ID: ' + response.json()['id']
        except ValueError:
            print 'Error: User already in account'

    if response.status_code == 201:
        print 'Added user to account:\n' + \
              'Name: ' + response.json()['first_name'] + ' ' + response.json()['last_name'] + '\n' \
              'Login: ' + response.json()['login_name'] + '\n' \
              'Email: ' + response.json()['email'] + '\n' \
              'User ID: ' + response.json()['id']

    if response.status_code == 403:
        print "User you attempted to add is the account owner"

def list_users():
    """
    List users that is in the current account.
    """
    action = 'management/accounts/' + str(apiutils.get_account_resource_id()) + '/users'
    try:
        response = requests.request('GET', _url('user'), headers=apiutils.generate_headers(
            'owner', 'GET', action, ''))
        handle_userlist_response(response)
    except requests.exceptions.RequestException as error:
        print error
        exit(1)

def add_new_user(first_name, last_name, email):
    """
    Add a new user to the current account.
    """
    action = 'management/accounts/' + str(apiutils.get_account_resource_id()) + '/users'
    body = {
        "email": str(email),
        "first_name": str(first_name),
        "last_name": str(last_name)
    }
    body = json.dumps(body, separators=(',', ':'))
    headers = apiutils.generate_headers('owner', method='POST', action=action, body=body)

    try:
        response = requests.request('POST', _url('user'), data=body, headers=headers)
        handle_create_user_response(response)
    except requests.exceptions.RequestException as error:
        print error
        exit(1)

def add_existing_user(user_id):
    """
    Add a user that already exist to the current account.
    """
    url = _url('user') + '/' + str(user_id)
    action = url.split("com/")[1]
    headers = apiutils.generate_headers('owner', method='POST', action=action, body='')

    try:
        response = requests.request('POST', url, data='', headers=headers)
        handle_create_user_response(response)
    except requests.exceptions.RequestException as error:
        print error
        exit(1)

def delete_user(user_id):
    """
    Delete a user from the current account.
    """
    url = _url('user') + '/' + str(user_id)
    action = url.split("com/")[1]
    headers = apiutils.generate_headers('owner', method='DELETE', action=action, body='')

    try:
        response = requests.request('DELETE', url, data='', headers=headers)
        if response_error(response) is True:  # Check response has no errors
            print 'Delete user failed, status code: ' + str(response.status_code)
            exit(1)
        elif response.status_code == 204:
            print 'Deleted user'
    except requests.exceptions.RequestException as error:
        print error
        exit(1)

def get_owner():
    """
    Get owner information of the current account.
    """
    action = 'management/accounts/' + str(apiutils.get_account_resource_id()) + '/owners'
    try:
        response = requests.request('GET', _url('owner'), headers=apiutils.generate_headers(
            'owner', 'GET', action, ''))
        handle_userlist_response(response)
    except requests.exceptions.RequestException as error:
        print error
        exit(1)

def print_users(response):
    """
    Print users in the current account.
    """
    if 'users' in response.json():
        print tabulate(response.json()['users'], headers={})
    elif 'owners' in response.json():
        print tabulate(response.json()['owners'], headers={})
<commit_msg>create user api<commit_after>import json

import requests
from tabulate import tabulate

from lecli import apiutils

def _url(endpoint):
    """
    Get rest query url of account resource id.
    """
    if endpoint == 'owner':
        return 'https://rest.logentries.com/management/accounts/' + str(
            apiutils.get_account_resource_id()) + '/owners'
    elif endpoint == 'user':
        return 'https://rest.logentries.com/management/accounts/' + str(
            apiutils.get_account_resource_id()) + '/users'

def response_error(response):
    """
    Check response if it has any errors.
    """
    if response.headers.get('X-RateLimit-Remaining') is not None:
        if int(response.headers['X-RateLimit-Remaining']) == 0:
            print 'Error: Rate Limit Reached, will reset in ' + response.headers.get(
                'X-RateLimit-Reset') + ' seconds'
            return True
    try:
        response.raise_for_status()
    except requests.exceptions.HTTPError as error:
        print "Request Error:", error.message
        if response.status_code == 500:
            print 'Your account may have no owner assigned. ' \
                  'Please visit www.logentries.com for information on assigning an account owner.'
        return True

    if response.status_code == 200:
        if response.headers['Content-Type'] != 'application/json':
            print 'Unexpected Content Type Received in Response: ' + response.headers[
                'Content-Type']
            return True
        else:
            return False
    return False

def handle_userlist_response(response):
    """
    Handle userlist response. Exit if it has any errors, print if status code is 200.
    """
    if response_error(response) is True:  # Check response has no errors
        exit(1)
    elif response.status_code == 200:
        print_users(response)

def handle_create_user_response(response):
    """
    Handle create user response. If it has any errors, print help.
    """
    if response_error(response) is True:  # Check response has no errors
        if response.status_code >= 400:
            print 'Failed to add user - User may have already been added this account or have a ' \
                  'Logentries account'
            print 'To add a new user: lecli useradd -f John -l Smyth -e john@smyth.com'
            print 'To add an existing user using their user ID: lecli useradd -u ' \
                  '12345678-aaaa-bbbb-1234-1234cb123456'
        exit(1)

    if response.status_code == 200:
        user = response.json()['user']
        print 'Added user to account:\nName: %s %s \nLogin: %s \nEmail: %s \nUser ID: %s' % \
              (user['first_name'], user['last_name'], user['login_name'], user['email'], user['id'])

    if response.status_code == 201:
        print response.json()
        user = response.json()['user']
        print 'Added user to account:\nName: %s %s \nLogin: %s \nEmail: %s \nUser ID: %s' % \
              (user['first_name'], user['last_name'], user['login_name'], user['email'], user['id'])

    if response.status_code == 403:
        print "User you attempted to add is the account owner"

def list_users():
    """
    List users that is in the current account.
    """
    action = 'management/accounts/' + str(apiutils.get_account_resource_id()) + '/users'
    try:
        response = requests.request('GET', _url('user'), headers=apiutils.generate_headers(
            'owner', 'GET', action, ''))
        handle_userlist_response(response)
    except requests.exceptions.RequestException as error:
        print error
        exit(1)

def add_new_user(first_name, last_name, email):
    """
    Add a new user to the current account.
    """
    action = 'management/accounts/' + str(apiutils.get_account_resource_id()) + '/users'
    json_content = {
        "user": {"email": str(email),
                 "first_name": str(first_name),
                 "last_name": str(last_name)
                 }
    }
    body = json.dumps(json_content)
    headers = apiutils.generate_headers('owner', method='POST', action=action, body=body)

    try:
        response = requests.request('POST', _url('user'), json=json_content, headers=headers)
        handle_create_user_response(response)
    except requests.exceptions.RequestException as error:
        print error
        exit(1)

def add_existing_user(user_id):
    """
    Add a user that already exist to the current account.
    """
    url = _url('user') + '/' + str(user_id)
    action = url.split("com/")[1]
    headers = apiutils.generate_headers('owner', method='POST', action=action, body='')

    try:
        response = requests.request('POST', url, data='', headers=headers)
        handle_create_user_response(response)
    except requests.exceptions.RequestException as error:
        print error
        exit(1)

def delete_user(user_id):
    """
    Delete a user from the current account.
    """
    url = _url('user') + '/' + str(user_id)
    action = url.split("com/")[1]
    headers = apiutils.generate_headers('owner', method='DELETE', action=action, body='')

    try:
        response = requests.request('DELETE', url, data='', headers=headers)
        if response_error(response) is True:  # Check response has no errors
            print 'Delete user failed, status code: ' + str(response.status_code)
            exit(1)
        elif response.status_code == 204:
            print 'Deleted user'
    except requests.exceptions.RequestException as error:
        print error
        exit(1)

def get_owner():
    """
    Get owner information of the current account.
    """
    action = 'management/accounts/' + str(apiutils.get_account_resource_id()) + '/owners'
    try:
        response = requests.request('GET', _url('owner'), headers=apiutils.generate_headers(
            'owner', 'GET', action, ''))
        handle_userlist_response(response)
    except requests.exceptions.RequestException as error:
        print error
        exit(1)

def print_users(response):
    """
    Print users in the current account.
    """
    if 'users' in response.json():
        print tabulate(response.json()['users'], headers={})
    elif 'owners' in response.json():
        print tabulate(response.json()['owners'], headers={})
<|endoftext|>


<commit_before>#Copyright (c) 2011 Fabula Solutions. All rights reserved.
#Use of this source code is governed by a BSD-style license that can be
#found in the license.txt file.

# leveldb server
import threading
import zmq
import leveldb
import json
import optparse

class workerThread(threading.Thread):
    """workerThread"""
    def __init__(self, context, db):
        threading.Thread.__init__ (self)
        self.context = context
        self.db = db

    def run(self):
        socket = self.context.socket(zmq.XREQ)
        socket.connect('inproc://backend')
        while True:
            msg = socket.recv_multipart()
            if  len(msg) != 3:
                value = 'None'
                reply = [msg[0], value]
                socket.send_multipart(reply)
                continue
            id = msg[0]
            op = msg[1]
            data = json.loads(msg[2])
            reply = [id]
            if op == 'get':
                try:
                    value = self.db.Get(data)
                except:
                    value = ""
                reply.append(value)

            elif op == 'put':
                try:
                    self.db.Put(data[0], data[1])
                    value = "True"
                except:
                    value = ""
                reply.append(value)

            elif op == 'delete':
                self.db.Delete(data)
                value = ""
                reply.append(value)

            elif op == 'range':
                start = data[0]
                end = data[1]
                if start and end:
                    try:
                        arr = []
                        for value in self.db.RangeIter(start, end):
                            arr.append({value[0]: value[1]})
                        reply.append(json.dumps(arr))
                    except:
                        value = ""
                        reply.append(value)
                else:
                    try:
                        arr = []
                        for value in self.db.RangeIter():
                            arr.append({value[0]: value[1]})
                        reply.append(json.dumps(arr))
                    except:
                        value = ""
                        reply.append(value)
            else:
                value = ""
                reply.append(value)
            #print reply
            socket.send_multipart(reply)

        socket.close()

if __name__ == "__main__":
    optparser = optparse.OptionParser(
        prog='leveldb-server.py',
        version='0.1.1',
        description='leveldb-server',
        usage='%prog \n\t-p [port and host settings] Default: tcp://127.0.0.1:5147\n' + \
                    '\t-f [database file name] Default: level.db')
    optparser.add_option('--host', '-p', dest='host',
        default='tcp://127.0.0.1:5147')
    optparser.add_option('--dbfile', '-d', dest='dbfile',
        default='level.db')
    options, arguments = optparser.parse_args()

    if not (options.host and options.dbfile):
        optparser.print_help()

    print "Starting leveldb-server %s" % options.host
    context = zmq.Context()
    frontend = context.socket(zmq.XREP)
    frontend.bind(options.host)
    backend = context.socket(zmq.XREQ)
    backend.bind('inproc://backend')

    poll = zmq.Poller()
    poll.register(frontend, zmq.POLLIN)
    poll.register(backend,  zmq.POLLIN)

    db = leveldb.LevelDB(options.dbfile)

    workers = []
    for i in xrange(3):
        worker = workerThread(context, db)
        worker.start()
        workers.append(worker)

    while True:
        sockets = dict(poll.poll())
        if frontend in sockets:
            if sockets[frontend] == zmq.POLLIN:
                msg = frontend.recv_multipart()
                backend.send_multipart(msg)

        if backend in sockets:
            if sockets[backend] == zmq.POLLIN:
                msg = backend.recv_multipart()
                frontend.send_multipart(msg)

    #never here
    frontend.close()
    backend.close()
    context.term()

<commit_msg>stop all the hangin!  it's still not a graceful shutdown, it blows up because the workers are still in that True loop trying to access the dead socket.<commit_after>#Copyright (c) 2011 Fabula Solutions. All rights reserved.
#Use of this source code is governed by a BSD-style license that can be
#found in the license.txt file.

# leveldb server
import threading
import zmq
import leveldb
import json
import optparse

class workerThread(threading.Thread):
    """workerThread"""
    def __init__(self, context, db):
        threading.Thread.__init__ (self)
        self.context = context
        self.db = db

    def run(self):
        socket = self.context.socket(zmq.XREQ)
        socket.connect('inproc://backend')
        while True:
            msg = socket.recv_multipart()
            if  len(msg) != 3:
                value = 'None'
                reply = [msg[0], value]
                socket.send_multipart(reply)
                continue
            id = msg[0]
            op = msg[1]
            data = json.loads(msg[2])
            reply = [id]
            if op == 'get':
                try:
                    value = self.db.Get(data)
                except:
                    value = ""
                reply.append(value)

            elif op == 'put':
                try:
                    self.db.Put(data[0], data[1])
                    value = "True"
                except:
                    value = ""
                reply.append(value)

            elif op == 'delete':
                self.db.Delete(data)
                value = ""
                reply.append(value)

            elif op == 'range':
                start = data[0]
                end = data[1]
                if start and end:
                    try:
                        arr = []
                        for value in self.db.RangeIter(start, end):
                            arr.append({value[0]: value[1]})
                        reply.append(json.dumps(arr))
                    except:
                        value = ""
                        reply.append(value)
                else:
                    try:
                        arr = []
                        for value in self.db.RangeIter():
                            arr.append({value[0]: value[1]})
                        reply.append(json.dumps(arr))
                    except:
                        value = ""
                        reply.append(value)
            else:
                value = ""
                reply.append(value)
            #print reply
            socket.send_multipart(reply)

        socket.close()

if __name__ == "__main__":
    optparser = optparse.OptionParser(
        prog='leveldb-server.py',
        version='0.1.1',
        description='leveldb-server',
        usage='%prog \n\t-p [port and host settings] Default: tcp://127.0.0.1:5147\n' + \
                    '\t-f [database file name] Default: level.db')
    optparser.add_option('--host', '-p', dest='host',
        default='tcp://127.0.0.1:5147')
    optparser.add_option('--dbfile', '-d', dest='dbfile',
        default='level.db')
    options, arguments = optparser.parse_args()

    if not (options.host and options.dbfile):
        optparser.print_help()

    print "Starting leveldb-server %s" % options.host
    context = zmq.Context()
    frontend = context.socket(zmq.XREP)
    frontend.bind(options.host)
    backend = context.socket(zmq.XREQ)
    backend.bind('inproc://backend')

    poll = zmq.Poller()
    poll.register(frontend, zmq.POLLIN)
    poll.register(backend,  zmq.POLLIN)

    db = leveldb.LevelDB(options.dbfile)

    workers = []
    for i in xrange(3):
        worker = workerThread(context, db)
        worker.start()
        workers.append(worker)

    try:
        sockets = dict(poll.poll())
        if frontend in sockets:
            if sockets[frontend] == zmq.POLLIN:
                msg = frontend.recv_multipart()
                backend.send_multipart(msg)

        if backend in sockets:
            if sockets[backend] == zmq.POLLIN:
                msg = backend.recv_multipart()
                frontend.send_multipart(msg)
    except KeyboardInterrupt:
        frontend.close()
        backend.close()
        context.term()

<|endoftext|>


<commit_before>import locale

from django.contrib.auth.models import AnonymousUser
from django.contrib.sessions.backends.file import SessionStore
from django.template import Template
from django.template import Context
from django.test import TestCase

import lfs.core.utils
from lfs.core.models import Shop
from lfs.core.templatetags.lfs_tags import currency
from lfs.order.models import Order
from lfs.tests.utils import RequestFactory

from lfs.cart.tests import *  # NOQA
from lfs.caching.tests import *  # NOQA
from lfs.catalog.tests import *  # NOQA
from lfs.customer_tax.tests import *  # NOQA
from lfs.marketing.tests import *  # NOQA
from lfs.order.tests import *  # NOQA
from lfs.page.tests import *  # NOQA
from lfs.search.tests import *  # NOQA
from lfs.shipping.tests import *  # NOQA
from lfs.voucher.tests import *  # NOQA
from lfs.customer.tests import *  # NOQA
from lfs.checkout.tests import *  # NOQA
from lfs.manage.tests import *  # NOQA
from lfs.gross_price.tests import *  # NOQA
from lfs.net_price.tests import *  # NOQA
# from lfs.core.wmtests import *

try:
    from lfs_order_numbers.tests import *  # NOQA
except ImportError:
    pass

try:
    from lfs_paypal.tests import *  # NOQA
except ImportError:
    pass

class ShopTestCase(TestCase):
    """Tests the views of the lfs.catalog.
    """
    fixtures = ['lfs_shop.xml']

    def test_shop_defaults(self):
        """Tests the shop values right after creation of an instance
        """
        shop = Shop.objects.get(pk=1)

        self.assertEqual(shop.name, "LFS")
        self.assertEqual(shop.shop_owner, "John Doe")
        self.assertEqual(shop.product_cols, 1)
        self.assertEqual(shop.product_rows, 10)
        self.assertEqual(shop.category_cols, 1)
        self.assertEqual(shop.google_analytics_id, "")
        self.assertEqual(shop.ga_site_tracking, False)
        self.assertEqual(shop.ga_ecommerce_tracking, False)
        self.assertEqual(shop.default_country.name, u"Deutschland")
        self.assertEqual(shop.get_default_country().name, u"Deutschland")
        self.assertEqual(shop.meta_title, u"<name>")
        self.assertEqual(shop.meta_keywords, u"")
        self.assertEqual(shop.meta_description, u"")

    def test_unsupported_locale(self):
        """
        """
        from django.conf import settings
        settings.LFS_LOCALE = "unsupported"

        self.client.get("/")

    def test_from_email(self):
        """
        """
        shop = lfs.core.utils.get_default_shop()

        shop.from_email = "john@doe.com"
        self.assertEqual(shop.from_email, "john@doe.com")

    def test_get_notification_emails(self):
        """
        """
        shop = lfs.core.utils.get_default_shop()

        shop.notification_emails = "john@doe.com, jane@doe.com, baby@doe.com"

        self.assertEqual(
            shop.get_notification_emails(),
            ["john@doe.com", "jane@doe.com", "baby@doe.com"])

        shop.notification_emails = "john@doe.com\njane@doe.com\nbaby@doe.com"
        self.assertEqual(
            shop.get_notification_emails(),
            ["john@doe.com", "jane@doe.com", "baby@doe.com"])

        shop.notification_emails = "john@doe.com\r\njane@doe.com\r\nbaby@doe.com"
        self.assertEqual(
            shop.get_notification_emails(),
            ["john@doe.com", "jane@doe.com", "baby@doe.com"])

        shop.notification_emails = "john@doe.com\n\rjane@doe.com\n\rbaby@doe.com"
        self.assertEqual(
            shop.get_notification_emails(),
            ["john@doe.com", "jane@doe.com", "baby@doe.com"])

        shop.notification_emails = "john@doe.com,,,,\n\n\n\njane@doe.com"
        self.assertEqual(
            shop.get_notification_emails(),
            ["john@doe.com", "jane@doe.com"])

    def test_get_meta_title(self):
        shop = lfs.core.utils.get_default_shop()
        self.assertEqual("LFS", shop.get_meta_title())

        shop.meta_title = "John Doe"
        shop.save()

        self.assertEqual("John Doe", shop.get_meta_title())

        shop.meta_title = "<name> - John Doe"
        shop.save()

        self.assertEqual("LFS - John Doe", shop.get_meta_title())

        shop.meta_title = "John Doe - <name>"
        shop.save()

        self.assertEqual("John Doe - LFS", shop.get_meta_title())

    def test_get_meta_keywords(self):
        shop = lfs.core.utils.get_default_shop()
        self.assertEqual("", shop.get_meta_keywords())

        shop.meta_keywords = "John Doe"
        shop.save()

        self.assertEqual("John Doe", shop.get_meta_keywords())

        shop.meta_keywords = "<name> - John Doe"
        shop.save()

        self.assertEqual("LFS - John Doe", shop.get_meta_keywords())

        shop.meta_keywords = "<name> - John Doe"
        shop.save()

        self.assertEqual("LFS - John Doe", shop.get_meta_keywords())

        shop.meta_keywords = "<name> - John Doe - <name>"
        shop.save()

        self.assertEqual("LFS - John Doe - LFS", shop.get_meta_keywords())

    def test_get_meta_description(self):
        shop = lfs.core.utils.get_default_shop()
        self.assertEqual("", shop.get_meta_description())

        shop.meta_description = "John Doe"
        shop.save()

        self.assertEqual("John Doe", shop.get_meta_description())

        shop.meta_description = "<name> - John Doe"
        shop.save()

        self.assertEqual("LFS - John Doe", shop.get_meta_description())

        shop.meta_description = "<name> - John Doe"
        shop.save()

        self.assertEqual("LFS - John Doe", shop.get_meta_description())

        shop.meta_description = "<name> - John Doe - <name>"
        shop.save()

        self.assertEqual("LFS - John Doe - LFS", shop.get_meta_description())

class TagsTestCase(TestCase):
    """
    """
    fixtures = ['lfs_shop.xml']

    def test_ga_site_tracking(self):
        """
        """
        shop = Shop.objects.get(pk=1)
        shop.google_analytics_id = ""
        shop.ga_site_tracking = False
        shop.ga_ecommerce_tracking = False
        shop.save()

        template = Template("""{% load lfs_tags %}{% google_analytics_tracking %}""")
        content = template.render(Context())
        self.failIf(content.find("pageTracker") != -1)

        # Enter a google_analytics_id
        shop.google_analytics_id = "UA-XXXXXXXXXX"
        shop.save()

        # But this is not enough
        content = template.render(Context())
        self.failIf(content.find("pageTracker") != -1)

        # It has to be activated first
        shop.ga_site_tracking = True
        shop.save()

        # Now it works and "pageTracker" is found
        content = template.render(Context())
        self.failIf(content.find("pageTracker") == -1)

    def test_ga_ecommerce_tracking(self):
        """
        """
        shop = lfs.core.utils.get_default_shop()
        shop.google_analytics_id = ""
        shop.ga_site_tracking = False
        shop.ga_ecommerce_tracking = False
        shop.save()

        session = SessionStore()

        rf = RequestFactory()
        request = rf.get('/')
        request.session = session

        template = Template("""{% load lfs_tags %}{% google_analytics_ecommerce %}""")
        content = template.render(Context({"request": request}))
        self.failIf(content.find("pageTracker") != -1)

        # Enter a google_analytics_id
        shop.google_analytics_id = "UA-XXXXXXXXXX"
        shop.save()

        # Simulating a new request
        rf = RequestFactory()
        request = rf.get('/')
        request.session = session

        # But this is not enough
        content = template.render(Context({"request": request}))
        self.failIf(content.find("pageTracker") != -1)

        # It has to be activated first
        shop.ga_ecommerce_tracking = True
        shop.save()

        # Simulating a new request
        rf = RequestFactory()
        request = rf.get('/')
        request.session = session

        # But this is still not enough
        content = template.render(Context({"request": request}))
        self.failIf(content.find("pageTracker") != -1)

        # There has to be an order within the session
        session["order"] = Order()

        # Now it works and "pageTracker" is found
        content = template.render(Context({"request": request}))
        self.failIf(content.find("_trackPageview") == -1)

    def test_currency(self):
        """
        """
        locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
        self.assertEqual(currency(0.0), '$0.00')
        self.assertEqual(currency(1.0), '$1.00')

        shop = lfs.core.utils.get_default_shop()
        shop.use_international_currency_code = True
        shop.save()

        self.assertEqual(currency(0.0, None, False), 'USD 0.00')
        self.assertEqual(currency(1.0, None, False), 'USD 1.00')

        self.assertEqual(currency(-1.0, None, False), '-USD 1.00')

class ManageURLsTestCase(TestCase):
    def test_manage_urls_anonymous(self):
        """Tests that all manage urls cannot accessed by anonymous user.
        """
        rf = RequestFactory()
        request = rf.get("/")
        request.user = AnonymousUser()

        from lfs.manage.urls import urlpatterns
        for url in urlpatterns:
            result = url.callback(request)
            self.failUnless(result["Location"].startswith("/login/?next=/"))
<commit_msg>sitemaps: add tests for pluggable/adaptable sitemaps<commit_after>import locale
import sys

from django.contrib.auth.models import AnonymousUser
from django.contrib.sessions.backends.file import SessionStore
from django.template import Template
from django.template import Context
from django.test import TestCase

import lfs.core.utils
from lfs.core.models import Shop
from lfs.core.templatetags.lfs_tags import currency
from lfs.order.models import Order
from lfs.tests.utils import RequestFactory

from lfs.cart.tests import *  # NOQA
from lfs.caching.tests import *  # NOQA
from lfs.catalog.tests import *  # NOQA
from lfs.customer_tax.tests import *  # NOQA
from lfs.marketing.tests import *  # NOQA
from lfs.order.tests import *  # NOQA
from lfs.page.tests import *  # NOQA
from lfs.search.tests import *  # NOQA
from lfs.shipping.tests import *  # NOQA
from lfs.voucher.tests import *  # NOQA
from lfs.customer.tests import *  # NOQA
from lfs.checkout.tests import *  # NOQA
from lfs.manage.tests import *  # NOQA
from lfs.gross_price.tests import *  # NOQA
from lfs.net_price.tests import *  # NOQA
# from lfs.core.wmtests import *

try:
    from lfs_order_numbers.tests import *  # NOQA
except ImportError:
    pass

try:
    from lfs_paypal.tests import *  # NOQA
except ImportError:
    pass

class ShopTestCase(TestCase):
    """Tests the views of the lfs.catalog.
    """
    fixtures = ['lfs_shop.xml']

    def test_shop_defaults(self):
        """Tests the shop values right after creation of an instance
        """
        shop = Shop.objects.get(pk=1)

        self.assertEqual(shop.name, "LFS")
        self.assertEqual(shop.shop_owner, "John Doe")
        self.assertEqual(shop.product_cols, 1)
        self.assertEqual(shop.product_rows, 10)
        self.assertEqual(shop.category_cols, 1)
        self.assertEqual(shop.google_analytics_id, "")
        self.assertEqual(shop.ga_site_tracking, False)
        self.assertEqual(shop.ga_ecommerce_tracking, False)
        self.assertEqual(shop.default_country.name, u"Deutschland")
        self.assertEqual(shop.get_default_country().name, u"Deutschland")
        self.assertEqual(shop.meta_title, u"<name>")
        self.assertEqual(shop.meta_keywords, u"")
        self.assertEqual(shop.meta_description, u"")

    def test_unsupported_locale(self):
        """
        """
        from django.conf import settings
        settings.LFS_LOCALE = "unsupported"

        self.client.get("/")

    def test_from_email(self):
        """
        """
        shop = lfs.core.utils.get_default_shop()

        shop.from_email = "john@doe.com"
        self.assertEqual(shop.from_email, "john@doe.com")

    def test_get_notification_emails(self):
        """
        """
        shop = lfs.core.utils.get_default_shop()

        shop.notification_emails = "john@doe.com, jane@doe.com, baby@doe.com"

        self.assertEqual(
            shop.get_notification_emails(),
            ["john@doe.com", "jane@doe.com", "baby@doe.com"])

        shop.notification_emails = "john@doe.com\njane@doe.com\nbaby@doe.com"
        self.assertEqual(
            shop.get_notification_emails(),
            ["john@doe.com", "jane@doe.com", "baby@doe.com"])

        shop.notification_emails = "john@doe.com\r\njane@doe.com\r\nbaby@doe.com"
        self.assertEqual(
            shop.get_notification_emails(),
            ["john@doe.com", "jane@doe.com", "baby@doe.com"])

        shop.notification_emails = "john@doe.com\n\rjane@doe.com\n\rbaby@doe.com"
        self.assertEqual(
            shop.get_notification_emails(),
            ["john@doe.com", "jane@doe.com", "baby@doe.com"])

        shop.notification_emails = "john@doe.com,,,,\n\n\n\njane@doe.com"
        self.assertEqual(
            shop.get_notification_emails(),
            ["john@doe.com", "jane@doe.com"])

    def test_get_meta_title(self):
        shop = lfs.core.utils.get_default_shop()
        self.assertEqual("LFS", shop.get_meta_title())

        shop.meta_title = "John Doe"
        shop.save()

        self.assertEqual("John Doe", shop.get_meta_title())

        shop.meta_title = "<name> - John Doe"
        shop.save()

        self.assertEqual("LFS - John Doe", shop.get_meta_title())

        shop.meta_title = "John Doe - <name>"
        shop.save()

        self.assertEqual("John Doe - LFS", shop.get_meta_title())

    def test_get_meta_keywords(self):
        shop = lfs.core.utils.get_default_shop()
        self.assertEqual("", shop.get_meta_keywords())

        shop.meta_keywords = "John Doe"
        shop.save()

        self.assertEqual("John Doe", shop.get_meta_keywords())

        shop.meta_keywords = "<name> - John Doe"
        shop.save()

        self.assertEqual("LFS - John Doe", shop.get_meta_keywords())

        shop.meta_keywords = "<name> - John Doe"
        shop.save()

        self.assertEqual("LFS - John Doe", shop.get_meta_keywords())

        shop.meta_keywords = "<name> - John Doe - <name>"
        shop.save()

        self.assertEqual("LFS - John Doe - LFS", shop.get_meta_keywords())

    def test_get_meta_description(self):
        shop = lfs.core.utils.get_default_shop()
        self.assertEqual("", shop.get_meta_description())

        shop.meta_description = "John Doe"
        shop.save()

        self.assertEqual("John Doe", shop.get_meta_description())

        shop.meta_description = "<name> - John Doe"
        shop.save()

        self.assertEqual("LFS - John Doe", shop.get_meta_description())

        shop.meta_description = "<name> - John Doe"
        shop.save()

        self.assertEqual("LFS - John Doe", shop.get_meta_description())

        shop.meta_description = "<name> - John Doe - <name>"
        shop.save()

        self.assertEqual("LFS - John Doe - LFS", shop.get_meta_description())

class TagsTestCase(TestCase):
    """
    """
    fixtures = ['lfs_shop.xml']

    def test_ga_site_tracking(self):
        """
        """
        shop = Shop.objects.get(pk=1)
        shop.google_analytics_id = ""
        shop.ga_site_tracking = False
        shop.ga_ecommerce_tracking = False
        shop.save()

        template = Template("""{% load lfs_tags %}{% google_analytics_tracking %}""")
        content = template.render(Context())
        self.failIf(content.find("pageTracker") != -1)

        # Enter a google_analytics_id
        shop.google_analytics_id = "UA-XXXXXXXXXX"
        shop.save()

        # But this is not enough
        content = template.render(Context())
        self.failIf(content.find("pageTracker") != -1)

        # It has to be activated first
        shop.ga_site_tracking = True
        shop.save()

        # Now it works and "pageTracker" is found
        content = template.render(Context())
        self.failIf(content.find("pageTracker") == -1)

    def test_ga_ecommerce_tracking(self):
        """
        """
        shop = lfs.core.utils.get_default_shop()
        shop.google_analytics_id = ""
        shop.ga_site_tracking = False
        shop.ga_ecommerce_tracking = False
        shop.save()

        session = SessionStore()

        rf = RequestFactory()
        request = rf.get('/')
        request.session = session

        template = Template("""{% load lfs_tags %}{% google_analytics_ecommerce %}""")
        content = template.render(Context({"request": request}))
        self.failIf(content.find("pageTracker") != -1)

        # Enter a google_analytics_id
        shop.google_analytics_id = "UA-XXXXXXXXXX"
        shop.save()

        # Simulating a new request
        rf = RequestFactory()
        request = rf.get('/')
        request.session = session

        # But this is not enough
        content = template.render(Context({"request": request}))
        self.failIf(content.find("pageTracker") != -1)

        # It has to be activated first
        shop.ga_ecommerce_tracking = True
        shop.save()

        # Simulating a new request
        rf = RequestFactory()
        request = rf.get('/')
        request.session = session

        # But this is still not enough
        content = template.render(Context({"request": request}))
        self.failIf(content.find("pageTracker") != -1)

        # There has to be an order within the session
        session["order"] = Order()

        # Now it works and "pageTracker" is found
        content = template.render(Context({"request": request}))
        self.failIf(content.find("_trackPageview") == -1)

    def test_currency(self):
        """
        """
        locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
        self.assertEqual(currency(0.0), '$0.00')
        self.assertEqual(currency(1.0), '$1.00')

        shop = lfs.core.utils.get_default_shop()
        shop.use_international_currency_code = True
        shop.save()

        self.assertEqual(currency(0.0, None, False), 'USD 0.00')
        self.assertEqual(currency(1.0, None, False), 'USD 1.00')

        self.assertEqual(currency(-1.0, None, False), '-USD 1.00')

class ManageURLsTestCase(TestCase):
    def test_manage_urls_anonymous(self):
        """Tests that all manage urls cannot accessed by anonymous user.
        """
        rf = RequestFactory()
        request = rf.get("/")
        request.user = AnonymousUser()

        from lfs.manage.urls import urlpatterns
        for url in urlpatterns:
            result = url.callback(request)
            self.failUnless(result["Location"].startswith("/login/?next=/"))

class SiteMapsTestCase(TestCase):
    fixtures = ['lfs_shop.xml']

    def test_sitemaps_default_1(self):
        result = self.client.get("/sitemap.xml")
        self.assertContains(result, "<priority>0.5</priority>")
        self.assertContains(result, "<changefreq>weekly</changefreq>")
        self.assertContains(result, "http://")

    def test_sitemaps_default_2(self):
        from lfs.core.sitemaps import CategorySitemap
        from lfs.core.sitemaps import ShopSitemap
        from lfs.core.sitemaps import PageSitemap
        from lfs.core.sitemaps import ProductSitemap

        self.assertEqual(CategorySitemap.priority, 0.5)
        self.assertEqual(CategorySitemap.changefreq, "weekly")
        self.assertEqual(CategorySitemap.protocol, "http")

        self.assertEqual(PageSitemap.priority, 0.5)
        self.assertEqual(PageSitemap.changefreq, "weekly")
        self.assertEqual(PageSitemap.protocol, "http")

        self.assertEqual(ProductSitemap.priority, 0.5)
        self.assertEqual(ProductSitemap.changefreq, "weekly")
        self.assertEqual(ProductSitemap.protocol, "http")

        self.assertEqual(ShopSitemap.priority, 0.5)
        self.assertEqual(ShopSitemap.changefreq, "weekly")
        self.assertEqual(ShopSitemap.protocol, "http")

    def test_sitemaps_settings(self):
        my_sitemaps = {
            "category": {
                "priority": 0.1,
                "changefreq": "category-daily",
                "protocol": "category-https",
            },
            "page": {
                "priority": 0.2,
                "changefreq": "page-daily",
                "protocol": "page-https",
            },
            "product": {
                "priority": 0.3,
                "changefreq": "product-daily",
                "protocol": "product-https",
            },
            "shop": {
                "priority": 0.4,
                "changefreq": "shop-daily",
                "protocol": "shop-https",
            },
        }

        with self.settings(LFS_SITEMAPS=my_sitemaps):
            if "lfs.core.sitemaps" in sys.modules:
                del sys.modules["lfs.core.sitemaps"]

            from lfs.core.sitemaps import CategorySitemap
            from lfs.core.sitemaps import ShopSitemap
            from lfs.core.sitemaps import PageSitemap
            from lfs.core.sitemaps import ProductSitemap

            self.assertEqual(CategorySitemap.priority, 0.1)
            self.assertEqual(CategorySitemap.changefreq, "category-daily")
            self.assertEqual(CategorySitemap.protocol, "category-https")

            self.assertEqual(PageSitemap.priority, 0.2)
            self.assertEqual(PageSitemap.changefreq, "page-daily")
            self.assertEqual(PageSitemap.protocol, "page-https")

            self.assertEqual(ProductSitemap.priority, 0.3)
            self.assertEqual(ProductSitemap.changefreq, "product-daily")
            self.assertEqual(ProductSitemap.protocol, "product-https")

            self.assertEqual(ShopSitemap.priority, 0.4)
            self.assertEqual(ShopSitemap.changefreq, "shop-daily")
            self.assertEqual(ShopSitemap.protocol, "shop-https")
<|endoftext|>


<commit_before>import random
UNK = 'UNK'
# This file contains the dataset in a useful way. We populate a list of
# Trees to train/test our Neural Nets such that each Tree contains any
# number of Node objects.

# The best way to get a feel for how these objects are used in the program is to drop pdb.set_trace() in a few places throughout the codebase
# to see how the trees are used.. look where loadtrees() is called etc..

class Node:  # a node in the tree
    def __init__(self, label, word=None):
        self.label = label
        self.word = word
        self.parent = None  # reference to parent
        self.left = None  # reference to left child
        self.right = None  # reference to right child
        # true if I am a leaf (could have probably derived this from if I have
        # a word)
        self.isLeaf = False
        # true if we have finished performing fowardprop on this node (note,
        # there are many ways to implement the recursion.. some might not
        # require this flag)

class Tree:

    def __init__(self, treeString, openChar='(', closeChar=')'):
        tokens = []
        self.open = '('
        self.close = ')'
        for toks in treeString.strip().split():
            tokens += list(toks)
        self.root = self.parse(tokens)
        # get list of labels as obtained through a post-order traversal
        self.labels = get_labels(self.root)
        self.num_words = len(self.labels)

    def parse(self, tokens, parent=None):
        assert tokens[0] == self.open, "Malformed tree"
        assert tokens[-1] == self.close, "Malformed tree"

        split = 2  # position after open and label
        countOpen = countClose = 0

        if tokens[split] == self.open:
            countOpen += 1
            split += 1
        # Find where left child and right child split
        while countOpen != countClose:
            if tokens[split] == self.open:
                countOpen += 1
            if tokens[split] == self.close:
                countClose += 1
            split += 1

        # New node
        node = Node(int(tokens[1]))  # zero index labels

        node.parent = parent

        # leaf Node
        if countOpen == 0:
            node.word = ''.join(tokens[2:-1]).lower()  # lower case?
            node.isLeaf = True
            return node

        node.left = self.parse(tokens[2:split], parent=node)
        node.right = self.parse(tokens[split:-1], parent=node)

        return node

    def get_words(self):
        leaves = getLeaves(self.root)
        words = [node.word for node in leaves]
        return words

def leftTraverse(node, nodeFn=None, args=None):
    """
    Recursive function traverses tree
    from left to right.
    Calls nodeFn at each node
    """
    if node is None:
        return
    leftTraverse(node.left, nodeFn, args)
    leftTraverse(node.right, nodeFn, args)
    nodeFn(node, args)

def getLeaves(node):
    if node is None:
        return []
    if node.isLeaf:
        return [node]
    else:
        return getLeaves(node.left) + getLeaves(node.right)

def get_labels(node):
    if node is None:
        return []
    return get_labels(node.left) + get_labels(node.right) + [node.label]

def clearFprop(node, words):
    node.fprop = False

def loadTrees(dataSet='train'):
    """
    Loads training trees. Maps leaf node words to word ids.
    """
    file = 'trees/%s.txt' % dataSet
    print("Loading %s trees.." % dataSet)
    with open(file, 'r') as fid:
        trees = [Tree(l) for l in fid.readlines()]

    return trees

def simplified_data(num_train, num_dev, num_test):
    rndstate = random.getstate()
    random.seed(0)
    trees = loadTrees('train') + loadTrees('dev') + loadTrees('test')

    #filter extreme trees
    pos_trees = [t for t in trees if t.root.label==4]
    neg_trees = [t for t in trees if t.root.label==0]

    #binarize labels
    binarize_labels(pos_trees)
    binarize_labels(neg_trees)

    #split into train, dev, test
    print(len(pos_trees), len(neg_trees))
    pos_trees = sorted(pos_trees, key=lambda t: len(t.get_words()))
    neg_trees = sorted(neg_trees, key=lambda t: len(t.get_words()))
    num_train/=2
    num_dev/=2
    num_test/=2
    train = pos_trees[:num_train] + neg_trees[:num_train]
    dev = pos_trees[num_train : num_train+num_dev] + neg_trees[num_train : num_train+num_dev]
    test = pos_trees[num_train+num_dev : num_train+num_dev+num_test] + neg_trees[num_train+num_dev : num_train+num_dev+num_test]
    random.shuffle(train)
    random.shuffle(dev)
    random.shuffle(test)
    random.setstate(rndstate)

    return train, dev, test

def binarize_labels(trees):
    def binarize_node(node, _):
        if node.label<2:
            node.label = 0
        elif node.label>2:
            node.label = 1
    for tree in trees:
        leftTraverse(tree.root, binarize_node, None)
        tree.labels = get_labels(tree.root)
<commit_msg>Replace float divide by 'int' divide<commit_after>import random
UNK = 'UNK'
# This file contains the dataset in a useful way. We populate a list of
# Trees to train/test our Neural Nets such that each Tree contains any
# number of Node objects.

# The best way to get a feel for how these objects are used in the program is to drop pdb.set_trace() in a few places throughout the codebase
# to see how the trees are used.. look where loadtrees() is called etc..

class Node:  # a node in the tree
    def __init__(self, label, word=None):
        self.label = label
        self.word = word
        self.parent = None  # reference to parent
        self.left = None  # reference to left child
        self.right = None  # reference to right child
        # true if I am a leaf (could have probably derived this from if I have
        # a word)
        self.isLeaf = False
        # true if we have finished performing fowardprop on this node (note,
        # there are many ways to implement the recursion.. some might not
        # require this flag)

class Tree:

    def __init__(self, treeString, openChar='(', closeChar=')'):
        tokens = []
        self.open = '('
        self.close = ')'
        for toks in treeString.strip().split():
            tokens += list(toks)
        self.root = self.parse(tokens)
        # get list of labels as obtained through a post-order traversal
        self.labels = get_labels(self.root)
        self.num_words = len(self.labels)

    def parse(self, tokens, parent=None):
        assert tokens[0] == self.open, "Malformed tree"
        assert tokens[-1] == self.close, "Malformed tree"

        split = 2  # position after open and label
        countOpen = countClose = 0

        if tokens[split] == self.open:
            countOpen += 1
            split += 1
        # Find where left child and right child split
        while countOpen != countClose:
            if tokens[split] == self.open:
                countOpen += 1
            if tokens[split] == self.close:
                countClose += 1
            split += 1

        # New node
        node = Node(int(tokens[1]))  # zero index labels

        node.parent = parent

        # leaf Node
        if countOpen == 0:
            node.word = ''.join(tokens[2:-1]).lower()  # lower case?
            node.isLeaf = True
            return node

        node.left = self.parse(tokens[2:split], parent=node)
        node.right = self.parse(tokens[split:-1], parent=node)

        return node

    def get_words(self):
        leaves = getLeaves(self.root)
        words = [node.word for node in leaves]
        return words

def leftTraverse(node, nodeFn=None, args=None):
    """
    Recursive function traverses tree
    from left to right.
    Calls nodeFn at each node
    """
    if node is None:
        return
    leftTraverse(node.left, nodeFn, args)
    leftTraverse(node.right, nodeFn, args)
    nodeFn(node, args)

def getLeaves(node):
    if node is None:
        return []
    if node.isLeaf:
        return [node]
    else:
        return getLeaves(node.left) + getLeaves(node.right)

def get_labels(node):
    if node is None:
        return []
    return get_labels(node.left) + get_labels(node.right) + [node.label]

def clearFprop(node, words):
    node.fprop = False

def loadTrees(dataSet='train'):
    """
    Loads training trees. Maps leaf node words to word ids.
    """
    file = 'trees/%s.txt' % dataSet
    print("Loading %s trees.." % dataSet)
    with open(file, 'r') as fid:
        trees = [Tree(l) for l in fid.readlines()]

    return trees

def simplified_data(num_train, num_dev, num_test):
    rndstate = random.getstate()
    random.seed(0)
    trees = loadTrees('train') + loadTrees('dev') + loadTrees('test')

    #filter extreme trees
    pos_trees = [t for t in trees if t.root.label==4]
    neg_trees = [t for t in trees if t.root.label==0]

    #binarize labels
    binarize_labels(pos_trees)
    binarize_labels(neg_trees)

    #split into train, dev, test
    print(len(pos_trees), len(neg_trees))
    pos_trees = sorted(pos_trees, key=lambda t: len(t.get_words()))
    neg_trees = sorted(neg_trees, key=lambda t: len(t.get_words()))
    num_train = num_train // 2
    num_dev = num_dev // 2
    num_test = num_test // 2
    train = pos_trees[:num_train] + neg_trees[:num_train]
    dev = pos_trees[num_train : num_train+num_dev] + neg_trees[num_train : num_train+num_dev]
    test = pos_trees[num_train+num_dev : num_train+num_dev+num_test] + neg_trees[num_train+num_dev : num_train+num_dev+num_test]
    random.shuffle(train)
    random.shuffle(dev)
    random.shuffle(test)
    random.setstate(rndstate)

    return train, dev, test

def binarize_labels(trees):
    def binarize_node(node, _):
        if node.label<2:
            node.label = 0
        elif node.label>2:
            node.label = 1
    for tree in trees:
        leftTraverse(tree.root, binarize_node, None)
        tree.labels = get_labels(tree.root)
<|endoftext|>