<commit_before>from os import path
import sys
import math
import random
from random import seed
import heapq
from operator import itemgetter
from itertools import chain
import time
from datetime import datetime, timezone

here_dir = path.dirname(path.abspath(__file__))
sys.path.append(path.join(here_dir, '..'))

from neat import (Mutator, Pipeline, GeneSpec, ParamSpec as PS, validate_genome,
    gen, mut, bounds,
    neuron, connection, default_gene_factory)

from nn_impl import NN

#### CONFIG #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### ####
generation_size = 20 # number of genomes in each generation across all species
num_species = 4      # number of species
elite_num = 1        # how many top performing members of a species will be
                     #   copied unchanged into the next generation

conf = dict(
selection_sample_size = int(.75 * generation_size) // num_species, # size of the selection sample (must be in the range [2, pop_size])
neuron_param_mut_proba = 0.8,     # probability to mutate each single neuron in the genome
connection_param_mut_proba = 0.8, # probability to mutate each single connection in the genome
topology_augmentation_proba = 0,  # probability to augment the topology of a newly created genome
topology_reduction_proba = 0,     # probability to reduce the topology of a newly created genome
)
#### ###### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### ####

def make_pipeline(mutator, **args):
    conf_ = conf.copy()
    conf_.update(args)
    return Pipeline(
        topology_mutator=mutator,
        # # disable 2-way crossover
        # crossover_step=lambda genomes: genomes[0].copy(),
        **conf_)

def species_sizes(n, num_species):
    return (n // num_species,) * (num_species - 1) + (n // num_species + n % num_species,)

def count_members(species_list):
    return sum(len(species) for species in species_list)

## CREATE GENE SPECS ##
neuron_sigma = 0.25  # mutation sigma value for neuron params
conn_sigma = 10.     # mutation sigma value for connection params

sigmoid_params = (
    PS('bias', gen.uniform(), mut.gauss(neuron_sigma), bounds(-1., 1.)),
    PS('gain', gen.uniform(), mut.gauss(neuron_sigma), bounds(0., 1.)),
)

input_spec = GeneSpec('sigmoid-i', *sigmoid_params)
hidden_spec = GeneSpec('sigmoid-h', *sigmoid_params)
output_spec = GeneSpec('sigmoid-o', *sigmoid_params)
connection_spec = GeneSpec(
    'connection',
    PS('weight', gen.gauss(0, conn_sigma), mut.gauss(conn_sigma)),
)

def produce_new_generation(pipeline, genome_fitness_list):
    for _ in range(len(genome_fitness_list) - elite_num):
        g = pipeline.produce_new_genome(genome_fitness_list)
        validate_genome(g, 'invalid genome')
        yield g

    # bringing the best genomes into next generation:
    best_genomes = heapq.nlargest(elite_num, genome_fitness_list, key=itemgetter(1))
    for genome, _ in best_genomes:
        yield genome

## INPUT VALUES FOR EVALUATING FITNESS OF THE SOLUTIONS ##
inputs = ((0, 0), (0, 1), (1, 0), (1, 1))

def evaluate(genome):
    nn = NN(genome)

    # print(genome)
    outp0 = nn.compute(inputs[0])[0]
    nn.reset() # reset network otherwise it will remember previous state
    outp1 = nn.compute(inputs[1])[0]
    nn.reset()
    outp2 = nn.compute(inputs[2])[0]
    nn.reset()
    outp3 = nn.compute(inputs[3])[0]

    fitness = abs(outp0 - outp1) * abs(outp0 - outp2) * abs(outp3 - outp1) * abs(outp3 - outp2)
    return fitness

def complexity(species_list):
    n = (sum(1 for _ in ge.neuron_genes()) for ge in chain(*species_list))
    c = (ge.num_connection_genes() for ge in chain(*species_list))
    return sum(n), sum(c)

total_eval_time = [0]
total_neat_time = [0]

class Attempt:
    def __init__(self):
        self.evals_num = 0
        self.best_fitness = None
        self.best_genome = None
        self.target_reached = False

    def make_next_generation(self, pipeline, current_gen):
        self.evals_num += count_members(current_gen)

        t0 = time.perf_counter()
        evaluated = [[(genome, evaluate(genome)) for genome in species] for species in current_gen]
        t1 = time.perf_counter()
        total_eval_time[0] += t1 - t0

        next_gen = [list(produce_new_generation(pipeline, species)) for species in evaluated]
        t2 = time.perf_counter()
        total_neat_time[0] += t2 - t1

        self.best_genome, self.best_fitness = max(chain(*evaluated), key=itemgetter(1))
        return next_gen

    def get_stats(self, genomes):
        n_neurons, n_conns = complexity(genomes)
        return ("evals: {}, best genome has: {}N, {}C, complexity: {}N, {}C, best fitness = {}"
            .format(self.evals_num,
                sum(1 for _ in self.best_genome.neuron_genes()),
                self.best_genome.num_connection_genes(),
                n_neurons, n_conns, self.best_fitness))

## CREATE INITIAL GENERATION ##
def copy_with_mutation(pipeline, source_genome):
    return pipeline.topology_augmentation_step(pipeline.parameters_mutation_step(source_genome.copy()))

## CREATE MUTATOR ##
mutator = Mutator(
    neuron_factory=default_gene_factory(hidden_spec),
    connection_factory=default_gene_factory(connection_spec),
    channels=(
        # (input_spec, output_spec),
        (input_spec, hidden_spec),
        # (hidden_spec, hidden_spec),
        (hidden_spec, output_spec),
    )
)

def make_attempt(num_epochs, gens_per_epoch):
    augmentation_proba = .8
    reduction_proba = .8

    augment_gens_per_epoch = int(.4 * gens_per_epoch)
    reduct_gens_per_epoch = gens_per_epoch - augment_gens_per_epoch

    ## CREATE THE REPRODUCTION PIPELINE ##
    pipeline = make_pipeline(mutator)

    ## CREATE INITIAL GENOME ##
    # we specify initial input and output neurons and protect them from removal

    init_genome = mutator.produce_genome(
        in1=neuron(input_spec, non_removable=True),
        in2=neuron(input_spec, non_removable=True),
        out1=neuron(output_spec, non_removable=True),
        connections=(
            # connection(connection_spec, src='in1', dst='out1'),
            # connection(connection_spec, src='in2', dst='out1')
        )
    )

    current_gen = list(
        list(copy_with_mutation(pipeline, init_genome) for _ in range(species_size)) \
        for species_size in species_sizes(generation_size, num_species))

    attempt = Attempt()

    ## RUN ALTERNATING AUGMENTATION AND REDUCTION STAGES ##
    for epoch in range(num_epochs):
        # print("Epoch #{}".format(epoch))

        pipeline = make_pipeline(mutator, topology_augmentation_proba=augmentation_proba, topology_reduction_proba=0)

        for _ in range(augment_gens_per_epoch):
            current_gen = attempt.make_next_generation(pipeline, current_gen)
        # print("  AUG " + attempt.get_stats(current_gen))

        pipeline = make_pipeline(mutator, topology_augmentation_proba=0, topology_reduction_proba=reduction_proba)

        for _ in range(reduct_gens_per_epoch):
            current_gen = attempt.make_next_generation(pipeline, current_gen)
        # print("  RED " + attempt.get_stats(current_gen))

        if abs(attempt.best_fitness - 1) < 1e-6:
            attempt.target_reached = True
            break
    return attempt

def timestamp():
    ts = datetime.fromtimestamp(time.time(), tz=timezone.utc)
    return ts.strftime("%b %d %y %H:%M")

## RUN MULTIPLE ATTEMPTS TO CREATE A XOR NETWORK ##
best_genome = None

num_attempts = 50
total_eval_num = 0
success_count = 0
start_timer = time.perf_counter()

for attempt_id in range(num_attempts):
    # attempt_id += 100#20400

    # attempt_id+=1000

    num_epochs = 20
    gens_per_epoch = 250

    seed(attempt_id)
    attempt = make_attempt(num_epochs=num_epochs, gens_per_epoch=gens_per_epoch)

    if attempt.target_reached:
        success_count += 1
        best_genome = attempt.best_genome
    total_eval_num += attempt.evals_num
    result = dict(
        id = attempt_id,
        t = timestamp(),
        species = tuple(species_sizes(generation_size, num_species)),
        gens_in_epoch = gens_per_epoch,
        n_epochs = num_epochs,
        n_evals = attempt.evals_num,
        fitness = attempt.best_fitness,
        target_reached = attempt.target_reached,
    )
    print(result)

total_time = time.perf_counter() - start_timer
print(f"took {total_time} ticks")
print(f"total eval time {total_eval_time[0]} ticks")
print(f"total neat time {total_neat_time[0]} ticks")
print(f"eval time per 10000 evals: {10000*total_eval_time[0] / total_eval_num} ticks")
print(f"neat time per 10000 evals: {10000*total_neat_time[0] / total_eval_num} ticks")
print(f"neat time percentage = {100*total_neat_time[0] / total_time}")
print(f"total number of innovations: {mutator.innovation_number}")
print(f"success rate: {success_count} / {num_attempts}")

time_per_success = total_time / success_count if success_count > 0 else "N/A"
print(f"time per success: {time_per_success}")

# print("Final test:")
# final_nn = NN(best_genome)

# test_inputs = list(inputs) * 4
# random.shuffle(test_inputs)
# for inp in test_inputs:
#     final_nn.reset()
#     print("{} -> {}".format(inp, final_nn.compute(inp)[0]))

# write final genome as YAML file
if best_genome is None:
    print("could not reach the target")
else:
    filename = path.join(here_dir, "xor_genome.yaml")
    try:
        with open(filename, 'w+') as genfile:
            genfile.write(best_genome.to_yaml())
        print(f"solution was written to file {filename}")

    except Exception:
        pass
<commit_msg>fixed an error in examples/xor.py<commit_after>from os import path
import sys
import math
import random
from random import seed
import heapq
from operator import itemgetter
from itertools import chain
import time
from datetime import datetime, timezone

here_dir = path.dirname(path.abspath(__file__))
sys.path.append(path.join(here_dir, '..'))

from neat import (Mutator, Pipeline, GeneSpec, ParamSpec as PS, validate_genome,
    gen, mut, bounds,
    neuron, connection, default_gene_factory)

from nn_impl import NN

#### CONFIG #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### ####
generation_size = 20 # number of genomes in each generation across all species
num_species = 4      # number of species
elite_num = 1        # how many top performing members of a species will be
                     #   copied unchanged into the next generation

conf = dict(
selection_sample_size = int(.75 * generation_size) // num_species, # size of the selection sample (must be in the range [2, pop_size])
neuron_param_mut_proba = 0.8,     # probability to mutate each single neuron in the genome
connection_param_mut_proba = 0.8, # probability to mutate each single connection in the genome
topology_augmentation_proba = 0,  # probability to augment the topology of a newly created genome
topology_reduction_proba = 0,     # probability to reduce the topology of a newly created genome
)
#### ###### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### ####

def make_pipeline(mutator, **args):
    conf_ = conf.copy()
    conf_.update(args)
    return Pipeline(
        topology_mutator=mutator,
        # # disable 2-way crossover
        # crossover_step=lambda genomes: genomes[0].copy(),
        **conf_)

def species_sizes(n, num_species):
    return (n // num_species,) * (num_species - 1) + (n // num_species + n % num_species,)

def count_members(species_list):
    return sum(len(species) for species in species_list)

## CREATE GENE SPECS ##
neuron_sigma = 0.25  # mutation sigma value for neuron params
conn_sigma = 10.     # mutation sigma value for connection params

sigmoid_params = (
    PS('bias', gen.uniform(), mut.gauss(neuron_sigma), bounds(-1., 1.)),
    PS('gain', gen.uniform(), mut.gauss(neuron_sigma), bounds(0., 1.)),
)

input_spec = GeneSpec('sigmoid-i', *sigmoid_params)
hidden_spec = GeneSpec('sigmoid-h', *sigmoid_params)
output_spec = GeneSpec('sigmoid-o', *sigmoid_params)
connection_spec = GeneSpec(
    'connection',
    PS('weight', gen.gauss(0, conn_sigma), mut.gauss(conn_sigma)),
)

def produce_new_generation(pipeline, genome_fitness_list):
    for _ in range(len(genome_fitness_list) - elite_num):
        g = pipeline(genome_fitness_list)
        validate_genome(g, 'invalid genome')
        yield g

    # bringing the best genomes into next generation:
    best_genomes = heapq.nlargest(elite_num, genome_fitness_list, key=itemgetter(1))
    for genome, _ in best_genomes:
        yield genome

## INPUT VALUES FOR EVALUATING FITNESS OF THE SOLUTIONS ##
inputs = ((0, 0), (0, 1), (1, 0), (1, 1))

def evaluate(genome):
    nn = NN(genome)

    # print(genome)
    outp0 = nn.compute(inputs[0])[0]
    nn.reset() # reset network otherwise it will remember previous state
    outp1 = nn.compute(inputs[1])[0]
    nn.reset()
    outp2 = nn.compute(inputs[2])[0]
    nn.reset()
    outp3 = nn.compute(inputs[3])[0]

    fitness = abs(outp0 - outp1) * abs(outp0 - outp2) * abs(outp3 - outp1) * abs(outp3 - outp2)
    return fitness

def complexity(species_list):
    n = (sum(1 for _ in ge.neuron_genes()) for ge in chain(*species_list))
    c = (ge.num_connection_genes() for ge in chain(*species_list))
    return sum(n), sum(c)

total_eval_time = [0]
total_neat_time = [0]

class Attempt:
    def __init__(self):
        self.evals_num = 0
        self.best_fitness = None
        self.best_genome = None
        self.target_reached = False

    def make_next_generation(self, pipeline, current_gen):
        self.evals_num += count_members(current_gen)

        t0 = time.perf_counter()
        evaluated = [[(genome, evaluate(genome)) for genome in species] for species in current_gen]
        t1 = time.perf_counter()
        total_eval_time[0] += t1 - t0

        next_gen = [list(produce_new_generation(pipeline, species)) for species in evaluated]
        t2 = time.perf_counter()
        total_neat_time[0] += t2 - t1

        self.best_genome, self.best_fitness = max(chain(*evaluated), key=itemgetter(1))
        return next_gen

    def get_stats(self, genomes):
        n_neurons, n_conns = complexity(genomes)
        return ("evals: {}, best genome has: {}N, {}C, complexity: {}N, {}C, best fitness = {}"
            .format(self.evals_num,
                sum(1 for _ in self.best_genome.neuron_genes()),
                self.best_genome.num_connection_genes(),
                n_neurons, n_conns, self.best_fitness))

## CREATE INITIAL GENERATION ##
def copy_with_mutation(pipeline, source_genome):
    return pipeline.topology_augmentation_step(pipeline.parameters_mutation_step(source_genome.copy()))

## CREATE MUTATOR ##
mutator = Mutator(
    neuron_factory=default_gene_factory(hidden_spec),
    connection_factory=default_gene_factory(connection_spec),
    channels=(
        # (input_spec, output_spec),
        (input_spec, hidden_spec),
        # (hidden_spec, hidden_spec),
        (hidden_spec, output_spec),
    )
)

def make_attempt(num_epochs, gens_per_epoch):
    augmentation_proba = .8
    reduction_proba = .8

    augment_gens_per_epoch = int(.4 * gens_per_epoch)
    reduct_gens_per_epoch = gens_per_epoch - augment_gens_per_epoch

    ## CREATE THE REPRODUCTION PIPELINE ##
    pipeline = make_pipeline(mutator)

    ## CREATE INITIAL GENOME ##
    # we specify initial input and output neurons and protect them from removal

    init_genome = mutator.produce_genome(
        in1=neuron(input_spec, non_removable=True),
        in2=neuron(input_spec, non_removable=True),
        out1=neuron(output_spec, non_removable=True),
        connections=(
            # connection(connection_spec, src='in1', dst='out1'),
            # connection(connection_spec, src='in2', dst='out1')
        )
    )

    current_gen = list(
        list(copy_with_mutation(pipeline, init_genome) for _ in range(species_size)) \
        for species_size in species_sizes(generation_size, num_species))

    attempt = Attempt()

    ## RUN ALTERNATING AUGMENTATION AND REDUCTION STAGES ##
    for epoch in range(num_epochs):
        # print("Epoch #{}".format(epoch))

        pipeline = make_pipeline(mutator, topology_augmentation_proba=augmentation_proba, topology_reduction_proba=0)

        for _ in range(augment_gens_per_epoch):
            current_gen = attempt.make_next_generation(pipeline, current_gen)
        # print("  AUG " + attempt.get_stats(current_gen))

        pipeline = make_pipeline(mutator, topology_augmentation_proba=0, topology_reduction_proba=reduction_proba)

        for _ in range(reduct_gens_per_epoch):
            current_gen = attempt.make_next_generation(pipeline, current_gen)
        # print("  RED " + attempt.get_stats(current_gen))

        if abs(attempt.best_fitness - 1) < 1e-6:
            attempt.target_reached = True
            break
    return attempt

def timestamp():
    ts = datetime.fromtimestamp(time.time(), tz=timezone.utc)
    return ts.strftime("%b %d %y %H:%M")

## RUN MULTIPLE ATTEMPTS TO CREATE A XOR NETWORK ##
best_genome = None

num_attempts = 50
total_eval_num = 0
success_count = 0
start_timer = time.perf_counter()

for attempt_id in range(num_attempts):
    # attempt_id += 100#20400

    # attempt_id+=1000

    num_epochs = 20
    gens_per_epoch = 250

    seed(attempt_id)
    attempt = make_attempt(num_epochs=num_epochs, gens_per_epoch=gens_per_epoch)

    if attempt.target_reached:
        success_count += 1
        best_genome = attempt.best_genome
    total_eval_num += attempt.evals_num
    result = dict(
        id = attempt_id,
        t = timestamp(),
        species = tuple(species_sizes(generation_size, num_species)),
        gens_in_epoch = gens_per_epoch,
        n_epochs = num_epochs,
        n_evals = attempt.evals_num,
        fitness = attempt.best_fitness,
        target_reached = attempt.target_reached,
    )
    print(result)

total_time = time.perf_counter() - start_timer
print(f"took {total_time} ticks")
print(f"total eval time {total_eval_time[0]} ticks")
print(f"total neat time {total_neat_time[0]} ticks")
print(f"eval time per 10000 evals: {10000*total_eval_time[0] / total_eval_num} ticks")
print(f"neat time per 10000 evals: {10000*total_neat_time[0] / total_eval_num} ticks")
print(f"neat time percentage = {100*total_neat_time[0] / total_time}")
print(f"total number of innovations: {mutator.innovation_number}")
print(f"success rate: {success_count} / {num_attempts}")

time_per_success = total_time / success_count if success_count > 0 else "N/A"
print(f"time per success: {time_per_success}")

# print("Final test:")
# final_nn = NN(best_genome)

# test_inputs = list(inputs) * 4
# random.shuffle(test_inputs)
# for inp in test_inputs:
#     final_nn.reset()
#     print("{} -> {}".format(inp, final_nn.compute(inp)[0]))

# write final genome as YAML file
if best_genome is None:
    print("could not reach the target")
else:
    filename = path.join(here_dir, "xor_genome.yaml")
    try:
        with open(filename, 'w+') as genfile:
            genfile.write(best_genome.to_yaml())
        print(f"solution was written to file {filename}")

    except Exception:
        pass
<|endoftext|>


<commit_before>VERSION="0.5.6.4"
<commit_msg>Version bump<commit_after>VERSION="0.5.6.4.1"
<|endoftext|>


<commit_before>#!/usr/bin/python

def make_weighted_corpus(testmode = False, maxweight = 9, fastmode = True):
    """
    make newest corpus, attempt 2
    two problems in weighter.py

        1. order in the cycle of iterations is important, when it shouldn't be
        2. infrequent labels don't make it up to the top n keywords for consideration

    to deal with these problems:

        for 1, we keep every function the same weight, except one. once it peaks in importance,
        we store its ideal weight
        for 2, weights don't just increment, they step up and down. this also saves time.

    once the script is done, we take the ideal weights and run that.
    """

    import os, glob, codecs, fileinput, pickle, re
    from contextlib import closing
    from collections import OrderedDict
    from itertools import izip

    labels = ['root', 'nsubj', 'acomp', 'agent', 'amod', 'nsubjpass', 'csubj',
              'csubjpass', 'dobj', 'ccomp', 'xcomp', 'iobj', 'advcl', 'advmod',
              'appos', 'acl', 'case', 'compound', 'dative', 'nummod', 'attr',
              'aux', 'auxpass', 'cc', 'complm', 'conj', 'dep', 'det', 'expl',
              'hmod', 'hyph', 'infmod', 'intj', 'mark', 'meta', 'neg', 'nmod', 'nn',
              'npadvmod', 'num', 'number', 'oprd', 'parataxis', 'partmod', 'pcomp',
              'predet', 'pobj', 'poss', 'possessive', 'preconj', 'prep', 'prt',
              'punct', 'quantmod', 'relcl']

    # make a list of labels and weights
    lab_scores = []
    for l in labels:
        for i in range(maxweight + 1):
            lab_scores.append((l, i))

    # paths to our data
    #dvtst_functions = 'parsed-data/functions-art.dev_test'
    train_functions = 'parsed-data/functions-art.train'
    #dvtst_lemmata = 'parsed-data/lemmata-art.dev_test'
    train_lemmata = 'parsed-data/lemmata-art.train'
    #dvtst_tokens = 'parsed-data/tokens-art.dev_test'
    train_tokens = 'parsed-data/tokens-art.train'
    output_train_name = 'risk.train'

    #dev_test = [dvtst_functions, dvtst_lemmata, dvtst_tokens]
    train = [train_functions, train_lemmata, train_tokens]

    baseline = float(codecs.open('baseline.score', 'r', encoding = 'utf-8').read().strip())
    prev_score = float(codecs.open('current.score', 'r', encoding = 'utf-8').read().strip())

    # add previous score to previous iteration
    if os.path.isfile('all.data'):
        data = pickle.load(codecs.open('all.data', 'rb'))
        if len(data[-1]) == 2:
            data[-1].append(prev_score)
        else:
            if len(data) == len(lab_scores):
                print('Apparently done.')
                print(0)
                return

        # if previous iteration was zero and it improved on baseline
        # skip further iterations on that function
        prev_func, prev_weight, prev_score = data[-1]
        if fastmode:
            if prev_weight == 0:
                if prev_score > baseline:
                    for x in range(1, maxweight + 1):
                        if x == 1:
                           scr = baseline
                        else:
                            scr = 0.0
                        data.append([prev_func, x, scr])
                else:
                    data.append([prev_func, 1, baseline])
            elif prev_weight == 1:
                data.append([prev_func, 1, baseline])
            elif prev_weight > 1 and prev_weight < maxweight:
                if prev_score <= baseline:
                    for x in range(prev_weight + 1, maxweight + 1):
                        data.append([prev_func, x, 0.0])
    else:
        data = []
    with codecs.open('all.data', 'wb', encoding = 'utf-8') as fo:
        pickle.dump(data, fo)

    # so now, at first iteration, we have []
    # or, at more than one iteration, we have:
    # [[funct, weight, score], ...]

    # now determine new stuff
    iteration = len(data)
    funct, weight = lab_scores[iteration]

    for index, (fline, lline, tline) in enumerate(izip(open(train[0], 'r'), open(train[1], 'r'), open(train[2], 'r'))):
        if index > 99:
            if testmode:
                break
        lline = lline.replace('  ', " , ")
        output_text = ''
        zipped = zip(fline.split(), lline.split(), tline.split())
        for f, l, t in zipped:
            if f == funct:
                score = weight
            else:
                score = 1
            if l == '-PRON-':
                continue
            if l == 'to':
                continue
            if score > 0:
                output_text += (l + ' ') * score

        if not os.path.isfile(output_train_name):
            open(output_train_name, 'w').close()
        with codecs.open(output_train_name, 'a', encoding = 'utf-8') as fo:
            fo.write(output_text.decode('utf-8') + '\n')

    newdata = [funct, weight]
    data.append(newdata)

    with codecs.open('all.data', 'wb', encoding = 'utf-8') as fo:
        pickle.dump(data, fo)
    print( 1 )
    return

if __name__ == '__main__':
    make_weighted_corpus()<commit_msg>final adjust<commit_after>#!/usr/bin/python

def make_weighted_corpus(testmode = False, maxweight = 9, fastmode = True):
    """
    make newest corpus, attempt 2
    two problems in weighter.py

        1. order in the cycle of iterations is important, when it shouldn't be
        2. infrequent labels don't make it up to the top n keywords for consideration

    to deal with these problems:

        for 1, we keep every function the same weight, except one. once it peaks in importance,
        we store its ideal weight
        for 2, weights don't just increment, they step up and down. this also saves time.

    once the script is done, we take the ideal weights and run that.
    """

    import os, glob, codecs, fileinput, pickle, re
    from contextlib import closing
    from collections import OrderedDict
    from itertools import izip

    labels = ['root', 'nsubj', 'acomp', 'agent', 'amod', 'nsubjpass', 'csubj',
              'csubjpass', 'dobj', 'ccomp', 'xcomp', 'iobj', 'advcl', 'advmod',
              'appos', 'acl', 'case', 'compound', 'dative', 'nummod', 'attr',
              'aux', 'auxpass', 'cc', 'complm', 'conj', 'dep', 'det', 'expl',
              'hmod', 'hyph', 'infmod', 'intj', 'mark', 'meta', 'neg', 'nmod', 'nn',
              'npadvmod', 'num', 'number', 'oprd', 'parataxis', 'partmod', 'pcomp',
              'predet', 'pobj', 'poss', 'possessive', 'preconj', 'prep', 'prt',
              'punct', 'quantmod', 'relcl']

    # make a list of labels and weights
    lab_scores = []
    for l in labels:
        for i in range(maxweight + 1):
            lab_scores.append((l, i))

    # paths to our data
    #dvtst_functions = 'parsed-data/functions-art.dev_test'
    train_functions = 'parsed-data/functions-art.train'
    #dvtst_lemmata = 'parsed-data/lemmata-art.dev_test'
    train_lemmata = 'parsed-data/lemmata-art.train'
    #dvtst_tokens = 'parsed-data/tokens-art.dev_test'
    train_tokens = 'parsed-data/tokens-art.train'
    output_train_name = 'risk.train'

    #dev_test = [dvtst_functions, dvtst_lemmata, dvtst_tokens]
    train = [train_functions, train_lemmata, train_tokens]

    baseline = float(codecs.open('baseline.score', 'r', encoding = 'utf-8').read().strip())
    prev_score = float(codecs.open('current.score', 'r', encoding = 'utf-8').read().strip())

    # add previous score to previous iteration
    if os.path.isfile('all.data'):
        data = pickle.load(codecs.open('all.data', 'rb'))
        if len(data[-1]) == 2:
            data[-1].append(prev_score)
        else:
            if len(data) == len(lab_scores):
                print('Apparently done.')
                print(0)
                return

        # if previous iteration was zero and it improved on baseline
        # skip further iterations on that function
        prev_func, prev_weight, prev_score = data[-1]
        if fastmode:
            if prev_weight == 0:
                if prev_score > baseline:
                    for x in range(1, maxweight + 1):
                        if x == 1:
                           scr = baseline
                        else:
                            scr = 0.0
                        data.append([prev_func, x, scr])
                else:
                    data.append([prev_func, 1, baseline])
            elif prev_weight == 1:
                data.append([prev_func, 1, baseline])
            elif prev_weight > 1 and prev_weight < maxweight:
                if prev_score <= data[-2][2]:
                    for x in range(prev_weight + 1, maxweight + 1):
                        data.append([prev_func, x, 0.0])
    else:
        data = []
    with codecs.open('all.data', 'wb', encoding = 'utf-8') as fo:
        pickle.dump(data, fo)

    # so now, at first iteration, we have []
    # or, at more than one iteration, we have:
    # [[funct, weight, score], ...]

    # now determine new stuff
    iteration = len(data)
    funct, weight = lab_scores[iteration]

    for index, (fline, lline, tline) in enumerate(izip(open(train[0], 'r'), open(train[1], 'r'), open(train[2], 'r'))):
        if index > 99:
            if testmode:
                break
        lline = lline.replace('  ', " , ")
        output_text = ''
        zipped = zip(fline.split(), lline.split(), tline.split())
        for f, l, t in zipped:
            if f == funct:
                score = weight
            else:
                score = 1
            if l == '-PRON-':
                continue
            if l == 'to':
                continue
            if score > 0:
                output_text += (l + ' ') * score

        if not os.path.isfile(output_train_name):
            open(output_train_name, 'w').close()
        with codecs.open(output_train_name, 'a', encoding = 'utf-8') as fo:
            fo.write(output_text.decode('utf-8') + '\n')

    newdata = [funct, weight]
    data.append(newdata)

    with codecs.open('all.data', 'wb', encoding = 'utf-8') as fo:
        pickle.dump(data, fo)
    print( 1 )
    return

if __name__ == '__main__':
    make_weighted_corpus()<|endoftext|>


<commit_before>from hashlib import sha1
from hmac import compare_digest
from os import stat
from os.path import isdir, join as path_join
import struct
import sys

from bencodepy import decode as bdecode

from xirvik.logging import cleanup

__all__ = [
    'cleanup_and_exit',
    'ctrl_c_handler',
    'VerificationError',
    'verify_torrent_contents',
]

def cleanup_and_exit(status=0):
    cleanup()
    sys.exit(status)

def ctrl_c_handler(signum, frame):
    cleanup()
    raise SystemExit('Signal raised')

def _chunks(l, n):
    """
    http://stackoverflow.com/a/312464/374110
    """
    for i in range(0, len(l), n):
        yield l[i:i + n]

def _get_torrent_pieces(filenames, basepath, piece_length):
    """
    Yes this is a generator and should not be used any other way (i.e. do not
    wrap in list()).
    """
    buf = b''
    p_delta = piece_length

    for name in filenames:
        name = path_join(basepath, name)
        try:
            size = stat(name).st_size
        except IOError:
            yield None

        if size <= piece_length and p_delta > size:
            with open(name, 'rb') as f:
                tmp = f.read()
                p_delta -= len(tmp)

                if p_delta <= 0:
                    p_delta = piece_length

                buf += tmp

            continue

        with open(name, 'rb') as f:
            while True:
                tmp = f.read(p_delta)
                p_delta -= len(tmp)

                if not tmp:
                    break

                buf += tmp

                if len(tmp) < p_delta:
                    break

                if p_delta <= 0:
                    yield buf
                    buf = b''
                    p_delta = piece_length

class VerificationError(Exception):
    pass

def verify_torrent_contents(torrent_file, path):
    orig_path = path

    if hasattr(torrent_file, 'seek') and hasattr(torrent_file, 'read'):
        torrent_file.seek(0)
        torrent = bdecode(torrent_file.read())
    else:
        try:
            with open(torrent_file, 'rb') as f:
                torrent = bdecode(f.read())
        except (IOError, TypeError):
            torrent = bdecode(torrent_file)

    path = path_join(path, torrent[b'info'][b'name'].decode('utf-8'))
    is_a_file = False
    try:
        with open(path, 'rb'):
            is_a_file = True
    except IOError:
        pass

    if not isdir(path) and not is_a_file:
        raise IOError('Path specified for torrent data is invalid')

    piece_length = torrent[b'info'][b'piece length']
    piece_hashes = torrent[b'info'][b'pieces']
    piece_hashes = struct.unpack('<{}B'.format(len(piece_hashes)),
                                 piece_hashes)
    piece_hashes = _chunks(piece_hashes, 20)

    try:
        filenames = ['/'.join([y.decode('utf-8') for y in x[b'path']])
                     for x in torrent[b'info'][b'files']]
    except KeyError as e:
        if e.args[0] != b'files':
            raise e

        # Single file torrent, never has a directory
        filenames = [path]
        path = orig_path

    pieces = _get_torrent_pieces(filenames, path, piece_length)

    for known_hash, piece in zip(piece_hashes, pieces):
        try:
            file_hash = sha1(piece).digest()
        except TypeError:
            raise VerificationError('Unable to get hash for piece')

        if not compare_digest(bytes(known_hash), file_hash):
            raise VerificationError('Computed hash does not match torrent '
                                    'file\'s hash')
<commit_msg>Fix possible false positive when last file is corrupted; use logical n argument to _chunks()<commit_after>from hashlib import sha1
from hmac import compare_digest
from os import stat
from os.path import isdir, join as path_join
import struct
import sys

from bencodepy import decode as bdecode

from xirvik.logging import cleanup

__all__ = [
    'cleanup_and_exit',
    'ctrl_c_handler',
    'VerificationError',
    'verify_torrent_contents',
]

def cleanup_and_exit(status=0):
    cleanup()
    sys.exit(status)

def ctrl_c_handler(signum, frame):
    cleanup()
    raise SystemExit('Signal raised')

def _chunks(l, n):
    """
    http://stackoverflow.com/a/312464/374110
    """
    for i in range(0, len(l), n):
        yield l[i:i + n]

def _get_torrent_pieces(filenames, basepath, piece_length):
    """
    Yes this is a generator and should not be used any other way (i.e. do not
    wrap in list()).
    """
    buf = b''
    p_delta = piece_length

    for name in filenames:
        name = path_join(basepath, name)
        try:
            size = stat(name).st_size
        except IOError:
            yield None

        if size <= piece_length and p_delta > size:
            with open(name, 'rb') as f:
                tmp = f.read()
                p_delta -= len(tmp)

                if p_delta <= 0:
                    p_delta = piece_length

                buf += tmp

            continue

        with open(name, 'rb') as f:
            while True:
                tmp = f.read(p_delta)
                p_delta -= len(tmp)

                if not tmp:
                    break

                buf += tmp

                if len(tmp) < p_delta:
                    break

                if p_delta <= 0:
                    yield buf
                    buf = b''
                    p_delta = piece_length

    # Very last set of bytes of the last file, and this will be <= piece size
    # If this is not returned, a false positive can be given if the last
    # file's last piece is not valid
    if buf:
        yield buf

class VerificationError(Exception):
    pass

def verify_torrent_contents(torrent_file, path):
    orig_path = path

    if hasattr(torrent_file, 'seek') and hasattr(torrent_file, 'read'):
        torrent_file.seek(0)
        torrent = bdecode(torrent_file.read())
    else:
        try:
            with open(torrent_file, 'rb') as f:
                torrent = bdecode(f.read())
        except (IOError, TypeError):
            torrent = bdecode(torrent_file)

    path = path_join(path, torrent[b'info'][b'name'].decode('utf-8'))
    is_a_file = False
    try:
        with open(path, 'rb'):
            is_a_file = True
    except IOError:
        pass

    if not isdir(path) and not is_a_file:
        raise IOError('Path specified for torrent data is invalid')

    piece_length = torrent[b'info'][b'piece length']
    piece_hashes = torrent[b'info'][b'pieces']
    piece_hashes = struct.unpack('<{}B'.format(len(piece_hashes)),
                                 piece_hashes)
    piece_hashes = _chunks(piece_hashes, sha1().digest_size)

    try:
        filenames = ['/'.join([y.decode('utf-8') for y in x[b'path']])
                     for x in torrent[b'info'][b'files']]
    except KeyError as e:
        if e.args[0] != b'files':
            raise e

        # Single file torrent, never has a directory
        filenames = [path]
        path = orig_path

    pieces = _get_torrent_pieces(filenames, path, piece_length)

    for known_hash, piece in zip(piece_hashes, pieces):
        try:
            file_hash = sha1(piece).digest()
        except TypeError:
            raise VerificationError('Unable to get hash for piece')

        if not compare_digest(bytes(known_hash), file_hash):
            raise VerificationError('Computed hash does not match torrent '
                                    'file\'s hash')
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
ffi.py

Created by Stephan Hügel on 2016-08-25

This file is part of polylabel-rs.

The MIT License (MIT)

Copyright (c) 2016 Stephan Hügel

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

"""

import os
from sys import platform
from ctypes import Structure, POINTER, c_void_p, c_size_t, c_double, cast, cdll
import numpy as np
import ipdb

file_path = os.path.dirname(__file__)
prefix = {'win32': ''}.get(platform, 'lib')
extension = {'darwin': '.dylib', 'win32': '.dll'}.get(platform, '.so')

lib = cdll.LoadLibrary(os.path.join(file_path, "target/release", prefix + "polylabel" + extension))

class _InnersArray(Structure):
    """
    Convert sequence of float lists to a C-compatible void array
    example: [[[1.0, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]]]

    Each sequence member is an interior Polygon ring

    """
    _fields_ = [("data", c_void_p),
                ("len", c_size_t)]

    @classmethod
    def from_param(cls, seq):
        """  Allow implicit conversions """
        return seq if isinstance(seq, cls) else cls(seq)

    def __init__(self, seq, data_type = c_double):
        self.arr = np.asarray([_FFIArray(s) for s in seq])
        ring_array_type = _FFIArray * len(seq)
        ring_array = ring_array_type()
        for i, arr in enumerate(self.arr):
            ring_array[i] = arr
        self.data = cast(ring_array, c_void_p)
        self.len = len(self.arr)

class _FFIArray(Structure):
    """
    Convert sequence of float lists to a C-compatible void array
    example: [[1.0, 2.0], [3.0, 4.0]]

    This represents a Polygon exterior ring
    """
    _fields_ = [("data", c_void_p),
                ("len", c_size_t)]

    @classmethod
    def from_param(cls, seq):
        """  Allow implicit conversions """
        return seq if isinstance(seq, cls) else cls(seq)

    def __init__(self, seq, data_type = c_double):
        self.data = cast(
            np.array(seq, dtype=np.float64).ctypes.data_as(POINTER(data_type)),
            c_void_p
        )
        self.len = len(seq)

class _CoordResult(Structure):
    """ Container for returned FFI coordinate data """
    _fields_ = [("x_pos", c_double), ("y_pos", c_double)]

def _unpack_coordresult(res, _func, _args):
    """ return our coordinates in a sensible format (a tuple) """
    return res.x_pos, res.y_pos

labelpos = lib.polylabel_ffi
labelpos.argtypes = (_FFIArray, _InnersArray, c_double)
labelpos.restype = _CoordResult
labelpos.errcheck = _unpack_coordresult

if __name__ == "__main__":
    # test that everything's working
    res = (labelpos(
        [[4.0, 1.0],
         [5.0, 2.0],
         [5.0, 3.0],
         [4.0, 4.0],
         [3.0, 4.0],
         [2.0, 3.0],
         [2.0, 2.0],
         [3.0, 1.0],
         [4.0, 1.0]],
        [
            [[3.5, 3.5], [4.4, 2.0], [2.6, 2.0], [3.5, 3.5]],
            [[4.0, 3.0], [4.0, 3.2], [4.5, 3.2], [4.0, 3.0]]
        ],
        0.1)
    )
    print(res)
<commit_msg>Add wrapper function<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
ffi.py

Created by Stephan Hügel on 2016-08-25

This file is part of polylabel-rs.

The MIT License (MIT)

Copyright (c) 2016 Stephan Hügel

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

"""

import os
from sys import platform
from ctypes import Structure, POINTER, c_void_p, c_size_t, c_double, cast, cdll
import numpy as np
from shapely.geometry import Polygon
import ipdb

file_path = os.path.dirname(__file__)
prefix = {'win32': ''}.get(platform, 'lib')
extension = {'darwin': '.dylib', 'win32': '.dll'}.get(platform, '.so')

lib = cdll.LoadLibrary(os.path.join(file_path, "target/release", prefix + "polylabel" + extension))

class _InnersArray(Structure):
    """
    Convert sequence of float lists to a C-compatible void array
    example: [[[1.0, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]]]

    Each sequence member is an interior Polygon ring

    """
    _fields_ = [("data", c_void_p),
                ("len", c_size_t)]

    @classmethod
    def from_param(cls, seq):
        """  Allow implicit conversions """
        return seq if isinstance(seq, cls) else cls(seq)

    def __init__(self, seq, data_type = c_double):
        self.arr = np.asarray([_FFIArray(s) for s in seq])
        ring_array_type = _FFIArray * len(seq)
        ring_array = ring_array_type()
        for i, arr in enumerate(self.arr):
            ring_array[i] = arr
        self.data = cast(ring_array, c_void_p)
        self.len = len(self.arr)

class _FFIArray(Structure):
    """
    Convert sequence of float lists to a C-compatible void array
    example: [[1.0, 2.0], [3.0, 4.0]]

    This represents a Polygon exterior ring
    """
    _fields_ = [("data", c_void_p),
                ("len", c_size_t)]

    @classmethod
    def from_param(cls, seq):
        """  Allow implicit conversions """
        return seq if isinstance(seq, cls) else cls(seq)

    def __init__(self, seq, data_type = c_double):
        self.data = cast(
            np.array(seq, dtype=np.float64).ctypes.data_as(POINTER(data_type)),
            c_void_p
        )
        self.len = len(seq)

class _CoordResult(Structure):
    """ Container for returned FFI coordinate data """
    _fields_ = [("x_pos", c_double), ("y_pos", c_double)]

def _unpack_coordresult(res, _func, _args):
    """ return our coordinates in a sensible format (a tuple) """
    return res.x_pos, res.y_pos

_labelpos = lib.polylabel_ffi
_labelpos.argtypes = (_FFIArray, _InnersArray, c_double)
_labelpos.restype = _CoordResult
_labelpos.errcheck = _unpack_coordresult

def label_position(ext, interiors=None, tolerance=1.0):
    """
    Calculate the optimum label position within a Polygon
    You may pass either a Shapely Polygon and a tolerance, or
    an exterior ring (list), a list of interior rings (list), and a tolerance

    Accepts Polygon instances as well as lists and lists of lists
    This is a terrible interface, but y'know, dynamic languages

    """
    if isinstance(ext, Polygon):
        _ext = list(ext.exterior.coords)
        _interiors = [list(ring.coords) for ring in ext.interiors]
        return _labelpos(_ext, _interiors, tolerance)
    else:
        return _labelpos(ext, interiors, tolerance)

if __name__ == "__main__":
    # test that everything's working
    res = label_position(
        [[4.0, 1.0],
         [5.0, 2.0],
         [5.0, 3.0],
         [4.0, 4.0],
         [3.0, 4.0],
         [2.0, 3.0],
         [2.0, 2.0],
         [3.0, 1.0],
         [4.0, 1.0]],
        interiors=[
            [[3.5, 3.5], [4.4, 2.0], [2.6, 2.0], [3.5, 3.5]],
            [[4.0, 3.0], [4.0, 3.2], [4.5, 3.2], [4.0, 3.0]]
        ],
        tolerance=0.1)
    print(res)
    pol = Polygon(
        [(4.0, 1.0),
         (5.0, 2.0),
         (5.0, 3.0),
         (4.0, 4.0),
         (3.0, 4.0),
         (2.0, 3.0),
         (2.0, 2.0),
         (3.0, 1.0),
         (4.0, 1.0)],
        [
         [(3.5, 3.5), (4.4, 2.0), (2.6, 2.0), (3.5, 3.5)],
         [(4.0, 3.0), (4.0, 3.2), (4.5, 3.2), (4.0, 3.0)]
        ])
    print label_position(pol, tolerance=0.1)
<|endoftext|>


<commit_before>#!/usr/bin/python

import os
import sys
import argparse
import traceback
from input_handling import findNumber
from group import Group,UnsortedInputGrouper
import scipy.stats

class FitGroup(Group):
    def __init__(self, tup):
        super(FitGroup, self).__init__(tup)
        self.rows = [[] for i in args.columns]

    def add(self, chunks):
        vals = [float(findNumber(chunks[i])) for i in args.columns]
        for v,r in zip(vals, self.rows):
            r.append(v)

    def done(self):
        jdelim = args.delimiter if args.delimiter != None else ' '
        for c,r in zip(args.columns, self.rows):
            if len(self.tup) > 0:
                args.outfile.write(jdelim.join(self.tup) + jdelim)
            if len(args.columns) > 1:
                args.outfile.write(str(c) + jdelim)
            shape_params = args.distf.fit(r, **eval(args.parameters))
            ks_res = scipy.stats.kstest(r, args.dist, shape_params)
            args.outfile.write(jdelim.join(map(str, shape_params)) + jdelim + jdelim.join(map(str, ks_res)) + '\n')

if __name__ == "__main__":
    # set up command line args
    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter,\
                                     description='Compute the distribution fit to column in the input')
    parser.add_argument('parameters', nargs='?', default="{}", help='dictionary of fit parameters')
    parser.add_argument('infile', nargs='?', type=argparse.FileType('r'), default=sys.stdin)
    parser.add_argument('outfile', nargs='?', type=argparse.FileType('w'), default=sys.stdout)
    parser.add_argument('-c', '--columns', nargs='+', type=int, default=[0])
    parser.add_argument('-i', '--dist', default='norm')
    parser.add_argument('-g', '--group', nargs='+', type=int, default=[])
    parser.add_argument('-d', '--delimiter', default=None)
    args = parser.parse_args()
    args.distf = getattr(scipy.stats, args.dist)

    grouper = UnsortedInputGrouper(args.infile, FitGroup, args.group, args.delimiter)
    grouper.group()

<commit_msg>change fit to allow fitting to multiple distributions<commit_after>#!/usr/bin/python

import os
import sys
import argparse
import traceback
from input_handling import findNumber
from group import Group,UnsortedInputGrouper
import scipy.stats

class FitGroup(Group):
    def __init__(self, tup):
        super(FitGroup, self).__init__(tup)
        self.rows = [[] for i in args.columns]

    def add(self, chunks):
        vals = [float(findNumber(chunks[i])) for i in args.columns]
        for v,r in zip(vals, self.rows):
            r.append(v)

    def done(self):
        jdelim = args.delimiter if args.delimiter != None else ' '
        for dist,i in zip(args.dist, args.distf):
            for c,r in zip(args.columns, self.rows):
                if len(self.tup) > 0:
                    args.outfile.write(jdelim.join(self.tup) + jdelim)
                if len(args.columns) > 1:
                    args.outfile.write(str(c) + jdelim)
                shape_params = i.fit(r, **eval(args.parameters))
                ks_res = scipy.stats.kstest(r, args.dist, shape_params)
                args.outfile.write(dist + jdelim + jdelim.join(map(str, shape_params)) + jdelim + jdelim.join(map(str, ks_res)) + '\n')

if __name__ == "__main__":
    # set up command line args
    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter,\
                                     description='Compute the distribution fit to column in the input')
    parser.add_argument('parameters', nargs='?', default="{}", help='dictionary of fit parameters')
    parser.add_argument('infile', nargs='?', type=argparse.FileType('r'), default=sys.stdin)
    parser.add_argument('outfile', nargs='?', type=argparse.FileType('w'), default=sys.stdout)
    parser.add_argument('-c', '--columns', nargs='+', type=int, default=[0])
    parser.add_argument('-i', '--dist', nargs='+', default='[norm]')
    parser.add_argument('-g', '--group', nargs='+', type=int, default=[])
    parser.add_argument('-d', '--delimiter', default=None)
    args = parser.parse_args()
    args.distf = []
    for i in args.dist:
        args.distf.append(getattr(scipy.stats, i))

    grouper = UnsortedInputGrouper(args.infile, FitGroup, args.group, args.delimiter)
    grouper.group()

<|endoftext|>


<commit_before>#!/usr/bin/env python3

import discord
import asyncio
import sys
sys.path.append("..")
import jauxiliar as jaux
import joseerror as je
import joseconfig as jconfig

import wolframalpha
wac = wolframalpha.Client(jconfig.WOLFRAMALPHA_APP_ID)

import pyowm
owm = pyowm.OWM(jconfig.OWM_APIKEY)

class JoseMath(jaux.Auxiliar):
    def __init__(self, cl):
        jaux.Auxiliar.__init__(self, cl)
        self.wac = wac
        self.owm = owm

    async def ext_load(self):
        return True, ''

    async def ext_unload(self):
        return True, ''

    async def c_wolframalpha(self, message, args, cxt):
        '''`!wolframalpha terms` - make a request to Wolfram|Alpha
        **ratelimit GLOBAL: 2 chamadas por hora**'''
        if len(args) < 2:
            await cxt.say(self.c_wolframalpha.__doc__)
            return

        term_to_wolfram = ' '.join(args[1:])

        res = self.wac.query(term_to_wolfram)
        if getattr(res, 'results', False):
            try:
                response_wolfram = next(res.results).text
            except StopIteration:
                await cxt.say(":warning: Erro tentando pegar o texto da resposta :warning:")
                return
            await cxt.say("%s:\n%s" % (term_to_wolfram, self.codeblock("", response_wolfram)))
        else:
            await cxt.say(":cyclone: Sem resposta :cyclone:")
            return

    async def c_wa(self, message, args, cxt):
        '''`!wa terms` - alias para `!wolframalpha`'''
        await self.c_wolframalpha(message, args, cxt)

    async def c_temperature(self, message, args, cxt):
        '''`!temperature location` - Temperatura de um local, usando OpenWeatherMap
        mostra tanto em Celsius quanto em Fahrenheit
        **ratelimit GLOBAL: 60 chamadas / minuto**'''
        if len(args) < 2:
            await cxt.say(self.c_temperature.__doc__)
            return

        location = ' '.join(args[1:])

        try:
            observation = owm.weather_at_place(location)
        except:
            await cxt.say("Erro tentando conseguir a temperatura para esse local")
            return
        w = observation.get_weather()

        tempcelsius = w.get_temperature("celsius")
        tempfahren = w.get_temperature("fahrenheit")

        celsiusnow = tempcelsius['temp']
        fahnow = tempfahren['temp']

        await cxt.say("%s °C, %s °F" % (celsiusnow, fahnow))

    async def c_temp(self, message, args, cxt):
        '''`!temp location` - alias para `!temperature`'''
        await self.c_temperature(message, args, cxt)

    async def c_therm(self, message, args, cxt):
        '''`!therm location` - alias para `!temperature`'''
        await self.c_temperature(message, args, cxt)

    def lewd(n):
        num = 3
        t = 0
        while t != n:
            lewd = num ** 2 - 2
            for a in range(2, lewd):
                if lewd % a == 0:
                    break
            else:
                yield num
                t += 1
            num += 6

    async def c_lewd(self, message, args, cxt):
        '''`!lewd n` - shows the `n` lewd numbers'''
        if len(args) < 2:
            await cxt.say(self.c_lewd.__doc__)

        try:
            n = int(args[1])
        except Exception as e:
            await cxt.say("Error parsing arguments: %r" % e)
            return

        if n > 100:
            await cxt.say("nope")
            return

        await cxt.say(self.codeblock("", list(lewd(n))))
<commit_msg>make lewd a method<commit_after>#!/usr/bin/env python3

import discord
import asyncio
import sys
sys.path.append("..")
import jauxiliar as jaux
import joseerror as je
import joseconfig as jconfig

import wolframalpha
wac = wolframalpha.Client(jconfig.WOLFRAMALPHA_APP_ID)

import pyowm
owm = pyowm.OWM(jconfig.OWM_APIKEY)

class JoseMath(jaux.Auxiliar):
    def __init__(self, cl):
        jaux.Auxiliar.__init__(self, cl)
        self.wac = wac
        self.owm = owm

    async def ext_load(self):
        return True, ''

    async def ext_unload(self):
        return True, ''

    async def c_wolframalpha(self, message, args, cxt):
        '''`!wolframalpha terms` - make a request to Wolfram|Alpha
        **ratelimit GLOBAL: 2 chamadas por hora**'''
        if len(args) < 2:
            await cxt.say(self.c_wolframalpha.__doc__)
            return

        term_to_wolfram = ' '.join(args[1:])

        res = self.wac.query(term_to_wolfram)
        if getattr(res, 'results', False):
            try:
                response_wolfram = next(res.results).text
            except StopIteration:
                await cxt.say(":warning: Erro tentando pegar o texto da resposta :warning:")
                return
            await cxt.say("%s:\n%s" % (term_to_wolfram, self.codeblock("", response_wolfram)))
        else:
            await cxt.say(":cyclone: Sem resposta :cyclone:")
            return

    async def c_wa(self, message, args, cxt):
        '''`!wa terms` - alias para `!wolframalpha`'''
        await self.c_wolframalpha(message, args, cxt)

    async def c_temperature(self, message, args, cxt):
        '''`!temperature location` - Temperatura de um local, usando OpenWeatherMap
        mostra tanto em Celsius quanto em Fahrenheit
        **ratelimit GLOBAL: 60 chamadas / minuto**'''
        if len(args) < 2:
            await cxt.say(self.c_temperature.__doc__)
            return

        location = ' '.join(args[1:])

        try:
            observation = owm.weather_at_place(location)
        except:
            await cxt.say("Erro tentando conseguir a temperatura para esse local")
            return
        w = observation.get_weather()

        tempcelsius = w.get_temperature("celsius")
        tempfahren = w.get_temperature("fahrenheit")

        celsiusnow = tempcelsius['temp']
        fahnow = tempfahren['temp']

        await cxt.say("%s °C, %s °F" % (celsiusnow, fahnow))

    async def c_temp(self, message, args, cxt):
        '''`!temp location` - alias para `!temperature`'''
        await self.c_temperature(message, args, cxt)

    async def c_therm(self, message, args, cxt):
        '''`!therm location` - alias para `!temperature`'''
        await self.c_temperature(message, args, cxt)

    def lewd(self, n):
        num = 3
        t = 0
        while t != n:
            lewd = num ** 2 - 2
            for a in range(2, lewd):
                if lewd % a == 0:
                    break
            else:
                yield num
                t += 1
            num += 6

    async def c_lewd(self, message, args, cxt):
        '''`!lewd n` - shows the `n` lewd numbers'''
        if len(args) < 2:
            await cxt.say(self.c_lewd.__doc__)

        try:
            n = int(args[1])
        except Exception as e:
            await cxt.say("Error parsing arguments: %r" % e)
            return

        if n > 100:
            await cxt.say("nope")
            return

        await cxt.say(self.codeblock("", list(self.lewd(n))))
<|endoftext|>


<commit_before>
# -*- coding: utf-8 -*-

import sys
import json
import logging
import traceback

import falcon
import xutils

from xutils.util import json_loads
from wsgiref.simple_server import WSGIRequestHandler, WSGIServer as _WSGIServer
try:
    from socketserver import ThreadingMixIn
except ImportError:
    from SocketServer import ThreadingMixIn

LOG = logging.getLogger("gunicorn.error" if "gunicorn" in sys.modules else __name__)

Application = API = falcon.API
STATUS_CODES = {}
for v in vars(falcon.status_codes).values():
    if xutils.is_string(v):
        try:
            code = int(v.split()[0])
            STATUS_CODES[code] = v
        except Exception:
            pass

class WSGIServer(ThreadingMixIn, _WSGIServer, object):
    daemon_threads = True       # The entire program exits when the main thread left.
    timeout = 30                # Wait until a request arrives or the timeout expires
    request_queue_size = 128    # The bocklog size of the listening socket
    allow_reuse_address = True  # Reuse the address listened to

    def __init__(self, addr, application=None, RequestHandlerClass=WSGIRequestHandler,
                 bind_and_activate=True):
        super(WSGIServer, self).__init__(addr, RequestHandlerClass, bind_and_activate)
        if application:
            self.set_app(application)

    # (xgfone) Support the with context manager for Python 2.7.
    def __enter__(self):
        return self

    def __exit__(self, *args):
        self.server_close()

def SimpleWSGIServer(app, host=None, port=None, log=None, **kwargs):
    with WSGIServer((host, port), app) as httpd:
        msg = "WSGI is listening on %s:%d" % (host, port)
        if log:
            log.info(msg)
        else:
            print(msg)
        httpd.serve_forever()

def EventletWSGIServer(app, sock=None, host=None, port=None, max_size=None, **kwargs):
    """A wrapper of eventlet.wsgi.server()."""
    from eventlet import listen, wsgi

    if not sock:
        sock = listen((host, port))
    wsgi.server(sock, app, max_size=max_size, **kwargs)

class Resource(object):
    def status(self, status):
        if isinstance(status, int):
            return STATUS_CODES[status]
        return status

    def respond(self, resp, status=None, body=None, content_type=None):
        resp.status = self.status(status or falcon.HTTP_200)
        if content_type:
            resp.content_type = content_type
        if body:
            resp.body = body

    def dump_json(self, resp, result, status=None, separators=(',', ':')):
        body = json.dumps(result, separators=separators)
        self.respond(resp, status, body, falcon.MEDIA_JSON)

    def load_json(self, req, data=None):
        data = data or req.bounded_stream.read()
        return json_loads(data) if data else None

class Router(falcon.routing.DefaultRouter):
    def _get_action(self, resource, action):
        return action if callable(action) else getattr(resource, action)

    def map_http_methods(self, resource, **kwargs):
        """Map the HTTP methods.

        Support:
            >>> add_route("/path/to", resource) # The origin
            >>> add_route("/path/to", resource, map={"GET": "resource_method"})
            >>> add_route("/path/to", resource, map={"GET": resource.method})
            >>> add_route("/path/to", resource, get="resource_method")
            >>> add_route("/path/to", resource, get=resource.method)

        For Example:
            >>> class Resource(object):
            >>>     def on_get(self, req, resp, name):
            >>>         resp.body = name
            >>>
            >>> resource = Resource()
            >>> add_route("/v1/hello/{name}", resource)
            >>> add_route("/v2/hello/{name}", resource, map={"GET": "on_get"})
            >>> add_route("/v3/hello/{name}", resource, map={"GET": resource.on_get})
            >>> add_route("/v3/hello/{name}", resource, get="on_get")
            >>> add_route("/v3/hello/{name}", resource, get=resource.on_get)
        """

        m = kwargs.pop("map", None)
        if m:
            return {k.upper(): self._get_action(resource, v) for k, v in m.items()}

        mmap = {}
        for method in tuple(kwargs.keys()):
            _method = method.upper()
            if _method in falcon.COMBINED_METHODS:
                mmap[_method] = self._get_action(resource, kwargs.pop(method))

        return mmap if mmap else self.map_http_methods(resource, **kwargs)

def append_error_handler(app, exception, handler=None):
    app._error_handlers.append((exception, handler or exception.handle))

def get_exception_handler(get_body=None, traceback_exception=False):
    """Return an exception handler of falcon.

    Keyword Arguments:
        get_body (function): a function to return a response body.
            Its arguments are the same of the exception handler of falcon,
            that's, ``get_body(ex, req, resp, params)``. Its return value
            will be assigned to ``resp.body``.
    """

    def _traceback_exception(ex, req, resp, params):
        resp.content_type = falcon.MEDIA_TEXT
        resp.status = falcon.HTTP_500
        resp.body = get_body(ex, req, resp, params) if callable(get_body) else str(ex)
        LOG.error("Get an exception: method=%s, url=%s, err=%s", req.method, req.uri, ex)
        if traceback_exception:
            LOG.error(traceback.format_exc())

    return _traceback_exception

def _add_route(self, uri_template, resource, *args, **kwargs):
    if not xutils.is_string(uri_template):
        raise TypeError('uri_template is not a string')

    if not uri_template.startswith('/'):
        raise ValueError("uri_template must start with '/'")

    if '//' in uri_template:
        raise ValueError("uri_template may not contain '//'")

    if kwargs and hasattr(self._router, "map_http_methods"):
        method_map = self._router.map_http_methods(resource, **kwargs)
    else:
        method_map = falcon.routing.map_http_methods(resource)
    falcon.routing.set_default_responders(method_map)
    self._router.add_route(uri_template, method_map, resource)
falcon.API.add_route = _add_route

if __name__ == "__main__":

    class _Resource(object):
        def on_get(self, req, resp, name):
            resp.body = name

    resource = _Resource()
    app = falcon.API(router=Router())
    append_error_handler(app, Exception, get_exception_handler())
    app.add_route("/v1/hello/{name}", resource)
    app.add_route("/v2/hello/{name}", resource, map={"GET": "on_get"})
    app.add_route("/v3/hello/{name}", resource, map={"GET": resource.on_get})
    app.add_route("/v4/hello/{name}", resource, get="on_get")
    app.add_route("/v5/hello/{name}", resource, get=resource.on_get)

    port = int(sys.argv[1]) if len(sys.argv) > 1 else 8888
    with WSGIServer(("", port), app) as httpd:
        print("WSGI server listening on %s" % port)
        httpd.serve_forever()
<commit_msg>add the log middleware for wsgi server<commit_after>
# -*- coding: utf-8 -*-

import sys
import json
import time
import logging
import traceback

import falcon
import xutils

from xutils.util import json_loads
from wsgiref.simple_server import WSGIRequestHandler, WSGIServer as _WSGIServer
try:
    from socketserver import ThreadingMixIn
except ImportError:
    from SocketServer import ThreadingMixIn

LOG = logging.getLogger("gunicorn.error" if "gunicorn" in sys.modules else __name__)

Application = API = falcon.API
STATUS_CODES = {}
for v in vars(falcon.status_codes).values():
    if xutils.is_string(v):
        try:
            code = int(v.split()[0])
            STATUS_CODES[code] = v
        except Exception:
            pass

class WSGIServer(ThreadingMixIn, _WSGIServer, object):
    daemon_threads = True       # The entire program exits when the main thread left.
    timeout = 30                # Wait until a request arrives or the timeout expires
    request_queue_size = 128    # The bocklog size of the listening socket
    allow_reuse_address = True  # Reuse the address listened to

    def __init__(self, addr, application=None, RequestHandlerClass=WSGIRequestHandler,
                 bind_and_activate=True):
        super(WSGIServer, self).__init__(addr, RequestHandlerClass, bind_and_activate)
        if application:
            self.set_app(application)

    # (xgfone) Support the with context manager for Python 2.7.
    def __enter__(self):
        return self

    def __exit__(self, *args):
        self.server_close()

def SimpleWSGIServer(app, host=None, port=None, log=None, **kwargs):
    with WSGIServer((host, port), app) as httpd:
        msg = "WSGI is listening on %s:%d" % (host, port)
        if log:
            log.info(msg)
        else:
            print(msg)
        httpd.serve_forever()

def EventletWSGIServer(app, sock=None, host=None, port=None, max_size=None, **kwargs):
    """A wrapper of eventlet.wsgi.server()."""
    from eventlet import listen, wsgi

    if not sock:
        sock = listen((host, port))
    wsgi.server(sock, app, max_size=max_size, **kwargs)

class Resource(object):
    def status(self, status):
        if isinstance(status, int):
            return STATUS_CODES[status]
        return status

    def respond(self, resp, status=None, body=None, content_type=None):
        resp.status = self.status(status or falcon.HTTP_200)
        if content_type:
            resp.content_type = content_type
        if body:
            resp.body = body

    def dump_json(self, resp, result, status=None, separators=(',', ':')):
        body = json.dumps(result, separators=separators)
        self.respond(resp, status, body, falcon.MEDIA_JSON)

    def load_json(self, req, data=None):
        data = data or req.bounded_stream.read()
        return json_loads(data) if data else None

class LogMiddleware(object):
    def __init__(self, logger=None, level="info"):
        if not logger:
            logger = logging.getLogger()
        self._output = getattr(logger, level.lower())

    def process_request(self, req, resp):
        req._starttime = time.time()

    def process_response(self, req, resp, resource, req_succeeded):
        req._endtime = time.time()
        self._output("Request: method=%s, url=%s, start=%s, end=%s, cost=%s",
                     req.method, req.url, req._starttime, req._endtime,
                     req._endtime - req._starttime)

class Router(falcon.routing.DefaultRouter):
    def _get_action(self, resource, action):
        return action if callable(action) else getattr(resource, action)

    def map_http_methods(self, resource, **kwargs):
        """Map the HTTP methods.

        Support:
            >>> add_route("/path/to", resource) # The origin
            >>> add_route("/path/to", resource, map={"GET": "resource_method"})
            >>> add_route("/path/to", resource, map={"GET": resource.method})
            >>> add_route("/path/to", resource, get="resource_method")
            >>> add_route("/path/to", resource, get=resource.method)

        For Example:
            >>> class Resource(object):
            >>>     def on_get(self, req, resp, name):
            >>>         resp.body = name
            >>>
            >>> resource = Resource()
            >>> add_route("/v1/hello/{name}", resource)
            >>> add_route("/v2/hello/{name}", resource, map={"GET": "on_get"})
            >>> add_route("/v3/hello/{name}", resource, map={"GET": resource.on_get})
            >>> add_route("/v3/hello/{name}", resource, get="on_get")
            >>> add_route("/v3/hello/{name}", resource, get=resource.on_get)
        """

        m = kwargs.pop("map", None)
        if m:
            return {k.upper(): self._get_action(resource, v) for k, v in m.items()}

        mmap = {}
        for method in tuple(kwargs.keys()):
            _method = method.upper()
            if _method in falcon.COMBINED_METHODS:
                mmap[_method] = self._get_action(resource, kwargs.pop(method))

        return mmap if mmap else self.map_http_methods(resource, **kwargs)

def append_error_handler(app, exception, handler=None):
    app._error_handlers.append((exception, handler or exception.handle))

def get_exception_handler(get_body=None, traceback_exception=False):
    """Return an exception handler of falcon.

    Keyword Arguments:
        get_body (function): a function to return a response body.
            Its arguments are the same of the exception handler of falcon,
            that's, ``get_body(ex, req, resp, params)``. Its return value
            will be assigned to ``resp.body``.
    """

    def _traceback_exception(ex, req, resp, params):
        resp.content_type = falcon.MEDIA_TEXT
        resp.status = falcon.HTTP_500
        resp.body = get_body(ex, req, resp, params) if callable(get_body) else str(ex)
        LOG.error("Get an exception: method=%s, url=%s, err=%s", req.method, req.uri, ex)
        if traceback_exception:
            LOG.error(traceback.format_exc())

    return _traceback_exception

def _add_route(self, uri_template, resource, *args, **kwargs):
    if not xutils.is_string(uri_template):
        raise TypeError('uri_template is not a string')

    if not uri_template.startswith('/'):
        raise ValueError("uri_template must start with '/'")

    if '//' in uri_template:
        raise ValueError("uri_template may not contain '//'")

    if kwargs and hasattr(self._router, "map_http_methods"):
        method_map = self._router.map_http_methods(resource, **kwargs)
    else:
        method_map = falcon.routing.map_http_methods(resource)
    falcon.routing.set_default_responders(method_map)
    self._router.add_route(uri_template, method_map, resource)
falcon.API.add_route = _add_route

if __name__ == "__main__":

    class _Resource(object):
        def on_get(self, req, resp, name):
            resp.body = name

    resource = _Resource()
    app = falcon.API(router=Router())
    append_error_handler(app, Exception, get_exception_handler())
    app.add_route("/v1/hello/{name}", resource)
    app.add_route("/v2/hello/{name}", resource, map={"GET": "on_get"})
    app.add_route("/v3/hello/{name}", resource, map={"GET": resource.on_get})
    app.add_route("/v4/hello/{name}", resource, get="on_get")
    app.add_route("/v5/hello/{name}", resource, get=resource.on_get)

    port = int(sys.argv[1]) if len(sys.argv) > 1 else 8888
    with WSGIServer(("", port), app) as httpd:
        print("WSGI server listening on %s" % port)
        httpd.serve_forever()
<|endoftext|>


<commit_before>"""
BaseOAuth is inspired from Darren Kempiners YahooAPI https://github.com/dkempiners/python-yahooapi/blob/master/yahooapi.py
"""
from __future__ import absolute_import

try:
    input = raw_input
except NameError:
    pass

import pdb
import json
import time
import logging
import webbrowser

import base64

from rauth import OAuth1Service, OAuth2Service
from rauth.utils import parse_utf8_qsl

logging.basicConfig(level=logging.DEBUG, format="[%(asctime)s %(levelname)s] [%(name)s.%(module)s.%(funcName)s] %(message)s")
logging.getLogger('yahoo-oauth')

services = {
    'oauth1': dict(
        SERVICE = OAuth1Service,
        REQUEST_TOKEN_URL = "https://api.login.yahoo.com/oauth/v2/get_request_token",
        ACCESS_TOKEN_URL = "https://api.login.yahoo.com/oauth/v2/get_token",
        AUTHORIZE_TOKEN_URL = "https://api.login.yahoo.com/oauth/v2/request_auth"
    ),
    'oauth2': dict(
        SERVICE = OAuth2Service,
        AUTHORIZE_TOKEN_URL = "https://api.login.yahoo.com/oauth2/request_auth",
        ACCESS_TOKEN_URL = "https://api.login.yahoo.com/oauth2/get_token"
    )
}
CALLBACK_URI = 'oob'

def json_write_data(json_data, filename):
    """Write json data into a file
    """
    with open(filename, 'w') as fp:
        json.dump(json_data, fp, indent=4, sort_keys=True, ensure_ascii=False)
        return True
    return False

def json_get_data(filename):
    """Get data from json file
    """
    with open(filename) as fp:
        json_data = json.load(fp)
        return json_data

    return False

class BaseOAuth(object):
    """
    """
    def __init__(self, oauth_version, consumer_key, consumer_secret, **kwargs):
        """
        consumer_key : client key
        consumer_secret : client secret
        access_token : access token
        access_token_secret : access token secret
        from_file : file containing the credentials
        """
        self.oauth_version = oauth_version

        if kwargs.get('from_file'):
            logging.debug("Checking ")
            self.from_file = kwargs.get('from_file')
            json_data = json_get_data(self.from_file)
            vars(self).update(json_data)
        else:
            self.consumer_key = consumer_key
            self.consumer_secret = consumer_secret

        vars(self).update(kwargs)

        self.oauth_version = oauth_version
        self.callback_uri = vars(self).get('callback_uri',CALLBACK_URI)

        # Init OAuth
        if self.oauth_version == 'oauth1':
            service_params = {
                'consumer_key': self.consumer_key,
                'consumer_secret' : self.consumer_secret,
                'request_token_url': services[self.oauth_version]['REQUEST_TOKEN_URL']
            }
        else:
            service_params = {
                'client_id': self.consumer_key,
                'client_secret': self.consumer_secret
            }

        service_params.update({
            'name' : 'yahoo',
            'access_token_url' : services[self.oauth_version]['ACCESS_TOKEN_URL'],
            'authorize_url' : services[self.oauth_version]['AUTHORIZE_TOKEN_URL'],
            'base_url': vars(self).get('base_url',None)
        })

        # Defining oauth service
        self.oauth = services[oauth_version]['SERVICE'](**service_params)

        if vars(self).get('access_token') and vars(self).get('access_token_secret') and vars(self).get('session_handle'):
            if not self.token_is_valid():
                #self.session = self.refresh_token()
                json_data.update(self.refresh_token())
        elif vars(self).get('access_token') and vars(self).get('token_type') and vars(self).get('refresh_token'):
            if not self.token_is_valid():
                #self.session = self.refresh_token()
                json_data.update(self.refresh_token)
        else:
            json_data.update(self.handler())

        # Getting session
        if self.oauth_version == 'oauth1':
            self.session = self.oauth.get_session((self.access_token, self.access_token_secret))
        else:
            #self.session = self.oauth.get_auth_session(data={'code': self.verifier, 'redirect_uri': self.callback_uri}, decoder=lambda b: json.loads(str(b)))
            #self.session = self.oauth.get_auth_session(data={'code': self.verifier, 'redirect_uri': self.callback_uri}, decoder=self.custom_decoder)
            self.session = self.oauth.get_session(token=self.access_token)

        json_write_data(json_data, self.from_file)

    def handler(self,):
        """* get request token if OAuth1
            * Get user authorization
            * Get access token
        """

        if self.oauth_version == 'oauth1':
            request_token, request_token_secret = self.oauth.get_request_token(params={'oauth_callback': self.callback_uri})
            logging.debug("REQUEST_TOKEN = {0}\n REQUEST_TOKEN_SECRET = {1}\n".format(request_token, request_token_secret))
            authorize_url = self.oauth.get_authorize_url(request_token)
        else:
            authorize_url = self.oauth.get_authorize_url(client_secret=self.consumer_secret, redirect_uri=self.callback_uri, response_type='code')

        logging.debug("AUTHORISATION URL : {0}".format(authorize_url))
        # Open authorize_url
        webbrowser.open(authorize_url)
        self.verifier = input("Enter verifier : ")

        self.token_time = time.time()

        credentials = {'token_time': self.token_time}

        if self.oauth_version == 'oauth1':
            raw_access = self.oauth.get_raw_access_token(request_token, request_token_secret, params={"oauth_verifier": verifier})
            parsed_access = parse_utf8_qsl(raw_access.content)

            self.access_token = parsed_access['oauth_token']
            self.access_token_secret = parsed_access['oauth_token_secret']
            self.session_handle = parsed_access['oauth_session_handle']

            # Updating credentials
            credentials.update({
                'access_token': self.access_token,
                'access_token_secret': self.access_token_secret,
                'session_handle': self.session_handle
            })
        else:
            # Building headers
            headers = self.generate_oauth2_headers()
            # Getting access token
            raw_access = self.oauth.get_raw_access_token(data={"code": self.verifier, 'redirect_uri': self.callback_uri,'grant_type':'authorization_code'}, headers=headers)
            #parsed_access = parse_utf8_qsl(raw_access.content.decode('utf-8'))
            credentials.update(self.oauth2_access_parser(raw_access))

        return credentials

    def generate_oauth2_headers(self):
        """Generates header for oauth2
        """
        encoded_credentials = base64.b64encode(('{0}:{1}'.format(self.consumer_key,self.consumer_secret)).encode('utf-8'))
        headers={'Authorization':'Basic {0}'.format(encoded_credentials.decode('utf-8'))}

        return headers

    def custom_decoder(self, data, encoding='utf-8'):
        """Custom decoder for auth_session
        """
        import pdb
        pdb.set_trace()
        result = data.decode(encoding)
        if result:
            json_data = json.loads(result)
            return json_data
        else:
            return {}

    def oauth2_access_parser(self, raw_access):
        """Parse oauth2 access
        """
        parsed_access = json.loads(raw_access.content.decode('utf-8'))
        self.access_token = parsed_access['access_token']
        self.token_type = parsed_access['token_type']
        self.refresh_token = parsed_access['refresh_token']

        credentials = {
            'access_token': self.access_token,
            'token_type': self.token_type,
            'refresh_token': self.refresh_token
        }

        return credentials

    def refresh_token(self,):
        """Refresh access token
        """
        logging.debug("REFRESHING TOKEN")
        self.token_time = time.time()
        if self.oauth_version == 'oauth1':
            self.access_token, self.access_token_secret = self.oauth.get_access_token(self.access_token, self.access_token_secret, params={"oauth_session_handle": self.session_handle})
            credentials = {
                'access_token': self.access_token,
                'access_token_secret': self.access_token_secret,
                'session_handle': self.session_handle,
                'token_time': self.token_time
            }
        else:
            headers = self.generate_oauth2_headers()

            raw_access = self.oauth.get_raw_access_token(data={"refresh_token": self.refresh_token, 'redirect_uri': self.callback_uri,'grant_type':'refresh_token'}, headers=headers)
            credentials.update(self.oauth2_access_parser(raw_access))

        return credentials

    def token_is_valid(self,):
        """Check the validity of the token :3600s
        """
        elapsed_time = time.time() - self.token_time
        logging.debug("ELAPSED TIME : {0}".format(elapsed_time))
        if elapsed_time > 3540: # 1 minute before it expires
            logging.debug("TOKEN HAS EXPIRED")
            return False

        logging.debug("TOKEN IS STILL VALID")
        return True

class OAuth1(BaseOAuth):
    """Class handling OAuth v1
    """

    def __init__(self, consumer_key, consumer_secret, **kwargs):

        super(OAuth1, self).__init__('oauth1', consumer_key, consumer_secret, **kwargs)

class OAuth2(BaseOAuth):
    """Calss handling OAuth v2
    """

    def __init__(self, consumer_key, consumer_secret, **kwargs):

        super(OAuth2, self).__init__('oauth2', consumer_key, consumer_secret, **kwargs)

<commit_msg>fix refresh token issue for oauth2<commit_after>"""
BaseOAuth is inspired from Darren Kempiners YahooAPI https://github.com/dkempiners/python-yahooapi/blob/master/yahooapi.py
"""
from __future__ import absolute_import

try:
    input = raw_input
except NameError:
    pass

import pdb
import json
import time
import logging
import webbrowser

import base64

from rauth import OAuth1Service, OAuth2Service
from rauth.utils import parse_utf8_qsl

logging.basicConfig(level=logging.DEBUG, format="[%(asctime)s %(levelname)s] [%(name)s.%(module)s.%(funcName)s] %(message)s")
logging.getLogger('yahoo-oauth')

services = {
    'oauth1': dict(
        SERVICE = OAuth1Service,
        REQUEST_TOKEN_URL = "https://api.login.yahoo.com/oauth/v2/get_request_token",
        ACCESS_TOKEN_URL = "https://api.login.yahoo.com/oauth/v2/get_token",
        AUTHORIZE_TOKEN_URL = "https://api.login.yahoo.com/oauth/v2/request_auth"
    ),
    'oauth2': dict(
        SERVICE = OAuth2Service,
        AUTHORIZE_TOKEN_URL = "https://api.login.yahoo.com/oauth2/request_auth",
        ACCESS_TOKEN_URL = "https://api.login.yahoo.com/oauth2/get_token"
    )
}
CALLBACK_URI = 'oob'

def json_write_data(json_data, filename):
    """Write json data into a file
    """
    with open(filename, 'w') as fp:
        json.dump(json_data, fp, indent=4, sort_keys=True, ensure_ascii=False)
        return True
    return False

def json_get_data(filename):
    """Get data from json file
    """
    with open(filename) as fp:
        json_data = json.load(fp)
        return json_data

    return False

class BaseOAuth(object):
    """
    """
    def __init__(self, oauth_version, consumer_key, consumer_secret, **kwargs):
        """
        consumer_key : client key
        consumer_secret : client secret
        access_token : access token
        access_token_secret : access token secret
        from_file : file containing the credentials
        """
        self.oauth_version = oauth_version

        if kwargs.get('from_file'):
            logging.debug("Checking ")
            self.from_file = kwargs.get('from_file')
            json_data = json_get_data(self.from_file)
            vars(self).update(json_data)
        else:
            self.consumer_key = consumer_key
            self.consumer_secret = consumer_secret

        vars(self).update(kwargs)

        self.oauth_version = oauth_version
        self.callback_uri = vars(self).get('callback_uri',CALLBACK_URI)

        # Init OAuth
        if self.oauth_version == 'oauth1':
            service_params = {
                'consumer_key': self.consumer_key,
                'consumer_secret' : self.consumer_secret,
                'request_token_url': services[self.oauth_version]['REQUEST_TOKEN_URL']
            }
        else:
            service_params = {
                'client_id': self.consumer_key,
                'client_secret': self.consumer_secret
            }

        service_params.update({
            'name' : 'yahoo',
            'access_token_url' : services[self.oauth_version]['ACCESS_TOKEN_URL'],
            'authorize_url' : services[self.oauth_version]['AUTHORIZE_TOKEN_URL'],
            'base_url': vars(self).get('base_url',None)
        })

        # Defining oauth service
        self.oauth = services[oauth_version]['SERVICE'](**service_params)

        if vars(self).get('access_token') and vars(self).get('access_token_secret') and vars(self).get('session_handle'):
            if not self.token_is_valid():
                json_data.update(self.refresh_access_token())
        elif vars(self).get('access_token') and vars(self).get('token_type') and vars(self).get('refresh_token'):
            if not self.token_is_valid():
                json_data.update(self.refresh_access_token())
        else:
            json_data.update(self.handler())

        # Getting session
        if self.oauth_version == 'oauth1':
            self.session = self.oauth.get_session((self.access_token, self.access_token_secret))
        else:
            #self.session = self.oauth.get_auth_session(data={'code': self.verifier, 'redirect_uri': self.callback_uri}, decoder=lambda b: json.loads(str(b)))
            #self.session = self.oauth.get_auth_session(data={'code': self.verifier, 'redirect_uri': self.callback_uri}, decoder=self.custom_decoder)
            self.session = self.oauth.get_session(token=self.access_token)

        json_write_data(json_data, self.from_file)

    def handler(self,):
        """* get request token if OAuth1
            * Get user authorization
            * Get access token
        """

        if self.oauth_version == 'oauth1':
            request_token, request_token_secret = self.oauth.get_request_token(params={'oauth_callback': self.callback_uri})
            logging.debug("REQUEST_TOKEN = {0}\n REQUEST_TOKEN_SECRET = {1}\n".format(request_token, request_token_secret))
            authorize_url = self.oauth.get_authorize_url(request_token)
        else:
            authorize_url = self.oauth.get_authorize_url(client_secret=self.consumer_secret, redirect_uri=self.callback_uri, response_type='code')

        logging.debug("AUTHORISATION URL : {0}".format(authorize_url))
        # Open authorize_url
        webbrowser.open(authorize_url)
        self.verifier = input("Enter verifier : ")

        self.token_time = time.time()

        credentials = {'token_time': self.token_time}

        if self.oauth_version == 'oauth1':
            raw_access = self.oauth.get_raw_access_token(request_token, request_token_secret, params={"oauth_verifier": verifier})
            parsed_access = parse_utf8_qsl(raw_access.content)

            self.access_token = parsed_access['oauth_token']
            self.access_token_secret = parsed_access['oauth_token_secret']
            self.session_handle = parsed_access['oauth_session_handle']

            # Updating credentials
            credentials.update({
                'access_token': self.access_token,
                'access_token_secret': self.access_token_secret,
                'session_handle': self.session_handle
            })
        else:
            # Building headers
            headers = self.generate_oauth2_headers()
            # Getting access token
            raw_access = self.oauth.get_raw_access_token(data={"code": self.verifier, 'redirect_uri': self.callback_uri,'grant_type':'authorization_code'}, headers=headers)
            #parsed_access = parse_utf8_qsl(raw_access.content.decode('utf-8'))
            credentials.update(self.oauth2_access_parser(raw_access))

        return credentials

    def generate_oauth2_headers(self):
        """Generates header for oauth2
        """
        encoded_credentials = base64.b64encode(('{0}:{1}'.format(self.consumer_key,self.consumer_secret)).encode('utf-8'))
        headers={'Authorization':'Basic {0}'.format(encoded_credentials.decode('utf-8'))}

        return headers

    def custom_decoder(self, data, encoding='utf-8'):
        """Custom decoder for auth_session
        """
        result = data.decode(encoding)
        if result:
            json_data = json.loads(result)
            return json_data
        else:
            return {}

    def oauth2_access_parser(self, raw_access):
        """Parse oauth2 access
        """
        parsed_access = json.loads(raw_access.content.decode('utf-8'))
        self.access_token = parsed_access['access_token']
        self.token_type = parsed_access['token_type']
        self.refresh_token = parsed_access['refresh_token']

        credentials = {
            'access_token': self.access_token,
            'token_type': self.token_type,
            'refresh_token': self.refresh_token
        }

        return credentials

    def refresh_access_token(self,):
        """Refresh access token
        """
        logging.debug("REFRESHING TOKEN")
        self.token_time = time.time()
        credentials = {
            'token_time': self.token_time
        }

        if self.oauth_version == 'oauth1':
            self.access_token, self.access_token_secret = self.oauth.get_access_token(self.access_token, self.access_token_secret, params={"oauth_session_handle": self.session_handle})
            credentials.update({
                'access_token': self.access_token,
                'access_token_secret': self.access_token_secret,
                'session_handle': self.session_handle,
                'token_time': self.token_time
            })
        else:
            headers = self.generate_oauth2_headers()

            raw_access = self.oauth.get_raw_access_token(data={"refresh_token": self.refresh_token, 'redirect_uri': self.callback_uri,'grant_type':'refresh_token'}, headers=headers)
            credentials.update(self.oauth2_access_parser(raw_access))

        return credentials

    def token_is_valid(self,):
        """Check the validity of the token :3600s
        """
        elapsed_time = time.time() - self.token_time
        logging.debug("ELAPSED TIME : {0}".format(elapsed_time))
        if elapsed_time > 3540: # 1 minute before it expires
            logging.debug("TOKEN HAS EXPIRED")
            return False

        logging.debug("TOKEN IS STILL VALID")
        return True

class OAuth1(BaseOAuth):
    """Class handling OAuth v1
    """

    def __init__(self, consumer_key, consumer_secret, **kwargs):

        super(OAuth1, self).__init__('oauth1', consumer_key, consumer_secret, **kwargs)

class OAuth2(BaseOAuth):
    """Calss handling OAuth v2
    """

    def __init__(self, consumer_key, consumer_secret, **kwargs):

        super(OAuth2, self).__init__('oauth2', consumer_key, consumer_secret, **kwargs)

<|endoftext|>