<commit_before>"""
The is the basic illustration of training.
"""
from __future__ import division, print_function, absolute_import
import os
import sys
import math
import time
import cPickle
from datetime import datetime

import pymongo as pm
import gridfs
import numpy as np

import tensorflow as tf
#tf.logging.set_verbosity(tf.logging.ERROR)

from tensorflow.contrib.learn.python.learn.datasets.mnist import read_data_sets

from tfutils import base, model, utils

class MNIST(object):
    def __init__(self, batch_size=100, group='train'):
        """
        A specific reader for MNIST stored as a HDF5 file

        Args:
            - data_path: path to imagenet data
            - crop_size: for center crop (crop_size x crop_size)
            - *args: extra arguments for HDF5DataProvider
        Kwargs:
            - **kwargs: extra keyword arguments for HDF5DataProvider
        """
        self.batch_size = batch_size
        data = read_data_sets('/tmp')
        if group == 'train':
            self.data = data.train
        elif group == 'test':
            self.data = data.train
        elif group == 'validation':
            self.data = data.train
        else:
            raise ValueError('MNIST data input "{}" not known'.format(group))

    def __iter__(self):
        return self

    def next(self):
        batch = self.data.next_batch(self.batch_size)
        return {'images': batch[0],
                'labels': batch[1].astype(np.int32)}

num_batches_per_epoch = 10000//256

testhost = 'localhost'           #Host on which the MongoDB instance to be used by tests needs to be running
testport = 31001                 #port on which the MongoDB instance to be used by tests needs to be running
testdbname = 'tfutils-test'      #name of the mongodb database where results will be stored by tests
testcol = 'testcol'              #name of the mongodb collection where results will be stored by tests

def test_training():
    """This test illustrates how basic training is performed using the tfutils.base.train_from_params function.
       This is the first in a sequence of interconnected tests. It creates a pretrained model that is used by
       the next few tests (test_validation and test_feature_extraction).

       As can be seen by looking at how the test checks for correctness, after the training is run, results
       of training, including (intermittently) the full variables needed to re-initialize the tensorflow model,
       are stored in a MongoDB.

       Also see docstring of the tfutils.base.train_from_params function for more detailed information about usage.
    """
    #delete old database if it exists
    conn = pm.MongoClient(host=testhost,
                          port=testport)
    conn.drop_database(testdbname)
    nm = testdbname + '_' + testcol + '_training0'
    [conn.drop_database(x) for x in conn.database_names() if x.startswith(nm) and '___RECENT' in x]
    nm = testdbname + '_' + testcol + '_training1'
    [conn.drop_database(x) for x in conn.database_names() if x.startswith(nm) and '___RECENT' in x]

    #set up the parameters
    params = {}
    params['model_params'] = {'func': model.mnist_tfutils}
    params['save_params'] = {'host': testhost,
                             'port': testport,
                             'dbname': testdbname,
                             'collname': testcol,
                             'exp_id': 'training0',
                             'save_valid_freq': 20,
                             'save_filters_freq': 200,
                             'cache_filters_freq': 100}
    params['train_params'] = {'data_params': {'func': MNIST,
                                              'batch_size': 100,
                                              'group': 'train'},
                              'queue_params': {'queue_type': 'fifo',
                                               'batch_size': 100,
                                               'n_threads': 4},
                              'num_steps': 500}
    params['learning_rate_params'] = {'learning_rate': 0.05,
                                      'decay_steps': num_batches_per_epoch,
                                      'decay_rate': 0.95,
                                      'staircase': True}
    params['validation_params'] = {'valid0': {'data_params': {'func': MNIST,
                                                              'batch_size': 100,
                                                              'group': 'test'},
                                              'queue_params': {'queue_type': 'fifo',
                                                               'batch_size': 100,
                                                               'n_threads': 4},
                                              'num_steps': 10,
                                              'agg_func': utils.mean_dict}}

    #actually run the training
    base.train_from_params(**params)
    #test if results are as expected
    assert conn[testdbname][testcol+'.files'].find({'exp_id': 'training0'}).count() == 26
    assert conn[testdbname][testcol+'.files'].find({'exp_id': 'training0',
            'saved_filters': True}).distinct('step') == [0, 200, 400]

    r = conn[testdbname][testcol+'.files'].find({'exp_id': 'training0', 'step': 0})[0]
    asserts_for_record(r, params, train=True)
    r = conn[testdbname][testcol+'.files'].find({'exp_id': 'training0', 'step': 20})[0]
    asserts_for_record(r, params, train=True)

    #run another 500 steps of training on the same experiment id.
    params['train_params']['num_steps'] = 1000
    base.train_from_params(**params)
    #test if results are as expected
    assert conn[testdbname][testcol+'.files'].find({'exp_id': 'training0'}).count() == 51
    assert conn[testdbname][testcol+'.files'].find({'exp_id': 'training0',
         'saved_filters': True}).distinct('step') == [0, 200, 400, 600, 800, 1000]
    assert conn['tfutils-test']['testcol.files'].distinct('exp_id') == ['training0']
    r = conn[testdbname][testcol+'.files'].find({'exp_id': 'training0', 'step': 1000})[0]
    asserts_for_record(r, params, train=True)

    #run 500 more steps but save to a new experiment id.
    params['train_params']['num_steps'] = 1500
    params['load_params'] = {'exp_id': 'training0'}
    params['save_params']['exp_id'] = 'training1'
    base.train_from_params(**params)
    assert conn[testdbname][testcol+'.files'].find({'exp_id': 'training1',
                            'saved_filters': True}).distinct('step') == [1200, 1400]

def test_validation():
    """
    This is a test illustrating how to compute performance on a trained model on a new dataset,
    using the tfutils.base.test_from_params function.  This test assumes that test_train function
    has run first (to provide a pre-trained model to validate).

    After the test is run, results from the validation are stored in the MongoDB. (The test shows the record
    can be loaded for inspection.)

    See the docstring of tfutils.base.test_from_params for more detailed information on usage.

    """
    #specify the parameters for the validation
    params = {}
    params['model_params'] = {'func': model.mnist_tfutils}
    params['load_params'] = {'host': testhost,
                             'port': testport,
                             'dbname': testdbname,
                             'collname': testcol,
                             'exp_id': 'training0'}
    params['save_params'] = {'exp_id': 'validation0'}
    params['validation_params'] = {'valid0': {'data_params': {'func': MNIST,
                                                              'batch_size': 100,
                                                              'group': 'test'},
                                              'queue_params': {'queue_type': 'fifo',
                                                               'batch_size': 100,
                                                               'n_threads': 4},
                                              'num_steps': 10,
                                              'agg_func': utils.mean_dict}}

    #actually run the model
    base.test_from_params(**params)

    #check that the results are correct
    conn = pm.MongoClient(host=testhost,
                          port=testport)

    #... specifically, there is now a record containing the validation0 performance results
    assert conn[testdbname][testcol+'.files'].find({'exp_id': 'validation0'}).count() == 1
    #... here's how to load the record:
    r = conn[testdbname][testcol+'.files'].find({'exp_id': 'validation0'})[0]
    asserts_for_record(r, params, train=False)

    #... check that the recorrectly ties to the id information for the
    #pre-trained model it was supposed to validate
    assert r['validates']
    f = r['validation_results']['valid0']['loss']
    idval = conn[testdbname][testcol+'.files'].find({'exp_id': 'training0'})[50]['_id']
    v = conn[testdbname][testcol+'.files'].find({'exp_id': 'validation0'})[0]['validates']
    assert idval == v

def get_extraction_target(inputs, outputs, to_extract, **loss_params):
    """
    Example validation target function to use to provide targets for extracting features.  This function also adds a
    standard "loss" target which you may or not may not want

    The to_extract argument must be a dictionary of the form
          {name_for_saving: name_of_actual_tensor, ...}
    where the "name_for_saving" is a human-friendly name you want to save extracted features under, and
    name_of_actual_tensor is a name of the tensor in the tensorflow graph outputing the features desired
    to be extracted.  To figure out what the names of the tensors you want to extract are "to_extract" argument,
    uncomment the commented-out lines, which will print a list of all available tensor names.
    """

    #names = [[x.name for x in op.values()] for op in tf.get_default_graph().get_operations()]
    #print("NAMES are: ", names)

    targets = {k: tf.get_default_graph().get_tensor_by_name(v) for k, v in to_extract.items()}
    targets['loss'] = utils.get_loss(inputs, outputs, **loss_params)
    return targets

def test_feature_extraction():
    """
    This is a test illustrating how to perform feature extraction using tfutils.base.test_from_params.
    The basic idea is to specify a validation target that is simply the actual output of the model at some layer.
    (See the "get_extraction_target" function above as well.)  This test assumes that test_train has run first.

    After the test is run, the results of the feature extraction are saved in the Grid File System associated
    with the mongodb, with one file per batch of feature results.  See how the features are accessed by reading the
    test code below.
    """
    #set up parameters
    params = {}
    params['model_params'] = {'func': model.mnist_tfutils}
    params['load_params'] = {'host': testhost,
                             'port': testport,
                             'dbname': testdbname,
                             'collname': testcol,
                             'exp_id': 'training0'}
    params['save_params'] = {'exp_id': 'validation1',
                             'save_intermediate_freq': 1,
                             'save_to_gfs': ['features']}

    targdict = {'func': get_extraction_target,
                'to_extract': {'features': 'validation/valid1/hidden1/fc:0'}}
    targdict.update(base.DEFAULT_LOSS_PARAMS)
    params['validation_params'] = {'valid1': {'data_params': {'func': MNIST,
                                                              'batch_size': 100,
                                                              'group': 'test'},
                                              'queue_params': {'queue_type': 'fifo',
                                                               'batch_size': 100,
                                                               'n_threads': 4},
                                              'targets': targdict,
                                              'num_steps': 10,
                                              'online_agg_func': utils.reduce_mean_dict}}

    #actually run the feature extraction
    base.test_from_params(**params)

    #check that things are as expected.
    conn = pm.MongoClient(host=testhost,
                          port=testport)
    coll = conn[testdbname][testcol+'.files']
    assert coll.find({'exp_id': 'validation1'}).count() == 11

    #... load the containing the final "aggregate" result after all features have been extracted
    q = {'exp_id': 'validation1', 'validation_results.valid1.intermediate_steps': {'$exists': True}}
    assert coll.find(q).count() == 1
    r = coll.find(q)[0]
    #... check that the record is well-formed
    asserts_for_record(r, params, train=False)

    #... check that the correct "intermediate results" (the actual features extracted) records exist
    #and are correctly referenced.
    q1 = {'exp_id': 'validation1', 'validation_results.valid1.intermediate_steps': {'$exists': False}}
    ids = coll.find(q1).distinct('_id')
    assert r['validation_results']['valid1']['intermediate_steps'] == ids

    #... actually load feature batch 3
    idval = r['validation_results']['valid1']['intermediate_steps'][3]
    fn = coll.find({'item_for': idval})[0]['filename']
    fs = gridfs.GridFS(coll.database, testcol)
    fh = fs.get_last_version(fn)
    saved_data  = cPickle.loads(fh.read())
    fh.close()
    features = saved_data['validation_results']['valid1']['features']
    assert features.shape == (100, 128)
    assert features.dtype == np.float32

def asserts_for_record(r, params, train=False):
    if r.get('saved_filters'):
        assert r['_saver_write_version'] == 2
        assert r['_saver_num_data_files'] == 1
    assert type(r['duration']) == float

    should_contain = ['save_params', 'load_params', 'model_params', 'validation_params']
    assert set(should_contain).difference(r['params'].keys()) == set()

    vk = r['params']['validation_params'].keys()
    vk1 = r['validation_results'].keys()
    assert set(vk) == set(vk1)

    assert r['params']['model_params']['cfg_initial'] == None
    assert r['params']['model_params']['seed'] == 0
    assert r['params']['model_params']['func']['modname'] == 'tfutils.model'
    assert r['params']['model_params']['func']['objname'] == 'mnist_tfutils'
    assert set(['hidden1', 'hidden2', u'softmax_linear']).difference(r['params']['model_params']['cfg_final'].keys()) == set()

    _k = vk[0]
    should_contain = ['agg_func', 'data_params', 'num_steps', 'online_agg_func', 'queue_params', 'targets']
    assert set(should_contain).difference(r['params']['validation_params'][_k].keys()) == set()

    if train:
        assert r['params']['model_params']['train'] == True
        for k in ['num_steps', 'queue_params']:
            assert r['params']['train_params'][k] == params['train_params'][k]

        should_contain = ['loss_params', 'optimizer_params', 'train_params', 'learning_rate_params']
        assert set(should_contain).difference(r['params'].keys()) == set()
        assert r['params']['train_params']['thres_loss'] == 100
        assert r['params']['train_params']['data_params']['func']['modname'] == 'tfutils.tests.test'
        assert r['params']['train_params']['data_params']['func']['objname'] == 'MNIST'

        assert r['params']['loss_params']['agg_func']['modname'] == 'tensorflow.python.ops.math_ops'
        assert r['params']['loss_params']['agg_func']['objname'] == 'reduce_mean'
        assert r['params']['loss_params']['loss_per_case_func']['modname'] == 'tensorflow.python.ops.nn_ops'
        assert r['params']['loss_params']['loss_per_case_func']['objname'] == 'sparse_softmax_cross_entropy_with_logits'
        assert r['params']['loss_params']['targets'] == ['labels']
    else:
        assert not 'train' in r['params']['model_params']
        assert 'train_params' not in r['params']
<commit_msg>minor<commit_after>"""
The is the basic illustration of training.
"""
from __future__ import division, print_function, absolute_import
import os
import sys
import math
import time
import cPickle
from datetime import datetime

import pymongo as pm
import gridfs
import numpy as np

import tensorflow as tf
#tf.logging.set_verbosity(tf.logging.ERROR)

from tensorflow.contrib.learn.python.learn.datasets.mnist import read_data_sets

from tfutils import base, model, utils

class MNIST(object):
    def __init__(self, batch_size=100, group='train'):
        """
        A specific reader for MNIST stored as a HDF5 file

        Args:
            - data_path: path to imagenet data
            - crop_size: for center crop (crop_size x crop_size)
            - *args: extra arguments for HDF5DataProvider
        Kwargs:
            - **kwargs: extra keyword arguments for HDF5DataProvider
        """
        self.batch_size = batch_size
        data = read_data_sets('/tmp')
        if group == 'train':
            self.data = data.train
        elif group == 'test':
            self.data = data.train
        elif group == 'validation':
            self.data = data.train
        else:
            raise ValueError('MNIST data input "{}" not known'.format(group))

    def __iter__(self):
        return self

    def next(self):
        batch = self.data.next_batch(self.batch_size)
        return {'images': batch[0],
                'labels': batch[1].astype(np.int32)}

num_batches_per_epoch = 10000//256

testhost = 'localhost'           #Host on which the MongoDB instance to be used by tests needs to be running
testport = 31001                 #port on which the MongoDB instance to be used by tests needs to be running
testdbname = 'tfutils-test'      #name of the mongodb database where results will be stored by tests
testcol = 'testcol'              #name of the mongodb collection where results will be stored by tests

def test_training():
    """This test illustrates how basic training is performed using the tfutils.base.train_from_params function.
       This is the first in a sequence of interconnected tests. It creates a pretrained model that is used by
       the next few tests (test_validation and test_feature_extraction).

       As can be seen by looking at how the test checks for correctness, after the training is run, results
       of training, including (intermittently) the full variables needed to re-initialize the tensorflow model,
       are stored in a MongoDB.

       Also see docstring of the tfutils.base.train_from_params function for more detailed information about usage.
    """
    #delete old database if it exists
    conn = pm.MongoClient(host=testhost,
                          port=testport)
    conn.drop_database(testdbname)
    nm = testdbname + '_' + testcol + '_training0'
    [conn.drop_database(x) for x in conn.database_names() if x.startswith(nm) and '___RECENT' in x]
    nm = testdbname + '_' + testcol + '_training1'
    [conn.drop_database(x) for x in conn.database_names() if x.startswith(nm) and '___RECENT' in x]

    #set up the parameters
    params = {}
    params['model_params'] = {'func': model.mnist_tfutils}
    params['save_params'] = {'host': testhost,
                             'port': testport,
                             'dbname': testdbname,
                             'collname': testcol,
                             'exp_id': 'training0',
                             'save_valid_freq': 20,
                             'save_filters_freq': 200,
                             'cache_filters_freq': 100}
    params['train_params'] = {'data_params': {'func': MNIST,
                                              'batch_size': 100,
                                              'group': 'train'},
                              'queue_params': {'queue_type': 'fifo',
                                               'batch_size': 100,
                                               'n_threads': 4},
                              'num_steps': 500}
    params['learning_rate_params'] = {'learning_rate': 0.05,
                                      'decay_steps': num_batches_per_epoch,
                                      'decay_rate': 0.95,
                                      'staircase': True}
    params['validation_params'] = {'valid0': {'data_params': {'func': MNIST,
                                                              'batch_size': 100,
                                                              'group': 'test'},
                                              'queue_params': {'queue_type': 'fifo',
                                                               'batch_size': 100,
                                                               'n_threads': 4},
                                              'num_steps': 10,
                                              'agg_func': utils.mean_dict}}

    #actually run the training
    base.train_from_params(**params)
    #test if results are as expected
    assert conn[testdbname][testcol+'.files'].find({'exp_id': 'training0'}).count() == 26
    assert conn[testdbname][testcol+'.files'].find({'exp_id': 'training0',
            'saved_filters': True}).distinct('step') == [0, 200, 400]

    r = conn[testdbname][testcol+'.files'].find({'exp_id': 'training0', 'step': 0})[0]
    asserts_for_record(r, params, train=True)
    r = conn[testdbname][testcol+'.files'].find({'exp_id': 'training0', 'step': 20})[0]
    asserts_for_record(r, params, train=True)

    #run another 500 steps of training on the same experiment id.
    params['train_params']['num_steps'] = 1000
    base.train_from_params(**params)
    #test if results are as expected
    assert conn[testdbname][testcol+'.files'].find({'exp_id': 'training0'}).count() == 51
    assert conn[testdbname][testcol+'.files'].find({'exp_id': 'training0',
         'saved_filters': True}).distinct('step') == [0, 200, 400, 600, 800, 1000]
    assert conn['tfutils-test']['testcol.files'].distinct('exp_id') == ['training0']
    r = conn[testdbname][testcol+'.files'].find({'exp_id': 'training0', 'step': 1000})[0]
    asserts_for_record(r, params, train=True)

    #run 500 more steps but save to a new experiment id.
    params['train_params']['num_steps'] = 1500
    params['load_params'] = {'exp_id': 'training0'}
    params['save_params']['exp_id'] = 'training1'
    base.train_from_params(**params)
    assert conn[testdbname][testcol+'.files'].find({'exp_id': 'training1',
                            'saved_filters': True}).distinct('step') == [1200, 1400]

def test_validation():
    """
    This is a test illustrating how to compute performance on a trained model on a new dataset,
    using the tfutils.base.test_from_params function.  This test assumes that test_train function
    has run first (to provide a pre-trained model to validate).

    After the test is run, results from the validation are stored in the MongoDB. (The test shows the record
    can be loaded for inspection.)

    See the docstring of tfutils.base.test_from_params for more detailed information on usage.
    """
    #specify the parameters for the validation
    params = {}
    params['model_params'] = {'func': model.mnist_tfutils}
    params['load_params'] = {'host': testhost,
                             'port': testport,
                             'dbname': testdbname,
                             'collname': testcol,
                             'exp_id': 'training0'}
    params['save_params'] = {'exp_id': 'validation0'}
    params['validation_params'] = {'valid0': {'data_params': {'func': MNIST,
                                                              'batch_size': 100,
                                                              'group': 'test'},
                                              'queue_params': {'queue_type': 'fifo',
                                                               'batch_size': 100,
                                                               'n_threads': 4},
                                              'num_steps': 10,
                                              'agg_func': utils.mean_dict}}

    #actually run the model
    base.test_from_params(**params)

    #check that the results are correct
    conn = pm.MongoClient(host=testhost,
                          port=testport)

    #... specifically, there is now a record containing the validation0 performance results
    assert conn[testdbname][testcol+'.files'].find({'exp_id': 'validation0'}).count() == 1
    #... here's how to load the record:
    r = conn[testdbname][testcol+'.files'].find({'exp_id': 'validation0'})[0]
    asserts_for_record(r, params, train=False)

    #... check that the recorrectly ties to the id information for the
    #pre-trained model it was supposed to validate
    assert r['validates']
    f = r['validation_results']['valid0']['loss']
    idval = conn[testdbname][testcol+'.files'].find({'exp_id': 'training0'})[50]['_id']
    v = conn[testdbname][testcol+'.files'].find({'exp_id': 'validation0'})[0]['validates']
    assert idval == v

def get_extraction_target(inputs, outputs, to_extract, **loss_params):
    """
    Example validation target function to use to provide targets for extracting features.  This function also adds a
    standard "loss" target which you may or not may not want

    The to_extract argument must be a dictionary of the form
          {name_for_saving: name_of_actual_tensor, ...}
    where the "name_for_saving" is a human-friendly name you want to save extracted features under, and
    name_of_actual_tensor is a name of the tensor in the tensorflow graph outputing the features desired
    to be extracted.  To figure out what the names of the tensors you want to extract are "to_extract" argument,
    uncomment the commented-out lines, which will print a list of all available tensor names.
    """

    #names = [[x.name for x in op.values()] for op in tf.get_default_graph().get_operations()]
    #print("NAMES are: ", names)

    targets = {k: tf.get_default_graph().get_tensor_by_name(v) for k, v in to_extract.items()}
    targets['loss'] = utils.get_loss(inputs, outputs, **loss_params)
    return targets

def test_feature_extraction():
    """
    This is a test illustrating how to perform feature extraction using tfutils.base.test_from_params.
    The basic idea is to specify a validation target that is simply the actual output of the model at some layer.
    (See the "get_extraction_target" function above as well.)  This test assumes that test_train has run first.

    After the test is run, the results of the feature extraction are saved in the Grid File System associated
    with the mongodb, with one file per batch of feature results.  See how the features are accessed by reading the
    test code below.
    """
    #set up parameters
    params = {}
    params['model_params'] = {'func': model.mnist_tfutils}
    params['load_params'] = {'host': testhost,
                             'port': testport,
                             'dbname': testdbname,
                             'collname': testcol,
                             'exp_id': 'training0'}
    params['save_params'] = {'exp_id': 'validation1',
                             'save_intermediate_freq': 1,
                             'save_to_gfs': ['features']}

    targdict = {'func': get_extraction_target,
                'to_extract': {'features': 'validation/valid1/hidden1/fc:0'}}
    targdict.update(base.DEFAULT_LOSS_PARAMS)
    params['validation_params'] = {'valid1': {'data_params': {'func': MNIST,
                                                              'batch_size': 100,
                                                              'group': 'test'},
                                              'queue_params': {'queue_type': 'fifo',
                                                               'batch_size': 100,
                                                               'n_threads': 4},
                                              'targets': targdict,
                                              'num_steps': 10,
                                              'online_agg_func': utils.reduce_mean_dict}}

    #actually run the feature extraction
    base.test_from_params(**params)

    #check that things are as expected.
    conn = pm.MongoClient(host=testhost,
                          port=testport)
    coll = conn[testdbname][testcol+'.files']
    assert coll.find({'exp_id': 'validation1'}).count() == 11

    #... load the containing the final "aggregate" result after all features have been extracted
    q = {'exp_id': 'validation1', 'validation_results.valid1.intermediate_steps': {'$exists': True}}
    assert coll.find(q).count() == 1
    r = coll.find(q)[0]
    #... check that the record is well-formed
    asserts_for_record(r, params, train=False)

    #... check that the correct "intermediate results" (the actual features extracted) records exist
    #and are correctly referenced.
    q1 = {'exp_id': 'validation1', 'validation_results.valid1.intermediate_steps': {'$exists': False}}
    ids = coll.find(q1).distinct('_id')
    assert r['validation_results']['valid1']['intermediate_steps'] == ids

    #... actually load feature batch 3
    idval = r['validation_results']['valid1']['intermediate_steps'][3]
    fn = coll.find({'item_for': idval})[0]['filename']
    fs = gridfs.GridFS(coll.database, testcol)
    fh = fs.get_last_version(fn)
    saved_data  = cPickle.loads(fh.read())
    fh.close()
    features = saved_data['validation_results']['valid1']['features']
    assert features.shape == (100, 128)
    assert features.dtype == np.float32

def asserts_for_record(r, params, train=False):
    if r.get('saved_filters'):
        assert r['_saver_write_version'] == 2
        assert r['_saver_num_data_files'] == 1
    assert type(r['duration']) == float

    should_contain = ['save_params', 'load_params', 'model_params', 'validation_params']
    assert set(should_contain).difference(r['params'].keys()) == set()

    vk = r['params']['validation_params'].keys()
    vk1 = r['validation_results'].keys()
    assert set(vk) == set(vk1)

    assert r['params']['model_params']['cfg_initial'] == None
    assert r['params']['model_params']['seed'] == 0
    assert r['params']['model_params']['func']['modname'] == 'tfutils.model'
    assert r['params']['model_params']['func']['objname'] == 'mnist_tfutils'
    assert set(['hidden1', 'hidden2', u'softmax_linear']).difference(r['params']['model_params']['cfg_final'].keys()) == set()

    _k = vk[0]
    should_contain = ['agg_func', 'data_params', 'num_steps', 'online_agg_func', 'queue_params', 'targets']
    assert set(should_contain).difference(r['params']['validation_params'][_k].keys()) == set()

    if train:
        assert r['params']['model_params']['train'] == True
        for k in ['num_steps', 'queue_params']:
            assert r['params']['train_params'][k] == params['train_params'][k]

        should_contain = ['loss_params', 'optimizer_params', 'train_params', 'learning_rate_params']
        assert set(should_contain).difference(r['params'].keys()) == set()
        assert r['params']['train_params']['thres_loss'] == 100
        assert r['params']['train_params']['data_params']['func']['modname'] == 'tfutils.tests.test'
        assert r['params']['train_params']['data_params']['func']['objname'] == 'MNIST'

        assert r['params']['loss_params']['agg_func']['modname'] == 'tensorflow.python.ops.math_ops'
        assert r['params']['loss_params']['agg_func']['objname'] == 'reduce_mean'
        assert r['params']['loss_params']['loss_per_case_func']['modname'] == 'tensorflow.python.ops.nn_ops'
        assert r['params']['loss_params']['loss_per_case_func']['objname'] == 'sparse_softmax_cross_entropy_with_logits'
        assert r['params']['loss_params']['targets'] == ['labels']
    else:
        assert not 'train' in r['params']['model_params']
        assert 'train_params' not in r['params']
<|endoftext|>


<commit_before>from time import time
import os
from tempfile import gettempdir
from uuid import uuid4
from ..conf import settings
from ..const import ARGUMENT_PLACEHOLDER, USER_COMMAND_MARK
from ..utils import memoize
from .generic import Generic

class Zsh(Generic):
    def app_alias(self, alias_name):
        # It is VERY important to have the variables declared WITHIN the function
        return '''
            {name} () {{
                TF_PYTHONIOENCODING=$PYTHONIOENCODING;
                export TF_SHELL=zsh;
                export TF_ALIAS={name};
                export TF_SHELL_ALIASES=$(alias);
                export TF_HISTORY="$(fc -ln -10)";
                export PYTHONIOENCODING=utf-8;
                TF_CMD=$(
                    thefuck {argument_placeholder} $@
                ) && eval $TF_CMD;
                unset TF_HISTORY;
                export PYTHONIOENCODING=$TF_PYTHONIOENCODING;
                {alter_history}
            }}
        '''.format(
            name=alias_name,
            argument_placeholder=ARGUMENT_PLACEHOLDER,
            alter_history=('test -n "$TF_CMD" && print -s $TF_CMD'
                           if settings.alter_history else ''))

    def instant_mode_alias(self, alias_name):
        if os.environ.get('THEFUCK_INSTANT_MODE', '').lower() == 'true':
            mark = ('%{' +
                    USER_COMMAND_MARK + '\b' * len(USER_COMMAND_MARK)
                    + '%}')
            return '''
                export PS1="{user_command_mark}$PS1";
                {app_alias}
            '''.format(user_command_mark=mark,
                       app_alias=self.app_alias(alias_name))
        else:
            log_path = os.path.join(
                gettempdir(), 'thefuck-script-log-{}'.format(uuid4().hex))
            return '''
                export THEFUCK_INSTANT_MODE=True;
                export THEFUCK_OUTPUT_LOG={log};
                thefuck --shell-logger {log};
                rm {log};
                exit
            '''.format(log=log_path)

    def _parse_alias(self, alias):
        name, value = alias.split('=', 1)
        if value[0] == value[-1] == '"' or value[0] == value[-1] == "'":
            value = value[1:-1]
        return name, value

    @memoize
    def get_aliases(self):
        raw_aliases = os.environ.get('TF_SHELL_ALIASES', '').split('\n')
        return dict(self._parse_alias(alias)
                    for alias in raw_aliases if alias and '=' in alias)

    def _get_history_file_name(self):
        return os.environ.get("HISTFILE",
                              os.path.expanduser('~/.zsh_history'))

    def _get_history_line(self, command_script):
        return u': {}:0;{}\n'.format(int(time()), command_script)

    def _script_from_history(self, line):
        if ';' in line:
            return line.split(';', 1)[1]
        else:
            return ''

    def how_to_configure(self):
        return self._create_shell_configuration(
            content=u'eval $(thefuck --alias)',
            path='~/.zshrc',
            reload='source ~/.zshrc')
<commit_msg>Avoid masking shell return values in Zsh.app_alias (#820)<commit_after>from time import time
import os
from tempfile import gettempdir
from uuid import uuid4
from ..conf import settings
from ..const import ARGUMENT_PLACEHOLDER, USER_COMMAND_MARK
from ..utils import memoize
from .generic import Generic

class Zsh(Generic):
    def app_alias(self, alias_name):
        # It is VERY important to have the variables declared WITHIN the function
        return '''
            {name} () {{
                TF_PYTHONIOENCODING=$PYTHONIOENCODING;
                export TF_SHELL=zsh;
                export TF_ALIAS={name};
                TF_SHELL_ALIASES=$(alias);
                export TF_SHELL_ALIASES;
                TF_HISTORY="$(fc -ln -10)";
                export TF_HISTORY;
                export PYTHONIOENCODING=utf-8;
                TF_CMD=$(
                    thefuck {argument_placeholder} $@
                ) && eval $TF_CMD;
                unset TF_HISTORY;
                export PYTHONIOENCODING=$TF_PYTHONIOENCODING;
                {alter_history}
            }}
        '''.format(
            name=alias_name,
            argument_placeholder=ARGUMENT_PLACEHOLDER,
            alter_history=('test -n "$TF_CMD" && print -s $TF_CMD'
                           if settings.alter_history else ''))

    def instant_mode_alias(self, alias_name):
        if os.environ.get('THEFUCK_INSTANT_MODE', '').lower() == 'true':
            mark = ('%{' +
                    USER_COMMAND_MARK + '\b' * len(USER_COMMAND_MARK)
                    + '%}')
            return '''
                export PS1="{user_command_mark}$PS1";
                {app_alias}
            '''.format(user_command_mark=mark,
                       app_alias=self.app_alias(alias_name))
        else:
            log_path = os.path.join(
                gettempdir(), 'thefuck-script-log-{}'.format(uuid4().hex))
            return '''
                export THEFUCK_INSTANT_MODE=True;
                export THEFUCK_OUTPUT_LOG={log};
                thefuck --shell-logger {log};
                rm {log};
                exit
            '''.format(log=log_path)

    def _parse_alias(self, alias):
        name, value = alias.split('=', 1)
        if value[0] == value[-1] == '"' or value[0] == value[-1] == "'":
            value = value[1:-1]
        return name, value

    @memoize
    def get_aliases(self):
        raw_aliases = os.environ.get('TF_SHELL_ALIASES', '').split('\n')
        return dict(self._parse_alias(alias)
                    for alias in raw_aliases if alias and '=' in alias)

    def _get_history_file_name(self):
        return os.environ.get("HISTFILE",
                              os.path.expanduser('~/.zsh_history'))

    def _get_history_line(self, command_script):
        return u': {}:0;{}\n'.format(int(time()), command_script)

    def _script_from_history(self, line):
        if ';' in line:
            return line.split(';', 1)[1]
        else:
            return ''

    def how_to_configure(self):
        return self._create_shell_configuration(
            content=u'eval $(thefuck --alias)',
            path='~/.zshrc',
            reload='source ~/.zshrc')
<|endoftext|>


<commit_before>import os

from thinglang import utils
from thinglang.compiler.indexer import Indexer
from thinglang.execution.execution import ExecutionEngine
from thinglang.lexer.lexer import lexer
from thinglang.parser.analyzer import Analyzer
from thinglang.parser.parser import parse
from thinglang.parser.simplifier import Simplifier

BASE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'include')

def collect_includes():
    files = [os.path.join(BASE_DIR, path) for path in os.listdir(BASE_DIR)]
    return '\n' + '\n'.join(open(f).read() for f in files)

def compiler(source):
    if not source:
        raise ValueError('Source cannot be empty')

    source = source.strip().replace(' ' * 4, '\t')
    lexical_groups = list(lexer(source))
    ast = parse(lexical_groups)
    Simplifier(ast).run()
    Indexer(ast).run()
    Analyzer(ast).run()

    return ast

def run(source):
    ast = compiler(source)

    with ExecutionEngine(ast) as engine:
        engine.execute()
        return engine.results()
<commit_msg>Added collator and disable analyzer during compilation<commit_after>import os

from thinglang import utils
from thinglang.compiler.indexer import Indexer, Collator
from thinglang.execution.execution import ExecutionEngine
from thinglang.lexer.lexer import lexer
from thinglang.parser.analyzer import Analyzer
from thinglang.parser.parser import parse
from thinglang.parser.simplifier import Simplifier

BASE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'include')

def collect_includes():
    files = [os.path.join(BASE_DIR, path) for path in os.listdir(BASE_DIR)]
    return '\n' + '\n'.join(open(f).read() for f in files)

def compiler(source):
    if not source:
        raise ValueError('Source cannot be empty')

    source = source.strip().replace(' ' * 4, '\t')
    lexical_groups = list(lexer(source))
    ast = parse(lexical_groups)
    Simplifier(ast).run()
    Collator(ast).run()
    Indexer(ast).run()
    #Analyzer(ast).run()

    return ast

def run(source):
    ast = compiler(source)

    with ExecutionEngine(ast) as engine:
        engine.execute()
        return engine.results()
<|endoftext|>


<commit_before># Copyright 2019 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

bazel_version="6a16ab1a7478924630d1e9aeb7022c131254152f"
<commit_msg>Update Bazel to latest (74c274a)<commit_after># Copyright 2019 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

bazel_version = "74c274a4ed8f97ab32724641bb011cee2e057a76"
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Function:   Create a plugin project based on DemoCmds project.

Usage:      python newproj.py prjname
            or double click the file 'newproj.py'.

            prjname: name of the new project, the default value is 'EduCmds'.

Creator:    Zhang Yungui <rhcad@hotmail.com>
Date:       2014.5.4
"""

import os, sys, shutil, re, uuid

def multi_replace(text, adict):
    rx = re.compile('|'.join(map(re.escape, adict)))
    def xlat(match):
        return adict[match.group(0)]
    return rx.sub(xlat, text)

def copyfiles(srcdir, dstdir, pairs, callback):
    if srcdir.endswith(".git") or not os.path.exists(srcdir):
        return

    for fn in os.listdir(srcdir):
        srcfile = os.path.join(srcdir, fn)
        dstfile = os.path.join(dstdir, multi_replace(fn, pairs))

        if os.path.isdir(srcfile):
            copyfiles(srcfile, dstfile, pairs, callback)
            continue

        if not os.path.exists(dstfile) and callback(fn, pairs):
            if not os.path.exists(dstdir):
                os.makedirs(dstdir)
            isutf8 = False

            try:
                text = open(srcfile).read()
            except UnicodeDecodeError:
                try:
                    text = open(srcfile,'r',-1,'utf-8').read()
                    isutf8 = True
                except UnicodeDecodeError:
                    print("* Fail to read '%s' as utf-8 encoding." % (dstfile,))
                    continue

            newtext = multi_replace(text, pairs)
            if newtext != text:
                try:
                    if (isutf8):
                        open(dstfile, 'w',-1,'utf-8').write(newtext)
                        print('[replaced] %s [utf-8]' % (dstfile,))
                    else:
                        open(dstfile, 'w').write(newtext)
                        print('[replaced] %s' % (dstfile,))
                except UnicodeDecodeError:
                    open(dstfile, 'w',-1,'utf-8').write(newtext)
                    print('[replaced] %s [utf-8]' % (dstfile,))
            else:
                shutil.copy(srcfile, dstfile)
                print('[created] %s' % (dstfile,))

def makeproj(prjname):
    srcname = 'DemoCmds'
    srcdir  = os.path.abspath(srcname)
    dstdir  = os.path.abspath(prjname)

    if prjname == '' or prjname == srcname:
        print("Need input the project name.")
        return
    if not os.path.exists(srcdir):
        print("\nPlease input a valid exists template project name."
              "\n\'%s\' does not exist." % (srcdir,))
        return

    pairs = {srcname:prjname, srcname.lower():prjname.lower(), srcname.upper():prjname.upper() }

    def matchfile(filename, pairs):
        return True
    copyfiles(srcdir, dstdir, pairs, matchfile)

if __name__=="__main__":
    def inputparam(index, prompt, default=''):
        if len(sys.argv) > index: ret = sys.argv[index]
        else: ret = input(prompt)
        if ret == '': ret = default
        return ret

    prjname = inputparam(1, 'New project name: ', 'EduCmds')

    makeproj(prjname)
    if len(sys.argv) < 2: input("Press <ENTER> to end.")
<commit_msg>Fix newproj.py for Python 2.x.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Function:   Create a plugin project based on DemoCmds project.

Usage:      python newproj.py prjname
            or double click the file 'newproj.py'.

            prjname: name of the new project, the default value is 'EduCmds'.

Creator:    Zhang Yungui <rhcad@hotmail.com>
Date:       2014.5.4
"""

import os, sys, shutil, re, uuid

def multi_replace(text, adict):
    rx = re.compile('|'.join(map(re.escape, adict)))
    def xlat(match):
        return adict[match.group(0)]
    return rx.sub(xlat, text)

def copyfiles(srcdir, dstdir, pairs, callback):
    if srcdir.endswith(".git") or srcdir.endswith(".DS_Store") or not os.path.exists(srcdir):
        return

    for fn in os.listdir(srcdir):
        srcfile = os.path.join(srcdir, fn)
        dstfile = os.path.join(dstdir, multi_replace(fn, pairs))

        if os.path.isdir(srcfile):
            copyfiles(srcfile, dstfile, pairs, callback)
            continue

        if not os.path.exists(dstfile) and callback(fn, pairs):
            if not os.path.exists(dstdir):
                os.makedirs(dstdir)
            isutf8 = False

            try:
                text = open(srcfile).read()
            except UnicodeDecodeError:
                try:
                    text = open(srcfile,'r',-1,'utf-8').read()
                    isutf8 = True
                except UnicodeDecodeError:
                    print("* Fail to read '%s' as utf-8 encoding." % (dstfile,))
                    continue

            newtext = multi_replace(text, pairs)
            if newtext != text:
                try:
                    if (isutf8):
                        open(dstfile, 'w',-1,'utf-8').write(newtext)
                        print('[replaced] %s [utf-8]' % (dstfile,))
                    else:
                        open(dstfile, 'w').write(newtext)
                        print('[replaced] %s' % (dstfile,))
                except UnicodeDecodeError:
                    open(dstfile, 'w',-1,'utf-8').write(newtext)
                    print('[replaced] %s [utf-8]' % (dstfile,))
            else:
                shutil.copy(srcfile, dstfile)
                print('[created] %s' % (dstfile,))

def makeproj(prjname):
    srcname = 'DemoCmds'
    srcdir  = os.path.abspath(srcname)
    dstdir  = os.path.abspath(prjname)

    if prjname == '' or prjname == srcname:
        print("Need input the project name.")
        return
    if not os.path.exists(srcdir):
        print("\nPlease input a valid exists template project name."
              "\n\'%s\' does not exist." % (srcdir,))
        return

    pairs = {srcname:prjname, srcname.lower():prjname.lower(), srcname.upper():prjname.upper() }

    def matchfile(filename, pairs):
        return True
    copyfiles(srcdir, dstdir, pairs, matchfile)

if __name__=="__main__":
    def inputparam(index, prompt, default=''):
        if len(sys.argv) > index: ret = sys.argv[index]
        else: ret = raw_input(prompt)
        if ret == '': ret = default
        return ret

    prjname = inputparam(1, 'New project name: ', 'EduCmds')

    makeproj(prjname)
    if len(sys.argv) < 2: raw_input("Press <ENTER> to end.")
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# This file is part of thr library released under the MIT license.
# See the LICENSE file for more information.

from tornado import ioloop
from tornado import gen
from tornado.web import RequestHandler, Application, url
from tornado.options import define, options, parse_command_line
import tornadis
import datetime

from thr.http2redis.rules import Rules
from thr.http2redis.exchange import HTTPExchange
from thr.utils import make_unique_id, serialize_http_request, \
    unserialize_response_message
from thr import DEFAULT_REDIS_HOST, DEFAULT_REDIS_PORT, DEFAULT_REDIS_QUEUE
from thr import DEFAULT_TIMEOUT

define("timeout", type=int, help="Timeout in second for a request",
       default=DEFAULT_TIMEOUT)
define("config", help="Path to config file")
define("port", type=int, default=8888, help="Listening port")
define("redis_host", default=DEFAULT_REDIS_HOST,
       help="Default redis server hostname or ip")
define("redis_port", default=DEFAULT_REDIS_PORT, type=int,
       help="Default redis server port")
define("redis_queue", default=DEFAULT_REDIS_QUEUE,
       help="Default redis queue")

redis_pools = {}

def get_redis_pool(host, port):
    global redis_pools
    key = "%s:%i" % (host, port)
    if key not in redis_pools:
        redis_pools[key] = tornadis.ClientPool(host=host, port=port,
                                               connect_timeout=options.timeout)
    return redis_pools[key]

class Handler(RequestHandler):

    def get(self, *args, **kwargs):
        return self.handle(*args, **kwargs)

    def post(self, *args, **kwargs):
        return self.handle(*args, **kwargs)

    def delete(self, *args, **kwargs):
        return self.handle(*args, **kwargs)

    def put(self, *args, **kwargs):
        return self.handle(*args, **kwargs)

    def head(self, *args, **kwargs):
        return self.handle(*args, **kwargs)

    def options(self, *args, **kwargs):
        return self.handle(*args, **kwargs)

    def patch(self, *args, **kwargs):
        return self.handle(*args, **kwargs)

    def return_http_reply(self, exchange):
        if exchange.response.status_code is not None:
            self.set_status(exchange.response.status_code)
        for name in exchange.response.headers.keys():
            value = exchange.response.headers[name]
            self.set_header(name, value)
        if exchange.response.body is not None:
            self.finish(exchange.response.body)
        else:
            self.finish()

    def update_exchange_from_response_message(self, exchange, message):
        (status_code, body, body_link, headers, _) = \
            unserialize_response_message(message)
        exchange.response.status_code = status_code
        # FIXME: body_link ???
        exchange.response.body = body
        exchange.reponse.headers = headers

    @gen.coroutine
    def handle(self, *args, **kwargs):
        exchange = HTTPExchange(self.request,
                                default_redis_host=options.redis_host,
                                default_redis_port=options.redis_port,
                                default_redis_queue=options.redis_queue)
        yield Rules.execute_input_actions(exchange)
        if exchange.response.status_code is not None:
            # so we don't push the request on redis
            # let's call output actions for headers and body
            yield Rules.execute_output_actions(exchange)
            self.return_http_reply(exchange)
        elif exchange.redis_queue == "null":
            self.write("404 Not found")
            self.set_status(404)
        else:
            redis_pool = get_redis_pool(exchange.redis_host,
                                        exchange.redis_port)
            with (yield redis_pool.connected_client()) as redis:
                response_key = make_unique_id()
                serialized_request = serialize_http_request(
                    exchange.request,
                    dict_to_inject={
                        'response_key': response_key
                    })
                yield redis.call('LPUSH', exchange.redis_queue,
                                 serialized_request)
                before = datetime.datetime.now()
                while True:
                    result = yield redis.call('BRPOP', response_key, 1)
                    if result:
                        self.update_exchange_from_response_message(exchange,
                                                                   result[1])
                        yield Rules.execute_output_actions(exchange)
                        self.return_http_reply(exchange)
                        break
                    after = datetime.datetime.now()
                    delta = after - before
                    if delta.total_seconds() > options.timeout:
                        self.set_status(504)
                        self.finish("No reply from the backend")
                        break

def make_app():
    if options.config is not None:
        exec(open(options.config).read(), {})
    return Application([url(r"/.*", Handler)])

def main():
    parse_command_line()
    print("Start http2redis on http://localhost:{}".format(options.port))
    app = make_app()
    app.listen(options.port)
    ioloop.IOLoop.instance().start()
<commit_msg>refactor<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# This file is part of thr library released under the MIT license.
# See the LICENSE file for more information.

from tornado import ioloop
from tornado import gen
from tornado.web import RequestHandler, Application, url
from tornado.options import define, options, parse_command_line
import tornadis
import datetime

from thr.http2redis.rules import Rules
from thr.http2redis.exchange import HTTPExchange
from thr.utils import make_unique_id, serialize_http_request, \
    unserialize_response_message
from thr import DEFAULT_REDIS_HOST, DEFAULT_REDIS_PORT, DEFAULT_REDIS_QUEUE
from thr import DEFAULT_TIMEOUT

define("timeout", type=int, help="Timeout in second for a request",
       default=DEFAULT_TIMEOUT)
define("config", help="Path to config file")
define("port", type=int, default=8888, help="Listening port")
define("redis_host", default=DEFAULT_REDIS_HOST,
       help="Default redis server hostname or ip")
define("redis_port", default=DEFAULT_REDIS_PORT, type=int,
       help="Default redis server port")
define("redis_queue", default=DEFAULT_REDIS_QUEUE,
       help="Default redis queue")

redis_pools = {}

def get_redis_pool(host, port):
    global redis_pools
    key = "%s:%i" % (host, port)
    if key not in redis_pools:
        redis_pools[key] = tornadis.ClientPool(host=host, port=port,
                                               connect_timeout=options.timeout)
    return redis_pools[key]

class Handler(RequestHandler):

    def get(self, *args, **kwargs):
        return self.handle(*args, **kwargs)

    def post(self, *args, **kwargs):
        return self.handle(*args, **kwargs)

    def delete(self, *args, **kwargs):
        return self.handle(*args, **kwargs)

    def put(self, *args, **kwargs):
        return self.handle(*args, **kwargs)

    def head(self, *args, **kwargs):
        return self.handle(*args, **kwargs)

    def options(self, *args, **kwargs):
        return self.handle(*args, **kwargs)

    def patch(self, *args, **kwargs):
        return self.handle(*args, **kwargs)

    def return_http_reply(self, exchange, force_status=None, force_body=None):
        status = exchange.response.status_code
        body = exchange.response.body
        if force_status:
            status = force_status
        if force_body:
            body = force_body
        if status is not None:
            self.set_status(status)
        for name in exchange.response.headers.keys():
            value = exchange.response.headers[name]
            self.set_header(name, value)
        if body is not None:
            self.finish(body)
        else:
            self.finish()

    def update_exchange_from_response_message(self, exchange, message):
        (status_code, body, body_link, headers, _) = \
            unserialize_response_message(message)
        exchange.response.status_code = status_code
        # FIXME: body_link ???
        exchange.response.body = body
        exchange.reponse.headers = headers

    @gen.coroutine
    def handle(self, *args, **kwargs):
        exchange = HTTPExchange(self.request,
                                default_redis_host=options.redis_host,
                                default_redis_port=options.redis_port,
                                default_redis_queue=options.redis_queue)
        yield Rules.execute_input_actions(exchange)
        if exchange.response.status_code is not None and \
                exchange.response.status_code != "null":
            # so we don't push the request on redis
            # let's call output actions for headers and body
            yield Rules.execute_output_actions(exchange)
            self.return_http_reply(exchange)
        elif exchange.redis_queue == "null":
            # so we don't push the request on redis
            # let's call output actions for headers and body
            yield Rules.execute_output_actions(exchange)
            self.return_http_reply(exchange, force_status=404,
                                   force_body="no redis queue set")
        else:
            redis_pool = get_redis_pool(exchange.redis_host,
                                        exchange.redis_port)
            with (yield redis_pool.connected_client()) as redis:
                response_key = make_unique_id()
                serialized_request = serialize_http_request(
                    exchange.request,
                    dict_to_inject={
                        'response_key': response_key
                    })
                yield redis.call('LPUSH', exchange.redis_queue,
                                 serialized_request)
                before = datetime.datetime.now()
                while True:
                    result = yield redis.call('BRPOP', response_key, 1)
                    if result:
                        self.update_exchange_from_response_message(exchange,
                                                                   result[1])
                        yield Rules.execute_output_actions(exchange)
                        self.return_http_reply(exchange)
                        break
                    after = datetime.datetime.now()
                    delta = after - before
                    if delta.total_seconds() > options.timeout:
                        yield Rules.execute_output_actions(exchange)
                        self.return_http_reply(exchange, force_status=504,
                                               force_body="no reply from the backend")
                        break

def make_app():
    if options.config is not None:
        exec(open(options.config).read(), {})
    return Application([url(r"/.*", Handler)])

def main():
    parse_command_line()
    print("Start http2redis on http://localhost:{}".format(options.port))
    app = make_app()
    app.listen(options.port)
    ioloop.IOLoop.instance().start()
<|endoftext|>


<commit_before>"""
Django settings for timealign project.

Generated by 'django-admin startproject' using Django 1.11.13.

For more information on this file, see
https://docs.djangoproject.com/en/1.11/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.11/ref/settings/
"""
import os
from .settings_secret import *

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'bseaauw*m9wd2_h-&!+73yig@qd$r+th25k%*!79ckbvat)e3j'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

ALLOWED_HOSTS = []
if DEBUG:
    from subprocess import Popen, PIPE
    hostname = Popen(('hostname'), stdout=PIPE).communicate()[0].decode().strip()
    ALLOWED_HOSTS.append(hostname)
    ALLOWED_HOSTS.append('127.0.0.1')

# Application definition

INSTALLED_APPS = [
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    'bootstrap3',
    'ckeditor',
    'ckeditor_uploader',
    'django_filters',
    'django_object_actions',
    'nested_admin',
    'robots',
    'widget_tweaks',
    'perfectextractor_ui',

    'core.apps.CoreConfig',
    'news.apps.NewsConfig',
    'selections.apps.SelectionsConfig',
    'annotations.apps.AnnotationsConfig',
    'stats.apps.StatsConfig',

    'django.contrib.admin',

    'django.contrib.sites',
    'django.contrib.flatpages',
    'reversion'
]

MIDDLEWARE = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'django.contrib.flatpages.middleware.FlatpageFallbackMiddleware',
]

ROOT_URLCONF = 'timealign.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'timealign.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.11/ref/settings/#databases

DATABASES = {
    'default': user_db,
}

# Password validation
# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.11/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.11/howto/static-files/

STATIC_URL = '/static/'

LOGIN_REDIRECT_URL = '/timealign/status'

# Flatpages
# https://docs.djangoproject.com/en/1.11/ref/contrib/flatpages/

SITE_ID = 1

# File handling
MEDIA_ROOT = 'media'
MEDIA_URL = '/media/'

CKEDITOR_UPLOAD_PATH = 'news'
CKEDITOR_IMAGE_BACKEND = 'pillow'

# Path to where the PerfectExtractor corpora reside
PE_DATA_PATH = '/opt/corpora/'
<commit_msg>fixed settings.py to show core/base.html instead of perfectextractor-ui/base.html<commit_after>"""
Django settings for timealign project.

Generated by 'django-admin startproject' using Django 1.11.13.

For more information on this file, see
https://docs.djangoproject.com/en/1.11/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.11/ref/settings/
"""
import os
from .settings_secret import *

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'bseaauw*m9wd2_h-&!+73yig@qd$r+th25k%*!79ckbvat)e3j'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

ALLOWED_HOSTS = []
if DEBUG:
    from subprocess import Popen, PIPE
    hostname = Popen(('hostname'), stdout=PIPE).communicate()[0].decode().strip()
    ALLOWED_HOSTS.append(hostname)
    ALLOWED_HOSTS.append('localhost')
    ALLOWED_HOSTS.append('127.0.0.1')

# Application definition

INSTALLED_APPS = [
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    'bootstrap3',
    'ckeditor',
    'ckeditor_uploader',
    'django_filters',
    'django_object_actions',
    'nested_admin',
    'robots',

    'core.apps.CoreConfig',
    'news.apps.NewsConfig',
    'selections.apps.SelectionsConfig',
    'annotations.apps.AnnotationsConfig',
    'stats.apps.StatsConfig',

    'django.contrib.admin',

    'django.contrib.sites',
    'django.contrib.flatpages',
    'reversion',

    # The place where this is inserted in this list matters. If the core.apps.CoreConfig comes after the perfectextractor_ui, the base.html will be in conflict and core/base.html will not be loaded, but instead loading the template of perfectextractor-ui/base.html.
    'widget_tweaks',
    'perfectextractor_ui'
]

MIDDLEWARE = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'django.contrib.flatpages.middleware.FlatpageFallbackMiddleware',
]

ROOT_URLCONF = 'timealign.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'timealign.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.11/ref/settings/#databases

DATABASES = {
    'default': user_db,
}

# Password validation
# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.11/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.11/howto/static-files/

STATIC_URL = '/static/'

LOGIN_REDIRECT_URL = '/timealign/status'

# Flatpages
# https://docs.djangoproject.com/en/1.11/ref/contrib/flatpages/

SITE_ID = 1

# File handling
MEDIA_ROOT = 'media'
MEDIA_URL = '/media/'

CKEDITOR_UPLOAD_PATH = 'news'
CKEDITOR_IMAGE_BACKEND = 'pillow'

# Path to where the PerfectExtractor corpora reside
PE_DATA_PATH = '/opt/corpora/'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    tipfy.auth.twitter
    ~~~~~~~~~~~~~~~~~~

    Implementation of Twitter authentication scheme.

    Ported from `tornado.auth`_.

    :copyright: 2009 Facebook.
    :copyright: 2010 tipfy.org.
    :license: Apache License Version 2.0, see LICENSE.txt for more details.
"""
from __future__ import absolute_import
import functools
import logging
import urllib

from google.appengine.api import urlfetch

from tipfy import REQUIRED_VALUE
from tipfy.utils import json_decode, json_encode
from .oauth import OAuthMixin

#: Default configuration values for this module. Keys are:
#:
#: consumer_key
#:     Key provided when you register an application with Twitter.
#:
#: consumer_secret
#:     Secret provided when you register an application with Twitter.
default_config = {
    'consumer_key':    REQUIRED_VALUE,
    'consumer_secret': REQUIRED_VALUE,
}

class TwitterMixin(OAuthMixin):
    """A :class:`tipfy.RequestHandler` mixin that implements Twitter OAuth
    authentication.

    To authenticate with Twitter, register your application with
    Twitter at http://twitter.com/apps. Then copy your Consumer Key and
    Consumer Secret to ``config.py``::

        config['tipfyext.auth.twitter'] = {
            'consumer_key':    'XXXXXXXXXXXXXXX',
            'consumer_secret': 'XXXXXXXXXXXXXXX',
        }

    When your application is set up, you can use the TwitterMixin to
    authenticate the user with Twitter and get access to their stream.
    You must use the mixin on the handler for the URL you registered as your
    application's Callback URL. For example::

        from tipfy import RequestHandler
        from tipfyext.auth.twitter import TwitterMixin
        from tipfyext.session import CookieMixin, SessionMiddleware

        class TwitterHandler(RequestHandler, CookieMixin, TwitterMixin):
            middleware = [SessionMiddleware]

            def get(self):
                if self.request.args.get('oauth_token', None):
                    return self.get_authenticated_user(self._on_auth)

                return self.authorize_redirect()

            def _on_auth(self, user):
                if not user:
                    self.abort(403)

                # Set the user in the session.
                # ...

    The user object returned by get_authenticated_user() includes the
    attributes 'username', 'name', and all of the custom Twitter user
    attributes describe at
    http://apiwiki.twitter.com/Twitter-REST-API-Method%3A-users%C2%A0show
    in addition to 'access_token'. You should save the access token with
    the user; it is required to make requests on behalf of the user later
    with twitter_request().
    """
    _OAUTH_REQUEST_TOKEN_URL = 'http://api.twitter.com/oauth/request_token'
    _OAUTH_ACCESS_TOKEN_URL = 'http://api.twitter.com/oauth/access_token'
    _OAUTH_AUTHORIZE_URL = 'http://api.twitter.com/oauth/authorize'
    _OAUTH_AUTHENTICATE_URL = 'http://api.twitter.com/oauth/authenticate'
    _OAUTH_NO_CALLBACKS = True

    def _twitter_consumer_key(self):
        return self.app.config[__name__]['consumer_key']

    def _twitter_consumer_secret(self):
        return self.app.config[__name__]['consumer_secret']

    def _oauth_consumer_token(self):
        return dict(
            key=self._twitter_consumer_key(),
            secret=self._twitter_consumer_secret())

    def authenticate_redirect(self):
        """Just like authorize_redirect(), but auto-redirects if authorized.

        This is generally the right interface to use if you are using
        Twitter for single-sign on.
        """
        url = self._oauth_request_token_url()
        try:
            response = urlfetch.fetch(url, deadline=10)
        except urlfetch.DownloadError, e:
            logging.exception(e)
            response = None

        return self._on_request_token(self._OAUTH_AUTHENTICATE_URL, None,
            response)

    def twitter_request(self, path, callback=None, access_token=None,
                           post_args=None, **args):
        """Fetches the given API path, e.g., '/statuses/user_timeline/btaylor'

        The path should not include the format (we automatically append
        '.json' and parse the JSON output).

        If the request is a POST, post_args should be provided. Query
        string arguments should be given as keyword arguments.

        All the Twitter methods are documented at
        http://apiwiki.twitter.com/Twitter-API-Documentation.

        Many methods require an OAuth access token which you can obtain
        through authorize_redirect() and get_authenticated_user(). The
        user returned through that process includes an 'access_token'
        attribute that can be used to make authenticated requests via
        this method. Example usage::

            from tipfy import RequestHandler, Response
            from tipfyext.auth.twitter import TwitterMixin
            from tipfyext.session import CookieMixin, SessionMiddleware

            class TwitterHandler(RequestHandler, CookieMixin, TwitterMixin):
                middleware = [SessionMiddleware]

                def get(self):
                    return self.twitter_request(
                        '/statuses/update',
                        post_args={'status': 'Testing Twitter Mixin'},
                        access_token=user['access_token'],
                        callback=self._on_post)

                def _on_post(self, new_entry):
                    if not new_entry:
                        # Call failed; perhaps missing permission?
                        return self.authorize_redirect()

                    return Response('Posted a message!')
        """
        # Add the OAuth resource request signature if we have credentials
        url = 'http://api.twitter.com/1' + path + '.json'
        if access_token:
            all_args = {}
            all_args.update(args)
            all_args.update(post_args or {})
            method = 'POST' if post_args is not None else 'GET'
            oauth = self._oauth_request_parameters(
                url, access_token, all_args, method=method)
            args.update(oauth)

        if args:
            url += '?' + urllib.urlencode(args)

        try:
            if post_args is not None:
                response = urlfetch.fetch(url, method='POST',
                    payload=urllib.urlencode(post_args), deadline=10)
            else:
                response = urlfetch.fetch(url, deadline=10)
        except urlfetch.DownloadError, e:
            logging.exception(e)
            response = None

        if not callback:
            # Don't preprocess the response, just return a bare one.
            return response

        return self._on_twitter_request(callback, response)

    def _on_twitter_request(self, callback, response):
        if not response:
            logging.warning('Could not get Twitter request token.')
            return callback(None)
        elif response.status_code < 200 or response.status_code >= 300:
            logging.warning('Invalid Twitter response (%d): %s',
                response.status_code, response.content)
            return callback(None)

        return callback(json_decode(response.content))

    def _oauth_get_user(self, access_token, callback):
        callback = functools.partial(self._parse_user_response, callback)
        return self.twitter_request(
            '/users/show/' + access_token['screen_name'],
            access_token=access_token, callback=callback)

    def _parse_user_response(self, callback, user):
        if user:
            user['username'] = user['screen_name']

        return callback(user)
<commit_msg>Patch for Twitter /search endpoint. Thanks, till.<commit_after># -*- coding: utf-8 -*-
"""
    tipfy.auth.twitter
    ~~~~~~~~~~~~~~~~~~

    Implementation of Twitter authentication scheme.

    Ported from `tornado.auth`_.

    :copyright: 2009 Facebook.
    :copyright: 2010 tipfy.org.
    :license: Apache License Version 2.0, see LICENSE.txt for more details.
"""
from __future__ import absolute_import
import functools
import logging
import urllib

from google.appengine.api import urlfetch

from tipfy import REQUIRED_VALUE
from tipfy.utils import json_decode, json_encode
from .oauth import OAuthMixin

#: Default configuration values for this module. Keys are:
#:
#: consumer_key
#:     Key provided when you register an application with Twitter.
#:
#: consumer_secret
#:     Secret provided when you register an application with Twitter.
default_config = {
    'consumer_key':    REQUIRED_VALUE,
    'consumer_secret': REQUIRED_VALUE,
}

class TwitterMixin(OAuthMixin):
    """A :class:`tipfy.RequestHandler` mixin that implements Twitter OAuth
    authentication.

    To authenticate with Twitter, register your application with
    Twitter at http://twitter.com/apps. Then copy your Consumer Key and
    Consumer Secret to ``config.py``::

        config['tipfyext.auth.twitter'] = {
            'consumer_key':    'XXXXXXXXXXXXXXX',
            'consumer_secret': 'XXXXXXXXXXXXXXX',
        }

    When your application is set up, you can use the TwitterMixin to
    authenticate the user with Twitter and get access to their stream.
    You must use the mixin on the handler for the URL you registered as your
    application's Callback URL. For example::

        from tipfy import RequestHandler
        from tipfyext.auth.twitter import TwitterMixin
        from tipfyext.session import CookieMixin, SessionMiddleware

        class TwitterHandler(RequestHandler, CookieMixin, TwitterMixin):
            middleware = [SessionMiddleware]

            def get(self):
                if self.request.args.get('oauth_token', None):
                    return self.get_authenticated_user(self._on_auth)

                return self.authorize_redirect()

            def _on_auth(self, user):
                if not user:
                    self.abort(403)

                # Set the user in the session.
                # ...

    The user object returned by get_authenticated_user() includes the
    attributes 'username', 'name', and all of the custom Twitter user
    attributes describe at
    http://apiwiki.twitter.com/Twitter-REST-API-Method%3A-users%C2%A0show
    in addition to 'access_token'. You should save the access token with
    the user; it is required to make requests on behalf of the user later
    with twitter_request().
    """
    _OAUTH_REQUEST_TOKEN_URL = 'http://api.twitter.com/oauth/request_token'
    _OAUTH_ACCESS_TOKEN_URL = 'http://api.twitter.com/oauth/access_token'
    _OAUTH_AUTHORIZE_URL = 'http://api.twitter.com/oauth/authorize'
    _OAUTH_AUTHENTICATE_URL = 'http://api.twitter.com/oauth/authenticate'
    _OAUTH_NO_CALLBACKS = True

    def _twitter_consumer_key(self):
        return self.app.config[__name__]['consumer_key']

    def _twitter_consumer_secret(self):
        return self.app.config[__name__]['consumer_secret']

    def _oauth_consumer_token(self):
        return dict(
            key=self._twitter_consumer_key(),
            secret=self._twitter_consumer_secret())

    def authenticate_redirect(self):
        """Just like authorize_redirect(), but auto-redirects if authorized.

        This is generally the right interface to use if you are using
        Twitter for single-sign on.
        """
        url = self._oauth_request_token_url()
        try:
            response = urlfetch.fetch(url, deadline=10)
        except urlfetch.DownloadError, e:
            logging.exception(e)
            response = None

        return self._on_request_token(self._OAUTH_AUTHENTICATE_URL, None,
            response)

    def twitter_request(self, path, callback=None, access_token=None,
                           post_args=None, **args):
        """Fetches the given API path, e.g., '/statuses/user_timeline/btaylor'

        The path should not include the format (we automatically append
        '.json' and parse the JSON output).

        If the request is a POST, post_args should be provided. Query
        string arguments should be given as keyword arguments.

        All the Twitter methods are documented at
        http://apiwiki.twitter.com/Twitter-API-Documentation.

        Many methods require an OAuth access token which you can obtain
        through authorize_redirect() and get_authenticated_user(). The
        user returned through that process includes an 'access_token'
        attribute that can be used to make authenticated requests via
        this method. Example usage::

            from tipfy import RequestHandler, Response
            from tipfyext.auth.twitter import TwitterMixin
            from tipfyext.session import CookieMixin, SessionMiddleware

            class TwitterHandler(RequestHandler, CookieMixin, TwitterMixin):
                middleware = [SessionMiddleware]

                def get(self):
                    return self.twitter_request(
                        '/statuses/update',
                        post_args={'status': 'Testing Twitter Mixin'},
                        access_token=user['access_token'],
                        callback=self._on_post)

                def _on_post(self, new_entry):
                    if not new_entry:
                        # Call failed; perhaps missing permission?
                        return self.authorize_redirect()

                    return Response('Posted a message!')
        """
        # Add the OAuth resource request signature if we have credentials
        if path == '/search':
            url = 'http://search.twitter.com/search.json'
        else:
            url = 'http://api.twitter.com/1' + path + '.json'

        if access_token:
            all_args = {}
            all_args.update(args)
            all_args.update(post_args or {})
            method = 'POST' if post_args is not None else 'GET'
            oauth = self._oauth_request_parameters(
                url, access_token, all_args, method=method)
            args.update(oauth)

        if args:
            url += '?' + urllib.urlencode(args)

        try:
            if post_args is not None:
                response = urlfetch.fetch(url, method='POST',
                    payload=urllib.urlencode(post_args), deadline=10)
            else:
                response = urlfetch.fetch(url, deadline=10)
        except urlfetch.DownloadError, e:
            logging.exception(e)
            response = None

        if not callback:
            # Don't preprocess the response, just return a bare one.
            return response

        return self._on_twitter_request(callback, response)

    def _on_twitter_request(self, callback, response):
        if not response:
            logging.warning('Could not get Twitter request token.')
            return callback(None)
        elif response.status_code < 200 or response.status_code >= 300:
            logging.warning('Invalid Twitter response (%d): %s',
                response.status_code, response.content)
            return callback(None)

        return callback(json_decode(response.content))

    def _oauth_get_user(self, access_token, callback):
        callback = functools.partial(self._parse_user_response, callback)
        return self.twitter_request(
            '/users/show/' + access_token['screen_name'],
            access_token=access_token, callback=callback)

    def _parse_user_response(self, callback, user):
        if user:
            user['username'] = user['screen_name']

        return callback(user)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Original Perl version by: John Gruber http://daringfireball.net/ 10 May 2008
Python version by Stuart Colville http://muffinresearch.co.uk
License: http://www.opensource.org/licenses/mit-license.php
"""

import re

__all__ = ['titlecase']
__version__ = '0.8.1'

SMALL = 'a|an|and|as|at|but|by|en|for|if|in|of|on|or|the|to|v\.?|via|vs\.?'
PUNCT = r"""!"#$%&'‘()*+,\-./:;?@[\\\]_`{|}~"""

SMALL_WORDS = re.compile(r'^(%s)$' % SMALL, re.I)
INLINE_PERIOD = re.compile(r'[a-z][.][a-z]', re.I)
UC_ELSEWHERE = re.compile(r'[%s]*?[a-zA-Z]+[A-Z]+?' % PUNCT)
CAPFIRST = re.compile(r"^[%s]*?([A-Za-z])" % PUNCT)
SMALL_FIRST = re.compile(r'^([%s]*)(%s)\b' % (PUNCT, SMALL), re.I)
SMALL_LAST = re.compile(r'\b(%s)[%s]?$' % (SMALL, PUNCT), re.I)
SUBPHRASE = re.compile(r'([:.;?!\-\—][ ])(%s)' % SMALL)
APOS_SECOND = re.compile(r"^[dol]{1}['‘]{1}[a-z]+(?:['s]{2})?$", re.I)
ALL_CAPS = re.compile(r'^[A-Z\s\d%s]+$' % PUNCT)
UC_INITIALS = re.compile(r"^(?:[A-Z]{1}\.{1}|[A-Z]{1}\.{1}[A-Z]{1})+$")
MAC_MC = re.compile(r"^([Mm]c|MC)(\w.+)")

def set_small_word_list(small=SMALL):
    global SMALL_WORDS
    global SMALL_FIRST
    global SMALL_LAST
    global SUBPHRASE
    SMALL_WORDS = re.compile(r'^(%s)$' % small, re.I)
    SMALL_FIRST = re.compile(r'^([%s]*)(%s)\b' % (PUNCT, small), re.I)
    SMALL_LAST = re.compile(r'\b(%s)[%s]?$' % (small, PUNCT), re.I)
    SUBPHRASE = re.compile(r'([:.;?!][ ])(%s)' % small)

def titlecase(text, callback=None):
    """
    Titlecases input text

    This filter changes all words to Title Caps, and attempts to be clever
    about *un*capitalizing SMALL words like a/an/the in the input.

    The list of "SMALL words" which are not capped comes from
    the New York Times Manual of Style, plus 'vs' and 'v'.

    """

    lines = re.split('[\r\n]+', text)
    processed = []
    for line in lines:
        all_caps = ALL_CAPS.match(line)
        words = re.split('[\t ]', line)
        tc_line = []
        for word in words:
            if callback:
                new_word = callback(word, all_caps=all_caps)
                if new_word:
                    tc_line.append(new_word)
                    continue

            if all_caps:
                if UC_INITIALS.match(word):
                    tc_line.append(word)
                    continue

            if APOS_SECOND.match(word):
                if len(word[0]) == 1 and word[0] not in 'aeiouAEIOU':
                    word = word[0].lower() + word[1] + word[2].upper() + word[3:]
                else:
                    word = word[0].upper() + word[1] + word[2].upper() + word[3:]
                tc_line.append(word)
                continue

            match = MAC_MC.match(word)
            if match:
                tc_line.append("%s%s" % (match.group(1).capitalize(),
                                         titlecase(match.group(2),callback)))
                continue

            if INLINE_PERIOD.search(word) or (not all_caps and UC_ELSEWHERE.match(word)):
                tc_line.append(word)
                continue
            if SMALL_WORDS.match(word):
                tc_line.append(word.lower())
                continue

            if "/" in word and "//" not in word:
                slashed = map(lambda t: titlecase(t,callback), word.split('/'))
                tc_line.append("/".join(slashed))
                continue

            if '-' in word:
                hyphenated = map(lambda t: titlecase(t,callback), word.split('-'))
                tc_line.append("-".join(hyphenated))
                continue

            if all_caps:
                word = word.lower()

            # Just a normal word that needs to be capitalized
            tc_line.append(CAPFIRST.sub(lambda m: m.group(0).upper(), word))

        result = " ".join(tc_line)

        result = SMALL_FIRST.sub(lambda m: '%s%s' % (
            m.group(1),
            m.group(2).capitalize()
        ), result)

        result = SMALL_LAST.sub(lambda m: m.group(0).capitalize(), result)

        result = SUBPHRASE.sub(lambda m: '%s%s' % (
            m.group(1),
            m.group(2).capitalize()
        ), result)

        processed.append(result)

    return "\n".join(processed)
<commit_msg>Version bump for MAC_MC hyphen issues<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Original Perl version by: John Gruber http://daringfireball.net/ 10 May 2008
Python version by Stuart Colville http://muffinresearch.co.uk
License: http://www.opensource.org/licenses/mit-license.php
"""

import re

__all__ = ['titlecase']
__version__ = '0.8.2'

SMALL = 'a|an|and|as|at|but|by|en|for|if|in|of|on|or|the|to|v\.?|via|vs\.?'
PUNCT = r"""!"#$%&'‘()*+,\-./:;?@[\\\]_`{|}~"""

SMALL_WORDS = re.compile(r'^(%s)$' % SMALL, re.I)
INLINE_PERIOD = re.compile(r'[a-z][.][a-z]', re.I)
UC_ELSEWHERE = re.compile(r'[%s]*?[a-zA-Z]+[A-Z]+?' % PUNCT)
CAPFIRST = re.compile(r"^[%s]*?([A-Za-z])" % PUNCT)
SMALL_FIRST = re.compile(r'^([%s]*)(%s)\b' % (PUNCT, SMALL), re.I)
SMALL_LAST = re.compile(r'\b(%s)[%s]?$' % (SMALL, PUNCT), re.I)
SUBPHRASE = re.compile(r'([:.;?!\-\—][ ])(%s)' % SMALL)
APOS_SECOND = re.compile(r"^[dol]{1}['‘]{1}[a-z]+(?:['s]{2})?$", re.I)
ALL_CAPS = re.compile(r'^[A-Z\s\d%s]+$' % PUNCT)
UC_INITIALS = re.compile(r"^(?:[A-Z]{1}\.{1}|[A-Z]{1}\.{1}[A-Z]{1})+$")
MAC_MC = re.compile(r"^([Mm]c|MC)(\w.+)")

def set_small_word_list(small=SMALL):
    global SMALL_WORDS
    global SMALL_FIRST
    global SMALL_LAST
    global SUBPHRASE
    SMALL_WORDS = re.compile(r'^(%s)$' % small, re.I)
    SMALL_FIRST = re.compile(r'^([%s]*)(%s)\b' % (PUNCT, small), re.I)
    SMALL_LAST = re.compile(r'\b(%s)[%s]?$' % (small, PUNCT), re.I)
    SUBPHRASE = re.compile(r'([:.;?!][ ])(%s)' % small)

def titlecase(text, callback=None):
    """
    Titlecases input text

    This filter changes all words to Title Caps, and attempts to be clever
    about *un*capitalizing SMALL words like a/an/the in the input.

    The list of "SMALL words" which are not capped comes from
    the New York Times Manual of Style, plus 'vs' and 'v'.

    """

    lines = re.split('[\r\n]+', text)
    processed = []
    for line in lines:
        all_caps = ALL_CAPS.match(line)
        words = re.split('[\t ]', line)
        tc_line = []
        for word in words:
            if callback:
                new_word = callback(word, all_caps=all_caps)
                if new_word:
                    tc_line.append(new_word)
                    continue

            if all_caps:
                if UC_INITIALS.match(word):
                    tc_line.append(word)
                    continue

            if APOS_SECOND.match(word):
                if len(word[0]) == 1 and word[0] not in 'aeiouAEIOU':
                    word = word[0].lower() + word[1] + word[2].upper() + word[3:]
                else:
                    word = word[0].upper() + word[1] + word[2].upper() + word[3:]
                tc_line.append(word)
                continue

            match = MAC_MC.match(word)
            if match:
                tc_line.append("%s%s" % (match.group(1).capitalize(),
                                         titlecase(match.group(2),callback)))
                continue

            if INLINE_PERIOD.search(word) or (not all_caps and UC_ELSEWHERE.match(word)):
                tc_line.append(word)
                continue
            if SMALL_WORDS.match(word):
                tc_line.append(word.lower())
                continue

            if "/" in word and "//" not in word:
                slashed = map(lambda t: titlecase(t,callback), word.split('/'))
                tc_line.append("/".join(slashed))
                continue

            if '-' in word:
                hyphenated = map(lambda t: titlecase(t,callback), word.split('-'))
                tc_line.append("-".join(hyphenated))
                continue

            if all_caps:
                word = word.lower()

            # Just a normal word that needs to be capitalized
            tc_line.append(CAPFIRST.sub(lambda m: m.group(0).upper(), word))

        result = " ".join(tc_line)

        result = SMALL_FIRST.sub(lambda m: '%s%s' % (
            m.group(1),
            m.group(2).capitalize()
        ), result)

        result = SMALL_LAST.sub(lambda m: m.group(0).capitalize(), result)

        result = SUBPHRASE.sub(lambda m: '%s%s' % (
            m.group(1),
            m.group(2).capitalize()
        ), result)

        processed.append(result)

    return "\n".join(processed)
<|endoftext|>