<commit_before>from __future__ import print_function
import numpy as np
import tensorflow as tf

import argparse
import time
import os
from six.moves import cPickle

from utils import TextLoader
from model import Model

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--data_dir', type=str, default='data/tinyshakespeare',
                       help='data directory containing input.txt')
    parser.add_argument('--save_dir', type=str, default='save',
                       help='directory to store checkpointed models')
    parser.add_argument('--rnn_size', type=int, default=256,
                       help='size of RNN hidden state')
    parser.add_argument('--num_layers', type=int, default=2,
                       help='number of layers in the RNN')
    parser.add_argument('--model', type=str, default='lstm',
                       help='rnn, gru, or lstm')
    parser.add_argument('--batch_size', type=int, default=50,
                       help='minibatch size')
    parser.add_argument('--seq_length', type=int, default=25,
                       help='RNN sequence length')
    parser.add_argument('--num_epochs', type=int, default=50,
                       help='number of epochs')
    parser.add_argument('--save_every', type=int, default=1000,
                       help='save frequency')
    parser.add_argument('--grad_clip', type=float, default=5.,
                       help='clip gradients at this value')
    parser.add_argument('--learning_rate', type=float, default=0.002,
                       help='learning rate')
    parser.add_argument('--decay_rate', type=float, default=0.97,
                       help='decay rate for rmsprop')
    parser.add_argument('--init_from', type=str, default=None,
                       help="""continue training from saved model at this path. Path must contain files saved by previous training process:
                            'config.pkl'        : configuration;
                            'words_vocab.pkl'   : vocabulary definitions;
                            'checkpoint'        : paths to model file(s) (created by tf).
                                                  Note: this file contains absolute paths, be careful when moving files around;
                            'model.ckpt-*'      : file(s) with model definition (created by tf)
                        """)
    args = parser.parse_args()
    train(args)

def train(args):
    data_loader = TextLoader(args.data_dir, args.batch_size, args.seq_length)
    args.vocab_size = data_loader.vocab_size

    # check compatibility if training is continued from previously saved model
    if args.init_from is not None:
        # check if all necessary files exist
        assert os.path.isdir(args.init_from)," %s must be a path" % args.init_from
        assert os.path.isfile(os.path.join(args.init_from,"config.pkl")),"config.pkl file does not exist in path %s"%args.init_from
        assert os.path.isfile(os.path.join(args.init_from,"words_vocab.pkl")),"words_vocab.pkl.pkl file does not exist in path %s" % args.init_from
        ckpt = tf.train.get_checkpoint_state(args.init_from)
        assert ckpt,"No checkpoint found"
        assert ckpt.model_checkpoint_path,"No model path found in checkpoint"

        # open old config and check if models are compatible
        with open(os.path.join(args.init_from, 'config.pkl'), 'rb') as f:
            saved_model_args = cPickle.load(f)
        need_be_same=["model","rnn_size","num_layers","seq_length"]
        for checkme in need_be_same:
            assert vars(saved_model_args)[checkme]==vars(args)[checkme],"Command line argument and saved model disagree on '%s' "%checkme

        # open saved vocab/dict and check if vocabs/dicts are compatible
        with open(os.path.join(args.init_from, 'words_vocab.pkl'), 'rb') as f:
            saved_words, saved_vocab = cPickle.load(f)
        assert saved_words==data_loader.words, "Data and loaded model disagree on word set!"
        assert saved_vocab==data_loader.vocab, "Data and loaded model disagree on dictionary mappings!"

    with open(os.path.join(args.save_dir, 'config.pkl'), 'wb') as f:
        cPickle.dump(args, f)
    with open(os.path.join(args.save_dir, 'words_vocab.pkl'), 'wb') as f:
        cPickle.dump((data_loader.words, data_loader.vocab), f)

    model = Model(args)

    merged = tf.summary.merge_all()
    train_writer = tf.summary.FileWriter('logs')

    with tf.Session() as sess:
        train_writer.add_graph(sess.graph)
        tf.global_variables_initializer().run()
        saver = tf.train.Saver(tf.global_variables())
        # restore model
        if args.init_from is not None:
            saver.restore(sess, ckpt.model_checkpoint_path)
        for e in range(model.epoch_pointer.eval(), args.num_epochs):
            sess.run(tf.assign(model.lr, args.learning_rate * (args.decay_rate ** e)))
            data_loader.reset_batch_pointer()
            state = sess.run(model.initial_state)
            speed = 0
            if args.init_from is None:
                assign_op = model.batch_pointer.assign(0)
                sess.run(assign_op)
                assign_op = model.epoch_pointer.assign(e)
                sess.run(assign_op)
            if args.init_from is not None:
                data_loader.pointer = model.batch_pointer.eval()
                args.init_from = None
            for b in range(data_loader.pointer, data_loader.num_batches):
                start = time.time()
                x, y = data_loader.next_batch()
                feed = {model.input_data: x, model.targets: y, model.initial_state: state,
                        model.batch_time: speed}
                summary, train_loss, state, _, _ = sess.run([merged, model.cost, model.final_state,
                                                             model.train_op, model.inc_batch_pointer_op], feed)
                train_writer.add_summary(summary, e * data_loader.num_batches + b)
                speed = time.time() - start
                if (e * data_loader.num_batches + b) % args.batch_size == 0:
                    print("{}/{} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}" \
                        .format(e * data_loader.num_batches + b,
                                args.num_epochs * data_loader.num_batches,
                                e, train_loss, speed))
                if (e * data_loader.num_batches + b) % args.save_every == 0 \
                        or (e==args.num_epochs-1 and b == data_loader.num_batches-1): # save for the last result
                    checkpoint_path = os.path.join(args.save_dir, 'model.ckpt')
                    saver.save(sess, checkpoint_path, global_step = e * data_loader.num_batches + b)
                    print("model saved to {}".format(checkpoint_path))
        train_writer.close()

if __name__ == '__main__':
    main()
<commit_msg>Allow % of GPU memory to be allocated<commit_after>from __future__ import print_function
import numpy as np
import tensorflow as tf

import argparse
import time
import os
from six.moves import cPickle

from utils import TextLoader
from model import Model

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--data_dir', type=str, default='data/tinyshakespeare',
                       help='data directory containing input.txt')
    parser.add_argument('--save_dir', type=str, default='save',
                       help='directory to store checkpointed models')
    parser.add_argument('--rnn_size', type=int, default=256,
                       help='size of RNN hidden state')
    parser.add_argument('--num_layers', type=int, default=2,
                       help='number of layers in the RNN')
    parser.add_argument('--model', type=str, default='lstm',
                       help='rnn, gru, or lstm')
    parser.add_argument('--batch_size', type=int, default=50,
                       help='minibatch size')
    parser.add_argument('--seq_length', type=int, default=25,
                       help='RNN sequence length')
    parser.add_argument('--num_epochs', type=int, default=50,
                       help='number of epochs')
    parser.add_argument('--save_every', type=int, default=1000,
                       help='save frequency')
    parser.add_argument('--grad_clip', type=float, default=5.,
                       help='clip gradients at this value')
    parser.add_argument('--learning_rate', type=float, default=0.002,
                       help='learning rate')
    parser.add_argument('--decay_rate', type=float, default=0.97,
                       help='decay rate for rmsprop')
    parser.add_argument('--gpu_mem', type=float, default=0.666,
                       help='% of gpu memory to be allocated to this process. Default is 66.6%')
    parser.add_argument('--init_from', type=str, default=None,
                       help="""continue training from saved model at this path. Path must contain files saved by previous training process:
                            'config.pkl'        : configuration;
                            'words_vocab.pkl'   : vocabulary definitions;
                            'checkpoint'        : paths to model file(s) (created by tf).
                                                  Note: this file contains absolute paths, be careful when moving files around;
                            'model.ckpt-*'      : file(s) with model definition (created by tf)
                        """)
    args = parser.parse_args()
    train(args)

def train(args):
    data_loader = TextLoader(args.data_dir, args.batch_size, args.seq_length)
    args.vocab_size = data_loader.vocab_size

    # check compatibility if training is continued from previously saved model
    if args.init_from is not None:
        # check if all necessary files exist
        assert os.path.isdir(args.init_from)," %s must be a path" % args.init_from
        assert os.path.isfile(os.path.join(args.init_from,"config.pkl")),"config.pkl file does not exist in path %s"%args.init_from
        assert os.path.isfile(os.path.join(args.init_from,"words_vocab.pkl")),"words_vocab.pkl.pkl file does not exist in path %s" % args.init_from
        ckpt = tf.train.get_checkpoint_state(args.init_from)
        assert ckpt,"No checkpoint found"
        assert ckpt.model_checkpoint_path,"No model path found in checkpoint"

        # open old config and check if models are compatible
        with open(os.path.join(args.init_from, 'config.pkl'), 'rb') as f:
            saved_model_args = cPickle.load(f)
        need_be_same=["model","rnn_size","num_layers","seq_length"]
        for checkme in need_be_same:
            assert vars(saved_model_args)[checkme]==vars(args)[checkme],"Command line argument and saved model disagree on '%s' "%checkme

        # open saved vocab/dict and check if vocabs/dicts are compatible
        with open(os.path.join(args.init_from, 'words_vocab.pkl'), 'rb') as f:
            saved_words, saved_vocab = cPickle.load(f)
        assert saved_words==data_loader.words, "Data and loaded model disagree on word set!"
        assert saved_vocab==data_loader.vocab, "Data and loaded model disagree on dictionary mappings!"

    with open(os.path.join(args.save_dir, 'config.pkl'), 'wb') as f:
        cPickle.dump(args, f)
    with open(os.path.join(args.save_dir, 'words_vocab.pkl'), 'wb') as f:
        cPickle.dump((data_loader.words, data_loader.vocab), f)

    model = Model(args)

    merged = tf.summary.merge_all()
    train_writer = tf.summary.FileWriter('logs')
    gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=args.gpu_mem)

    with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
        train_writer.add_graph(sess.graph)
        tf.global_variables_initializer().run()
        saver = tf.train.Saver(tf.global_variables())
        # restore model
        if args.init_from is not None:
            saver.restore(sess, ckpt.model_checkpoint_path)
        for e in range(model.epoch_pointer.eval(), args.num_epochs):
            sess.run(tf.assign(model.lr, args.learning_rate * (args.decay_rate ** e)))
            data_loader.reset_batch_pointer()
            state = sess.run(model.initial_state)
            speed = 0
            if args.init_from is None:
                assign_op = model.batch_pointer.assign(0)
                sess.run(assign_op)
                assign_op = model.epoch_pointer.assign(e)
                sess.run(assign_op)
            if args.init_from is not None:
                data_loader.pointer = model.batch_pointer.eval()
                args.init_from = None
            for b in range(data_loader.pointer, data_loader.num_batches):
                start = time.time()
                x, y = data_loader.next_batch()
                feed = {model.input_data: x, model.targets: y, model.initial_state: state,
                        model.batch_time: speed}
                summary, train_loss, state, _, _ = sess.run([merged, model.cost, model.final_state,
                                                             model.train_op, model.inc_batch_pointer_op], feed)
                train_writer.add_summary(summary, e * data_loader.num_batches + b)
                speed = time.time() - start
                if (e * data_loader.num_batches + b) % args.batch_size == 0:
                    print("{}/{} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}" \
                        .format(e * data_loader.num_batches + b,
                                args.num_epochs * data_loader.num_batches,
                                e, train_loss, speed))
                if (e * data_loader.num_batches + b) % args.save_every == 0 \
                        or (e==args.num_epochs-1 and b == data_loader.num_batches-1): # save for the last result
                    checkpoint_path = os.path.join(args.save_dir, 'model.ckpt')
                    saver.save(sess, checkpoint_path, global_step = e * data_loader.num_batches + b)
                    print("model saved to {}".format(checkpoint_path))
        train_writer.close()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

from __future__ import division
import os
import sys
import time
import numpy
import theano
import theano.tensor as T
import time
import math
import argparse

from dlm.io.lmDatasetReader import LMDatasetReader
from dlm.io.irisDatasetReader import IRISDatasetReader
#from dlm.models.mlp import MLP
from dlm.models.ltmlp import MLP
import dlm.utils as U

def test_mlp(trainset, devset, testset, learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=100, n_hidden=10):

	n_train_batches = trainset.get_num_batches()
	n_dev_batches = devset.get_num_batches()
	n_test_batches = testset.get_num_batches()

	print '... building the model'

	index = T.lscalar()		# index to a [mini]batch
	x = T.imatrix('x')		# the data is presented as rasterized images
	y = T.ivector('y')		# the labels are presented as 1D vector of
							# [int] labels

	rng = numpy.random.RandomState(1234)

	# construct the MLP class
	classifier = MLP(
		rng=rng,
		input=x,
		vocab_size=trainset.get_vocab_size(),
		emb_dim=10,
		ngram_size=trainset.get_ngram_size(),
		n_hidden=n_hidden,
		n_out=trainset.get_num_classes()
	)

	cost = (
		classifier.negative_log_likelihood(y)
		#+ L1_reg * classifier.L1
		#+ L2_reg * classifier.L2_sqr
	)

	gparams = [T.grad(cost, param) for param in classifier.params]

	updates = [
		(param, param - learning_rate * gparam)
		for param, gparam in zip(classifier.params, gparams)
	]

	#f = theano.function([index], dataset.get_features(index))

	test_model = theano.function(
		inputs=[index],
		outputs=classifier.errors(y),
		givens={
			x: testset.get_x(index),
			y: testset.get_y(index)
		}
	)

	validate_model = theano.function(
		inputs=[index],
		outputs=classifier.errors(y),
		givens={
			x: devset.get_x(index),
			y: devset.get_y(index)
		}
	)

	train_model = theano.function(
		inputs=[index],
		outputs=cost,
		updates=updates,
		givens={
			x: trainset.get_x(index),
			y: trainset.get_y(index)
		}
	)

	dev_negative_log_likelihood = theano.function(
		inputs=[index],
		outputs=classifier.negative_log_likelihood(y),
		givens={
			x: devset.get_x(index),
			y: devset.get_y(index)
		}
	)

	print '... training'

	patience = 10000											# look as this many examples regardless
	patience_increase = 2										# wait this much longer when a new best is found
	improvement_threshold = 0.995								# a relative improvement of this much is considered significant
	validation_frequency = min(n_train_batches, patience / 2)	# go through this many minibatche before
																# checking the network on the validation set;
																# in this case we check every epoch

	best_validation_loss = numpy.inf
	best_iter = 0
	test_score = 0.
	start_time = time.clock()

	epoch = 0
	done_looping = False

	while (epoch < n_epochs) and (not done_looping):
		epoch = epoch + 1
		print "Epoch " + str(epoch)
		minibatch_avg_cost_sum = 0
		for minibatch_index in xrange(n_train_batches):
			minibatch_avg_cost = train_model(minibatch_index)
			minibatch_avg_cost_sum += minibatch_avg_cost
			if minibatch_index % 1000 == 0:
				print time.ctime() + ", " + str(minibatch_index) + "/" + str(n_train_batches) + ", " + str(minibatch_avg_cost_sum/(minibatch_index+1))
			# iteration number
			iter = (epoch - 1) * n_train_batches + minibatch_index

			if (iter + 1) % validation_frequency == 0:
				# compute zero-one loss on validation set
				validation_losses = [validate_model(i) for i in xrange(n_dev_batches)]
				this_validation_loss = numpy.mean(validation_losses)

				dev_sum_neg_likelihood = sum([dev_negative_log_likelihood(i) for i in xrange(n_dev_batches)])
				dev_perplexity = math.exp(dev_sum_neg_likelihood / n_dev_batches)

				print(
					'epoch %i, minibatch %i/%i, validation error %f %% , perplexity %f' %
					(
						epoch,
						minibatch_index + 1,
						n_train_batches,
						this_validation_loss * 100.,
						dev_perplexity
					)
				)

				# if we got the best validation score until now
				if this_validation_loss < best_validation_loss:
					#improve patience if loss improvement is good enough
					if (
						this_validation_loss < best_validation_loss *
						improvement_threshold
					):
						patience = max(patience, iter * patience_increase)

					best_validation_loss = this_validation_loss
					best_iter = iter

				# test it on the test set
				test_losses = [test_model(i) for i in xrange(n_test_batches)]
				test_score = numpy.mean(test_losses)

				print(('epoch %i, minibatch %i/%i, test error of '
					   'best model %f %%') %
					  (epoch, minibatch_index + 1, n_train_batches,
					   test_score * 100.))

			if patience <= iter:
				#done_looping = True
				#break
				pass

	end_time = time.clock()
	print(('Optimization complete. Best validation score of %f %% '
		   'obtained at iteration %i, with test performance %f %%') %
		  (best_validation_loss * 100., best_iter + 1, test_score * 100.))
	print >> sys.stderr, ('The code for file ' +
						  os.path.split(__file__)[1] +
						  ' ran for %.2fm' % ((end_time - start_time) / 60.))

if __name__ == '__main__':
	if len(sys.argv) != 4:
		print "USAGE: python " + sys.argv[0] + " train_mmap dev_mmap test_mmap"
		sys.exit()

	train_path = sys.argv[1]
	dev_path = sys.argv[2]
	test_path = sys.argv[3]

	batch_size = 10

	trainset = LMDatasetReader(train_path, batch_size=batch_size)
	devset = LMDatasetReader(dev_path, batch_size=batch_size)
	testset = LMDatasetReader(test_path, batch_size=batch_size)

	#dataset = IRISDatasetReader("iris.txt", batch_size=5)
	# check OMP variable

	test_mlp(trainset, devset, testset)

<commit_msg>rgparse object initialized.<commit_after>#!/usr/bin/env python

from __future__ import division
import os
import sys
import time
import numpy
import theano
import theano.tensor as T
import time
import math
import argparse

from dlm.io.lmDatasetReader import LMDatasetReader
from dlm.io.irisDatasetReader import IRISDatasetReader
#from dlm.models.mlp import MLP
from dlm.models.ltmlp import MLP
import dlm.utils as U

def test_mlp(trainset, devset, testset, learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=100, n_hidden=10):

	n_train_batches = trainset.get_num_batches()
	n_dev_batches = devset.get_num_batches()
	n_test_batches = testset.get_num_batches()

	print '... building the model'

	index = T.lscalar()		# index to a [mini]batch
	x = T.imatrix('x')		# the data is presented as rasterized images
	y = T.ivector('y')		# the labels are presented as 1D vector of
							# [int] labels

	rng = numpy.random.RandomState(1234)

	# construct the MLP class
	classifier = MLP(
		rng=rng,
		input=x,
		vocab_size=trainset.get_vocab_size(),
		emb_dim=10,
		ngram_size=trainset.get_ngram_size(),
		n_hidden=n_hidden,
		n_out=trainset.get_num_classes()
	)

	cost = (
		classifier.negative_log_likelihood(y)
		#+ L1_reg * classifier.L1
		#+ L2_reg * classifier.L2_sqr
	)

	gparams = [T.grad(cost, param) for param in classifier.params]

	updates = [
		(param, param - learning_rate * gparam)
		for param, gparam in zip(classifier.params, gparams)
	]

	#f = theano.function([index], dataset.get_features(index))

	test_model = theano.function(
		inputs=[index],
		outputs=classifier.errors(y),
		givens={
			x: testset.get_x(index),
			y: testset.get_y(index)
		}
	)

	validate_model = theano.function(
		inputs=[index],
		outputs=classifier.errors(y),
		givens={
			x: devset.get_x(index),
			y: devset.get_y(index)
		}
	)

	train_model = theano.function(
		inputs=[index],
		outputs=cost,
		updates=updates,
		givens={
			x: trainset.get_x(index),
			y: trainset.get_y(index)
		}
	)

	dev_negative_log_likelihood = theano.function(
		inputs=[index],
		outputs=classifier.negative_log_likelihood(y),
		givens={
			x: devset.get_x(index),
			y: devset.get_y(index)
		}
	)

	print '... training'

	patience = 10000											# look as this many examples regardless
	patience_increase = 2										# wait this much longer when a new best is found
	improvement_threshold = 0.995								# a relative improvement of this much is considered significant
	validation_frequency = min(n_train_batches, patience / 2)	# go through this many minibatche before
																# checking the network on the validation set;
																# in this case we check every epoch

	best_validation_loss = numpy.inf
	best_iter = 0
	test_score = 0.
	start_time = time.clock()

	epoch = 0
	done_looping = False

	while (epoch < n_epochs) and (not done_looping):
		epoch = epoch + 1
		print "Epoch " + str(epoch)
		minibatch_avg_cost_sum = 0
		for minibatch_index in xrange(n_train_batches):
			minibatch_avg_cost = train_model(minibatch_index)
			minibatch_avg_cost_sum += minibatch_avg_cost
			if minibatch_index % 1000 == 0:
				print time.ctime() + ", " + str(minibatch_index) + "/" + str(n_train_batches) + ", " + str(minibatch_avg_cost_sum/(minibatch_index+1))
			# iteration number
			iter = (epoch - 1) * n_train_batches + minibatch_index

			if (iter + 1) % validation_frequency == 0:
				# compute zero-one loss on validation set
				validation_losses = [validate_model(i) for i in xrange(n_dev_batches)]
				this_validation_loss = numpy.mean(validation_losses)

				dev_sum_neg_likelihood = sum([dev_negative_log_likelihood(i) for i in xrange(n_dev_batches)])
				dev_perplexity = math.exp(dev_sum_neg_likelihood / n_dev_batches)

				print(
					'epoch %i, minibatch %i/%i, validation error %f %% , perplexity %f' %
					(
						epoch,
						minibatch_index + 1,
						n_train_batches,
						this_validation_loss * 100.,
						dev_perplexity
					)
				)

				# if we got the best validation score until now
				if this_validation_loss < best_validation_loss:
					#improve patience if loss improvement is good enough
					if (
						this_validation_loss < best_validation_loss *
						improvement_threshold
					):
						patience = max(patience, iter * patience_increase)

					best_validation_loss = this_validation_loss
					best_iter = iter

				# test it on the test set
				test_losses = [test_model(i) for i in xrange(n_test_batches)]
				test_score = numpy.mean(test_losses)

				print(('epoch %i, minibatch %i/%i, test error of '
					   'best model %f %%') %
					  (epoch, minibatch_index + 1, n_train_batches,
					   test_score * 100.))

			if patience <= iter:
				#done_looping = True
				#break
				pass

	end_time = time.clock()
	print(('Optimization complete. Best validation score of %f %% '
		   'obtained at iteration %i, with test performance %f %%') %
		  (best_validation_loss * 100., best_iter + 1, test_score * 100.))
	print >> sys.stderr, ('The code for file ' +
						  os.path.split(__file__)[1] +
						  ' ran for %.2fm' % ((end_time - start_time) / 60.))

if __name__ == '__main__':
	if len(sys.argv) != 4:
		print "USAGE: python " + sys.argv[0] + " train_mmap dev_mmap test_mmap"
		sys.exit()
	# Parsing arguments
	parser = argparse.ArgumentParser()

	train_path = sys.argv[1]
	dev_path = sys.argv[2]
	test_path = sys.argv[3]

	batch_size = 10

	trainset = LMDatasetReader(train_path, batch_size=batch_size)
	devset = LMDatasetReader(dev_path, batch_size=batch_size)
	testset = LMDatasetReader(test_path, batch_size=batch_size)

	#dataset = IRISDatasetReader("iris.txt", batch_size=5)
	# check OMP variable

	test_mlp(trainset, devset, testset)

<|endoftext|>


<commit_before>#!/usr/bin/env python

import argparse
import dlm.utils as U
import dlm.io.logging as L

###############
## Arguments
#

parser = argparse.ArgumentParser()
parser.add_argument("-tr", "--train-mmap", dest="trainset", required=True, help="The memory-mapped training file")
parser.add_argument("-tu", "--tune-mmap", dest="devset", required=True, help="The memory-mapped development (tune) file")
parser.add_argument("-ts", "--test-mmap", dest="testset", help="The memory-mapped evaluation (test) file")
parser.add_argument("-D", "--device", dest="device", default="gpu", help="The computing device (cpu or gpu). Default: gpu")
parser.add_argument("-E", "--emb-dim", dest="emb_dim", default=50, type=int, help="Word embeddings dimension. Default: 50")
parser.add_argument("-H", "--hidden-units", dest="num_hidden", default="512", help="A comma seperated list for the number of units in each hidden layer. Default: 512")
parser.add_argument("-A", "--activation", dest="activation_name", default="tanh", help="Activation function (tanh|hardtanh|sigmoid|fastsigmoid|hardsigmoid|softplus|relu|cappedrelu). Default: tanh")
parser.add_argument("-a", "--training-algorithm", dest="algorithm", default="sgd", help="The training algorithm (only sgd is supported for now). Default: sgd")
parser.add_argument("-b", "--batch-size", dest="batchsize", default=128, type=int, help="Minibatch size for training. Default: 128")
parser.add_argument("-l", "--learning-rate", dest="learning_rate", default=0.01, type=float, help="Learning rate. Default: 0.01")
parser.add_argument("-d", "--learning-rate-decay", dest="learning_rate_decay", default=0, type=float, help="Learning rate decay (e.g. 0.995) (TO DO). Default: 0")
parser.add_argument("-m", "--momentum", dest="momentum", default=0, type=float, help="Momentum (TO DO). Default: 0")
parser.add_argument("-lf","--loss-function", dest="loss_function", default="log", help="Loss function (log|nce). Default: log")
parser.add_argument("-ns","--noise-samples", dest="num_noise_samples", default=100 ,type=int, help="Number of noise samples for noise contrastive estimation. Default:100")
parser.add_argument("-e", "--num-epochs", dest="num_epochs", default=50, type=int, help="Number of iterations (epochs). Default: 50")
parser.add_argument("-c", "--self-norm-coef", dest="alpha", default=0, type=float, help="Self normalization coefficient (alpha). Default: 0")
parser.add_argument("-L1", "--L1-regularizer", dest="L1_reg", default=0, type=float, help="L1 regularization coefficient. Default: 0")
parser.add_argument("-L2", "--L2-regularizer", dest="L2_reg", default=0, type=float, help="L2 regularization coefficient. Default: 0")
parser.add_argument("-dir", "--directory", dest="out_dir", help="The output directory for log file, model, etc.")
parser.add_argument("--threads", dest="threads", default=12, help="Number of threads when device is CPU (TO DO). Default: 12")
args = parser.parse_args()

if args.out_dir is None:
	args.out_dir = 'primelm-' + U.curr_time()
U.mkdir_p(args.out_dir)

L.set_file_path(args.out_dir + "/log.txt")

U.print_args(args)
U.set_theano_device(args.device)

import dlm.trainer
from dlm.io.mmapReader import MemMapReader
from dlm.criterions.likelihood import NegLogLikelihood
from dlm.models.ltmlp import MLP

#########################
## Loading datasets
#

trainset = MemMapReader(args.trainset, batch_size=args.batchsize)
devset = MemMapReader(args.devset)
testset = None
if args.testset:
	testset = MemMapReader(args.testset)

#########################
## Creating model
#

L.info('Building the model')

args.vocab_size = trainset.get_vocab_size()
args.ngram_size = trainset.get_ngram_size()
args.num_classes = trainset.get_num_classes()

classifier = MLP(args)

#########################
## Training criterion
#

criterion = NegLogLikelihood(classifier, args)

#########################
## Training
#

dlm.trainer.train(classifier, criterion, args, trainset, devset, testset)

<commit_msg>NCE parameter added in training script.<commit_after>#!/usr/bin/env python

import argparse
import dlm.utils as U
import dlm.io.logging as L

###############
## Arguments
#

parser = argparse.ArgumentParser()
parser.add_argument("-tr", "--train-mmap", dest="trainset", required=True, help="The memory-mapped training file")
parser.add_argument("-tu", "--tune-mmap", dest="devset", required=True, help="The memory-mapped development (tune) file")
parser.add_argument("-ts", "--test-mmap", dest="testset", help="The memory-mapped evaluation (test) file")
parser.add_argument("-d", "--device", dest="device", default="gpu", help="The computing device (cpu or gpu). Default: gpu")
parser.add_argument("-E", "--emb-dim", dest="emb_dim", default=50, type=int, help="Word embeddings dimension. Default: 50")
parser.add_argument("-H", "--hidden-units", dest="num_hidden", default="512", help="A comma seperated list for the number of units in each hidden layer. Default: 512")
parser.add_argument("-A", "--activation", dest="activation_name", default="tanh", help="Activation function (tanh|hardtanh|sigmoid|fastsigmoid|hardsigmoid|softplus|relu|cappedrelu). Default: tanh")
parser.add_argument("-a", "--training-algorithm", dest="algorithm", default="sgd", help="The training algorithm (only sgd is supported for now). Default: sgd")
parser.add_argument("-b", "--batch-size", dest="batchsize", default=128, type=int, help="Minibatch size for training. Default: 128")
parser.add_argument("-l", "--learning-rate", dest="learning_rate", default=0.01, type=float, help="Learning rate. Default: 0.01")
parser.add_argument("-D", "--learning-rate-decay", dest="learning_rate_decay", default=0, type=float, help="Learning rate decay (e.g. 0.995) (TO DO). Default: 0")
parser.add_argument("-m", "--momentum", dest="momentum", default=0, type=float, help="Momentum (TO DO). Default: 0")
parser.add_argument("-lf","--loss-function", dest="loss_function", default="log", help="Loss function (log|nce). Default: log")
parser.add_argument("-ns","--noise-samples", dest="num_noise_samples", default=100 ,type=int, help="Number of noise samples for noise contrastive estimation. Default:100")
parser.add_argument("-e", "--num-epochs", dest="num_epochs", default=50, type=int, help="Number of iterations (epochs). Default: 50")
parser.add_argument("-c", "--self-norm-coef", dest="alpha", default=0, type=float, help="Self normalization coefficient (alpha). Default: 0")
parser.add_argument("-L1", "--L1-regularizer", dest="L1_reg", default=0, type=float, help="L1 regularization coefficient. Default: 0")
parser.add_argument("-L2", "--L2-regularizer", dest="L2_reg", default=0, type=float, help="L2 regularization coefficient. Default: 0")
parser.add_argument("-dir", "--directory", dest="out_dir", help="The output directory for log file, model, etc.")
parser.add_argument("--threads", dest="threads", default=12, help="Number of threads when device is CPU (TO DO). Default: 12")
args = parser.parse_args()

if args.out_dir is None:
	args.out_dir = 'primelm-' + U.curr_time()
U.mkdir_p(args.out_dir)

L.set_file_path(args.out_dir + "/log.txt")

U.print_args(args)
U.set_theano_device(args.device)

import dlm.trainer
from dlm.io.mmapReader import MemMapReader
from dlm.models.ltmlp import MLP

#########################
## Loading datasets
#

trainset = MemMapReader(args.trainset, batch_size=args.batchsize)
devset = MemMapReader(args.devset)
testset = None
if args.testset:
	testset = MemMapReader(args.testset)

#########################
## Creating model
#

L.info('Building the model')

args.vocab_size = trainset.get_vocab_size()
args.ngram_size = trainset.get_ngram_size()
args.num_classes = trainset.get_num_classes()

classifier = MLP(args)

#########################
## Training criterion
#
if args.loss_function == "log":
	from dlm.criterions.likelihood import NegLogLikelihood
	criterion = NegLogLikelihood(classifier, args)
elif args.loss_function == "nce":
	from dlm.criterions.nce import NCELikelihood
	noise_dist = trainset.get_unigram_model()
	criterion = NCELikelihood(classifier, args, noise_dist)
else:
	L.error('Invalid loss function \'' + args.loss_function + '\'')

#########################
## Training
#

dlm.trainer.train(classifier, criterion, args, trainset, devset, testset)

<|endoftext|>


<commit_before>#
# Copyright (c) 2004 Specifix, Inc.
# All rights reserved
#
import versions

# this is the repository's idea of a package
class Package:
    def getName(self):
        return self.name

    def getVersion(self):
        return self.version

    def changeVersion(self, version):
        self.version = version

    def addFile(self, fileId, path, version):
	self.files[path] = (fileId, path, version)
	self.idMap[fileId] = (path, version)

    # fileId is the only thing that must be here; the other fields could
    # be "-"
    def updateFile(self, fileId, path, version):
	(origPath, origVersion) = self.idMap[fileId]

	if not path:
	    path = origPath
	else:
	    del self.files[path]

	if not version:
	    version = origVersion

	self.files[path] = (fileId, path, version)
	self.idMap[fileId] = (path, version)

    def removeFile(self, fileId):
	path = self.idMap[fileId][0]
	del self.files[path]
	del self.idMap[fileId]

    def fileList(self):
	l = []
        paths = self.files.keys()
        paths.sort()
        for path in paths:
	    l.append(self.files[path])

	return l

    def getFile(self, fileId):
	return self.idMap[fileId]

    def formatString(self):
	rc = ""
	for (fileId, path, version) in self.files.values():
	    rc += ("%s %s %s\n" % (fileId, path, version.asString()))
	return rc

    # returns a dictionary mapping a fileId to a (path, version, pkgName) tuple
    def applyChangeSet(self, pkgCS):
	fileMap = {}

	for (fileId, path, fileVersion) in pkgCS.getNewFileList():
	    self.addFile(fileId, path, fileVersion)
	    fileMap[fileId] = self.idMap[fileId] + (self.name, )

	for (fileId, path, fileVersion) in pkgCS.getChangedFileList():
	    self.updateFile(fileId, path, fileVersion)
	    # look up the path/version in self.idMap as the ones here
	    # could be None
	    fileMap[fileId] = self.idMap[fileId] + (self.name, )

	for fileId in pkgCS.getOldFileList():
	    self.removeFile(fileId)

	return fileMap

    def diff(self, them, themVersion, ourVersion):
	# find all of the file ids which have been added, removed, and
	# stayed the same
	if them:
	    themMap = them.idMap
	    chgSet = PackageChangeSet(self.name, themVersion, ourVersion)
	else:
	    themMap = {}
	    chgSet = PackageChangeSet(self.name, None, ourVersion)

	removedIds = []
	addedIds = []
	sameIds = {}
	filesNeeded = []

	allIds = self.idMap.keys() + themMap.keys()
	for id in allIds:
	    inSelf = self.idMap.has_key(id)
	    inThem = themMap.has_key(id)
	    if inSelf and inThem:
		sameIds[id] = None
	    elif inSelf:
		addedIds.append(id)
	    else:
		removedIds.append(id)

	for id in removedIds:
	    chgSet.oldFile(id)

	for id in addedIds:
	    (selfPath, selfVersion) = self.idMap[id]
	    filesNeeded.append((id, None, selfVersion))
	    chgSet.newFile(id, selfPath, selfVersion)

	for id in sameIds.keys():
	    (selfPath, selfVersion) = self.idMap[id]
	    (themPath, themVersion) = themMap[id]

	    newPath = None
	    newVersion = None

	    if selfPath != themPath:
		newPath = selfPath

	    if not selfVersion.equal(themVersion):
		newVersion = selfVersion
		filesNeeded.append((id, themVersion, selfVersion))

	    if newPath or newVersion:
		chgSet.changedFile(id, newPath, newVersion)

	return (chgSet, filesNeeded)

    def __init__(self, name, version):
	self.files = {}
	self.idMap = {}
	self.name = name
	self.version = version

class PackageChangeSet:

    def newFile(self, fileId, path, version):
	self.newFiles.append((fileId, path, version))

    def getNewFileList(self):
	return self.newFiles

    def oldFile(self, fileId):
	self.oldFiles.append(fileId)

    def getOldFileList(self):
	return self.oldFiles

    def getName(self):
	return self.name

    def getOldVersion(self):
	return self.oldVersion

    def getNewVersion(self):
	return self.newVersion

    # path and/or version can be None
    def changedFile(self, fileId, path, version):
	self.changedFiles.append((fileId, path, version))

    def getChangedFileList(self):
	return self.changedFiles

    def parse(self, line):
	action = line[0]

	if action == "+" or action == "~":
	    (fileId, path, version) = line[1:].split()

	    if version == "-":
		version = None
	    else:
		version = versions.ThawVersion(version)

	    if path == "-":
		path = None

	    if action == "+":
		self.newFile(fileId, path, version)
	    else:
		self.changedFile(fileId, path, version)
	elif action == "-":
	    self.oldFile(line[1:])

    def formatToFile(self, changeSet, cfg, f):
	f.write("%s " % self.name)
	if self.oldVersion:
	    f.write("from %s to " % self.oldVersion.asString(cfg.defaultbranch))
	else:
	    f.write("abstract ")
	f.write("%s\n" % self.newVersion.asString(cfg.defaultbranch))

	for (fileId, path, version) in self.newFiles:
	    f.write("\tadded %s (%s(.*)%s)\n" % (path, fileId[:6], fileId[-6:]))
	for (fileId, path, version) in self.changedFiles:
	    f.write("\tchanged %s\n" % path)
	    change = changeSet.getFileChange(fileId)
	    print "\t\t%s" % change
	for fileId in self.oldFiles:
	    f.write("\tremoved %s(.*)%s\n" % (fileId[:6], fileId[-6:]))

    def freeze(self):
	rc = ""

	for id in self.getOldFileList():
	    rc += "-%s\n" % id

	for (id, path, version) in self.getNewFileList():
	    rc += "+%s %s %s\n" % (id, path, version.freeze())

	for (id, path, version) in self.getChangedFileList():
	    rc += "~%s " % id
	    if path:
		rc += path
	    else:
		rc += "-"

	    if version:
		rc += " " + version.freeze() + "\n"
	    else:
		rc += " -\n"

	if self.oldVersion:
	    oldVerStr = self.oldVersion.freeze()
	else:
	    oldVerStr = "(none)"

	hdr = "SRS PKG CHANGESET %s %s %s %d\n" % \
		  (self.name, oldVerStr, self.newVersion.freeze(),
		   rc.count("\n"))
	return hdr + rc

    def __init__(self, name, oldVersion, newVersion):
	self.name = name
	self.oldVersion = oldVersion
	self.newVersion = newVersion
	self.newFiles = []
	self.oldFiles = []
	self.changedFiles = []

class PackageFromFile(Package):

    def read(self, dataFile):
	for line in dataFile.readLines():
	    (fileId, path, version) = line.split()
	    version = versions.VersionFromString(version)
	    self.addFile(fileId, path, version)

    def __init__(self, name, dataFile, version):
	Package.__init__(self, name, version)
	self.read(dataFile)

def stripNamespace(namespace, pkgName):
    if pkgName.startswith(namespace + ":"):
	return pkgName[len(namespace) + 1:]
    return pkgName
<commit_msg>store frozen versions in change set<commit_after>#
# Copyright (c) 2004 Specifix, Inc.
# All rights reserved
#
import versions

# this is the repository's idea of a package
class Package:
    def getName(self):
        return self.name

    def getVersion(self):
        return self.version

    def changeVersion(self, version):
        self.version = version

    def addFile(self, fileId, path, version):
	self.files[path] = (fileId, path, version)
	self.idMap[fileId] = (path, version)

    # fileId is the only thing that must be here; the other fields could
    # be "-"
    def updateFile(self, fileId, path, version):
	(origPath, origVersion) = self.idMap[fileId]

	if not path:
	    path = origPath
	else:
	    del self.files[path]

	if not version:
	    version = origVersion

	self.files[path] = (fileId, path, version)
	self.idMap[fileId] = (path, version)

    def removeFile(self, fileId):
	path = self.idMap[fileId][0]
	del self.files[path]
	del self.idMap[fileId]

    def fileList(self):
	l = []
        paths = self.files.keys()
        paths.sort()
        for path in paths:
	    l.append(self.files[path])

	return l

    def getFile(self, fileId):
	return self.idMap[fileId]

    def formatString(self):
	rc = ""
	for (fileId, path, version) in self.files.values():
	    rc += ("%s %s %s\n" % (fileId, path, version.freeze()))
	return rc

    # returns a dictionary mapping a fileId to a (path, version, pkgName) tuple
    def applyChangeSet(self, pkgCS):
	fileMap = {}

	for (fileId, path, fileVersion) in pkgCS.getNewFileList():
	    self.addFile(fileId, path, fileVersion)
	    fileMap[fileId] = self.idMap[fileId] + (self.name, )

	for (fileId, path, fileVersion) in pkgCS.getChangedFileList():
	    self.updateFile(fileId, path, fileVersion)
	    # look up the path/version in self.idMap as the ones here
	    # could be None
	    fileMap[fileId] = self.idMap[fileId] + (self.name, )

	for fileId in pkgCS.getOldFileList():
	    self.removeFile(fileId)

	return fileMap

    def diff(self, them, themVersion, ourVersion):
	# find all of the file ids which have been added, removed, and
	# stayed the same
	if them:
	    themMap = them.idMap
	    chgSet = PackageChangeSet(self.name, themVersion, ourVersion)
	else:
	    themMap = {}
	    chgSet = PackageChangeSet(self.name, None, ourVersion)

	removedIds = []
	addedIds = []
	sameIds = {}
	filesNeeded = []

	allIds = self.idMap.keys() + themMap.keys()
	for id in allIds:
	    inSelf = self.idMap.has_key(id)
	    inThem = themMap.has_key(id)
	    if inSelf and inThem:
		sameIds[id] = None
	    elif inSelf:
		addedIds.append(id)
	    else:
		removedIds.append(id)

	for id in removedIds:
	    chgSet.oldFile(id)

	for id in addedIds:
	    (selfPath, selfVersion) = self.idMap[id]
	    filesNeeded.append((id, None, selfVersion))
	    chgSet.newFile(id, selfPath, selfVersion)

	for id in sameIds.keys():
	    (selfPath, selfVersion) = self.idMap[id]
	    (themPath, themVersion) = themMap[id]

	    newPath = None
	    newVersion = None

	    if selfPath != themPath:
		newPath = selfPath

	    if not selfVersion.equal(themVersion):
		newVersion = selfVersion
		filesNeeded.append((id, themVersion, selfVersion))

	    if newPath or newVersion:
		chgSet.changedFile(id, newPath, newVersion)

	return (chgSet, filesNeeded)

    def __init__(self, name, version):
	self.files = {}
	self.idMap = {}
	self.name = name
	self.version = version

class PackageChangeSet:

    def newFile(self, fileId, path, version):
	self.newFiles.append((fileId, path, version))

    def getNewFileList(self):
	return self.newFiles

    def oldFile(self, fileId):
	self.oldFiles.append(fileId)

    def getOldFileList(self):
	return self.oldFiles

    def getName(self):
	return self.name

    def getOldVersion(self):
	return self.oldVersion

    def getNewVersion(self):
	return self.newVersion

    # path and/or version can be None
    def changedFile(self, fileId, path, version):
	self.changedFiles.append((fileId, path, version))

    def getChangedFileList(self):
	return self.changedFiles

    def parse(self, line):
	action = line[0]

	if action == "+" or action == "~":
	    (fileId, path, version) = line[1:].split()

	    if version == "-":
		version = None
	    else:
		version = versions.ThawVersion(version)

	    if path == "-":
		path = None

	    if action == "+":
		self.newFile(fileId, path, version)
	    else:
		self.changedFile(fileId, path, version)
	elif action == "-":
	    self.oldFile(line[1:])

    def formatToFile(self, changeSet, cfg, f):
	f.write("%s " % self.name)
	if self.oldVersion:
	    f.write("from %s to " % self.oldVersion.asString(cfg.defaultbranch))
	else:
	    f.write("abstract ")
	f.write("%s\n" % self.newVersion.asString(cfg.defaultbranch))

	for (fileId, path, version) in self.newFiles:
	    f.write("\tadded %s (%s(.*)%s)\n" % (path, fileId[:6], fileId[-6:]))
	for (fileId, path, version) in self.changedFiles:
	    f.write("\tchanged %s\n" % fileId)
	    change = changeSet.getFileChange(fileId)
	    print "\t\t%s" % change
	for fileId in self.oldFiles:
	    f.write("\tremoved %s(.*)%s\n" % (fileId[:6], fileId[-6:]))

    def freeze(self):
	rc = ""

	for id in self.getOldFileList():
	    rc += "-%s\n" % id

	for (id, path, version) in self.getNewFileList():
	    rc += "+%s %s %s\n" % (id, path, version.freeze())

	for (id, path, version) in self.getChangedFileList():
	    rc += "~%s " % id
	    if path:
		rc += path
	    else:
		rc += "-"

	    if version:
		rc += " " + version.freeze() + "\n"
	    else:
		rc += " -\n"

	if self.oldVersion:
	    oldVerStr = self.oldVersion.freeze()
	else:
	    oldVerStr = "(none)"

	hdr = "SRS PKG CHANGESET %s %s %s %d\n" % \
		  (self.name, oldVerStr, self.newVersion.freeze(),
		   rc.count("\n"))
	return hdr + rc

    def __init__(self, name, oldVersion, newVersion):
	self.name = name
	self.oldVersion = oldVersion
	self.newVersion = newVersion
	self.newFiles = []
	self.oldFiles = []
	self.changedFiles = []

class PackageFromFile(Package):

    def read(self, dataFile):
	for line in dataFile.readLines():
	    (fileId, path, version) = line.split()
	    version = versions.ThawVersion(version)
	    self.addFile(fileId, path, version)

    def __init__(self, name, dataFile, version):
	Package.__init__(self, name, version)
	self.read(dataFile)

def stripNamespace(namespace, pkgName):
    if pkgName.startswith(namespace + ":"):
	return pkgName[len(namespace) + 1:]
    return pkgName
<|endoftext|>


<commit_before>import constituentretrofit_fixed_word2vec_native as consfit
import sys
import getopt
import numpy as np
import re

# take a POS tagged word2vec format file and turn it into a txt file
help_message = '''
$ python untag.py -v <vectorsFile> [-f filterFile] [-o outputFile] [-h]
-v or --vectors to specify path to the word2vectors input file
-f or --filter to optionally set path to filter word vectors txt file
-o or --output to optionally set path to output untagged word vectors txt file
-h or --help (this message is displayed)
'''
phraseSeparator = '|'
tagSeparator = '_'

class Usage(Exception):
    def __init__(self, msg):
        self.msg = msg

# Read command line arguments
def readCommandLineInput(argv):
    try:
        try:
            # specify the possible option switches
            opts, _ = getopt.getopt(argv[1:], "hv:f:o:", ["help", "vectors=", "filter=", "output="])
        except getopt.error, msg:
            raise Usage(msg)

        # default values
        vectorsFile = None
        outputFile = None
        filterFile = None

        setOutput = False
        # option processing
        for option, value in opts:
            if option in ("-h", "--help"):
                raise Usage(help_message)
            elif option in ("-v", "--vectors"):
                vectorsFile = value
            elif option in ("-f", "--filter"):
                filterFile = value
            elif option in ("-o", "--output"):
                outputFile = value
                setOutput = True
            else:
                raise Usage(help_message)

        if (vectorsFile is None):
            raise Usage(help_message)
        else:
            if not setOutput:
                outputFile = vectorsFile + '.txt'
            return (vectorsFile, outputFile, filterFile)

    except Usage, err:
        print str(err.msg)
        return 2

def readInFilterFile(filterFile):
    filterSet = set()
    with open(filterFile, 'r') as f:
        for line in f:
            filterSet.add(line.rstrip('\n'))
    return filterSet

def untag(vocabs):
    untagVocab = {}
    for tagVocab in vocabs:
        if phraseSeparator not in tagVocab:
            vocab = tagVocab.split('_')[0]
            if vocab in untagVocab:
                untagVocab[vocab].append(tagVocab)
            else:
                untagVocab[vocab] = [tagVocab]
    return untagVocab

def untagWithVectors(untagDict, vocab, vectors, filterSet=None):
    untagVectors = {}
    for untagVocab in untagDict:
        if filterSet is None or untagVocab in filterSet:
            if len(untagDict[untagVocab]) > 1:
                def getFreqOfVocab(x):
                    vocab[x]
                mostFreqVocab = min(untagDict[untagVocab], key=getFreqOfVocab)
                untagVectors[untagVocab] = vectors[mostFreqVocab]
            else:
                untagVectors[untagVocab] = vectors[untagDict[untagVocab][0]]
    return untagVectors

def writeWordVectors(vectors, vectorDim, outputFile):
    sys.stderr.write('Writing untagged vec to file\n')
    with open(outputFile, 'w') as output:
        for vocab in vectors:
            output.write(vocab)
            npVec = vectors[vocab]
            vecStr = np.array2string(npVec, max_line_width='infty', precision=6, suppress=True)
            vecStr = vecStr.replace('[', ' ')
            vecStr = re.sub(r' +', ' ', vecStr)
            output.write(vecStr[:-1])
            output.write('\n')

if __name__ == '__main__':
    commandParse = readCommandLineInput(sys.argv)
    if commandParse == 2:
        sys.exit(2)

    vocab, vectors, vectorDim = consfit.readWordVectors(commandParse[0])
    # vocab is {word: frequency rank}
    # vectors is {word: vector}
    sys.stderr.write('vocab length is '+str(len(vocab.keys()))+'\n')
    filterSet = None
    if commandParse[2]:
        filterSet = readInFilterFile(commandParse[2])
    untagDict = untag(vocab)
    # untag and write to output file
    untagVectors = untagWithVectors(untagDict, vocab, vectors, filterSet)
    writeWordVectors(untagVectors, vectorDim, commandParse[1])

    sys.stderr.write('All done!\n')
<commit_msg>output is filtered<commit_after>import constituentretrofit_fixed_word2vec_native as consfit
import sys
import getopt
import numpy as np
import re

# take a POS tagged word2vec format file and turn it into a txt file
help_message = '''
$ python untag.py -v <vectorsFile> [-f filterFile] [-o outputFile] [-h]
-v or --vectors to specify path to the word2vectors input file
-f or --filter to optionally set path to filter word vectors txt file
-o or --output to optionally set path to output untagged word vectors txt file
-h or --help (this message is displayed)
'''

phraseSeparator = '|'
tagSeparator = '_'

class Usage(Exception):
    def __init__(self, msg):
        self.msg = msg

# Read command line arguments
def readCommandLineInput(argv):
    try:
        try:
            # specify the possible option switches
            opts, _ = getopt.getopt(argv[1:], "hv:f:o:", ["help", "vectors=", "filter=", "output="])
        except getopt.error, msg:
            raise Usage(msg)

        # default values
        vectorsFile = None
        outputFile = None
        filterFile = None

        setOutput = False
        # option processing
        for option, value in opts:
            if option in ("-h", "--help"):
                raise Usage(help_message)
            elif option in ("-v", "--vectors"):
                vectorsFile = value
            elif option in ("-f", "--filter"):
                filterFile = value
            elif option in ("-o", "--output"):
                outputFile = value
                setOutput = True
            else:
                raise Usage(help_message)

        if (vectorsFile is None):
            raise Usage(help_message)
        else:
            if not setOutput:
                outputFile = vectorsFile + '.txt'
            return (vectorsFile, outputFile, filterFile)

    except Usage, err:
        print str(err.msg)
        return 2

def readInFilterFile(filterFile):
    filterSet = set()
    with open(filterFile, 'r') as f:
        for line in f:
            filterSet.add(line.rstrip('\n'))
    return filterSet

def untag(vocabs):
    untagVocab = {}
    for tagVocab in vocabs:
        if phraseSeparator not in tagVocab:
            vocab = tagVocab.split('_')[0]
            if vocab in untagVocab:
                untagVocab[vocab].append(tagVocab)
            else:
                untagVocab[vocab] = [tagVocab]
    return untagVocab

def untagWithVectors(untagDict, vocab, vectors, filterSet=None):
    untagVectors = {}
    for untagVocab in untagDict:
        if filterSet is None or untagVocab in filterSet:
            if len(untagDict[untagVocab]) > 1:
                def getFreqOfVocab(x):
                    vocab[x]
                mostFreqVocab = min(untagDict[untagVocab], key=getFreqOfVocab)
                untagVectors[untagVocab] = vectors[mostFreqVocab]
            else:
                untagVectors[untagVocab] = vectors[untagDict[untagVocab][0]]
    return untagVectors

def writeWordVectors(vectors, vectorDim, outputFile):
    sys.stderr.write('Writing untagged vec to file\n')
    with open(outputFile, 'w') as output:
        for vocab in vectors:
            output.write(vocab)
            npVec = vectors[vocab]
            vecStr = np.array2string(npVec, max_line_width='infty', precision=6, suppress=True)
            vecStr = vecStr.replace('[', ' ')
            vecStr = re.sub(r' +', ' ', vecStr)
            output.write(vecStr[:-1])
            output.write('\n')

if __name__ == '__main__':
    commandParse = readCommandLineInput(sys.argv)
    if commandParse == 2:
        sys.exit(2)

    vocab, vectors, vectorDim = consfit.readWordVectors(commandParse[0])
    # vocab is {word: frequency rank}
    # vectors is {word: vector}
    sys.stderr.write('vocab length is '+str(len(vocab.keys()))+'\n')
    filterSet = None
    if commandParse[2]:
        filterSet = readInFilterFile(commandParse[2])
    untagDict = untag(vocab)
    # untag and write to output file
    untagVectors = untagWithVectors(untagDict, vocab, vectors, filterSet)
    writeWordVectors(untagVectors, vectorDim, commandParse[1])

    sys.stderr.write('All done!\n')
<|endoftext|>


<commit_before><commit_msg>add empty API<commit_after>#-------------------------------------------------------------------------------
def checkUser( username, passwd ):
    return False

#-------------------------------------------------------------------------------
def checkIfUserAvailable( username ):
    return False

#-------------------------------------------------------------------------------
def getUserEmail( username ):
    return None

#-------------------------------------------------------------------------------
<|endoftext|>


<commit_before># utile.py - Collection of useful functions
# Copyright (C) 2013 Marwan Alsabbagh
# license: BSD, see LICENSE for more details.

from __future__ import print_function
import time
import re
import os
import os.path
import sys
import string
import random
from itertools import chain
from timeit import default_timer as timer
from functools import wraps
from shutil import rmtree
from hashlib import sha256
from tempfile import mkdtemp
from contextlib import contextmanager
from fcntl import flock, LOCK_EX, LOCK_NB
from datetime import timedelta
from textwrap import dedent
from operator import itemgetter
from math import log10
from subprocess import check_call, Popen, PIPE
from inspect import getargspec
from argparse import (
    ArgumentParser, ArgumentDefaultsHelpFormatter, RawDescriptionHelpFormatter
)

__version__ = '0.4.dev'
PY3 = sys.version_info[0] == 3
string_types = str if PY3 else basestring
_builtin_print = print
alpha_numeric = string.ascii_letters + string.digits

def random_text(length, characters=alpha_numeric):
    return ''.join(random.choice(characters) for i in range(length))

# print(*objects, sep=' ', end='\n', file=sys.stdout, flush=False)
def print(*args, **kwargs):
    clean = {k: v for k, v in kwargs.items() if k != 'flush'}
    _builtin_print(*args, **clean)
    file = kwargs.get('file', sys.stdout)
    if kwargs.get('flush'):
        file.flush()

def force_print(*args, **kwargs):
    print(*args, file=sys.__stdout__, **kwargs)

def countdown(length, msg='Countdown', delay=0.1):
    start = timer()
    maxlen = 0
    template = '{msg}: {remaining:.%df}' % -log10(delay)
    while timer() - start < length:
        remaining = (start + length) - timer()
        text = template.format(msg=msg, remaining=remaining)
        maxlen = max(maxlen, len(text))
        print(text.ljust(maxlen), end='\r', flush=True)
        time.sleep(delay)
    print('{msg}: done'.format(msg=msg).ljust(maxlen))

def resolve(obj):
    if callable(obj):
        return obj()
    item, module = None, []
    for i in obj.split('.'):
        module.append(i)
        try:
            item = getattr(item, i)
        except AttributeError:
            top_module = __import__('.'.join(module))
            if item:
                item = getattr(item, i)
            else:
                item = top_module
    return item

class LazyResolve(object):
    def __init__(self, lookup=None):
        self.lookup = lookup or dict()

    def __getattr__(self, name):
        obj = resolve(self.lookup.get(name, name))
        setattr(self, name, obj)
        return obj

def safe_import(name, default=None):
    try:
        return resolve(name)
    except ImportError:
        return default

def bunch_or_dict(*args, **kwargs):
    _class = safe_import('bunch.Bunch', dict)
    return _class(*args, **kwargs)

def dir_dict(obj, default=None, only_public=True):
    names = dir(obj)
    if only_public:
        names = [i for i in names if not i.startswith('_')]
    return bunch_or_dict((i, getattr(obj, i, default)) for i in names)

def requires_package(name, pypi_name=None):
    pypi_name = pypi_name or name.split('.')[0]
    msg = 'Could not import %r. Install it by running:\npip install %s'
    enforce(safe_import(name), msg % (name, pypi_name))
    return safe_import(name)

def requires_commands(commands):
    missing = [i for i in commands.split() if not which(i)]
    enforce(not missing, '%r command(s) not found' % ' '.join(missing))

def pretty_xml(xml):
    etree = requires_package('lxml.etree')
    root = etree.fromstring(xml, etree.XMLParser(remove_blank_text=True))
    return etree.tostring(root, pretty_print=True, encoding='unicode')

def element_to_dict(elem, return_tuple=False):
    children = bunch_or_dict(element_to_dict(i, True) for i in elem)
    if return_tuple:
        return elem.tag, children or elem.text
    else:
        return children

def xml_to_dict(xml, *args, **kwargs):
    etree = requires_package('lxml.etree')
    root = etree.fromstring(xml, etree.XMLParser(*args, **kwargs))
    return element_to_dict(root)

def slicer_by_size(sizes):
    s = [slice(sum(sizes[0:i]), sum(sizes[0:i+1])) for i in range(len(sizes))]
    return itemgetter(*s)

def parse_table(text, yaml=False):
    if yaml:
        decoder = requires_package('yaml', 'PyYAML').load
    else:
        decoder = lambda x: x
    lines = dedent(text).strip().splitlines()
    slicer = slicer_by_size([len(i) for i in re.findall('=+ *', lines[0])])
    keys = [i.strip() for i in slicer(lines[1])]
    rows = []
    for line in lines[3:-1]:
        values = [decoder(i.strip()) for i in slicer(line)]
        rows.append(bunch_or_dict(zip(keys, values)))
    return rows

def git_version(version):
    if 'dev' not in version:
        return version
    describe = ''
    if which('git'):
        process = Popen(['git', 'describe'], stdout=PIPE, stderr=PIPE)
        describe, _ = process.communicate()
    if describe:
        return version + describe.decode('utf8').split('-')[1]
    elif os.path.exists('PKG-INFO'):
        info = open('PKG-INFO').read()
        return re.findall(r'^Version: (.*)$', info, re.MULTILINE)[0]
    else:
        return version

def save_args(f):

    @wraps(f)
    def wrapper(self, *args, **kwargs):
        names = getargspec(f).args[1:]
        defaults = zip(reversed(names), reversed(getargspec(f).defaults))
        positional = zip(names, args)
        keyword = kwargs.items()
        for k, v in chain(defaults, positional, keyword):
            setattr(self, k, v)
        return f(self, *args, **kwargs)
    return wrapper

def flatten(data):
    return list(chain.from_iterable(data))

def process_name(pid=None, ignore_errors=False):
    pid = pid or os.getpid()
    try:
        with open('/proc/%s/cmdline' % pid) as f:
            return f.read().strip('\x00').split('\x00')
    except IOError:
        if ignore_errors:
            return []
        else:
            raise

def get_pid_list():
    return sorted(int(i) for i in os.listdir('/proc') if i.isdigit())

def mac_address(interface='eth0'):
    output, _ = Popen(['ifconfig', interface], stdout=PIPE).communicate()
    return re.search(r'HWaddr ([\w:]+)', output.decode('utf8')).group(1)

@contextmanager
def TemporaryDirectory(suffix='', prefix='tmp', dir=None):
    path = mkdtemp(suffix, prefix, dir)
    try:
        yield path
    finally:
        rmtree(path)

@contextmanager
def file_lock(path):
    try:
        f = open(path, 'w')
        flock(f, LOCK_EX | LOCK_NB)
    except IOError:
        f.close()
        raise IOError('Could not lock %r' % path)
    try:
        yield
    finally:
        f.close()
        os.remove(path)

def _cipher(key):
    AES = requires_package('Crypto.Cipher.AES', 'pycrypto')
    return AES.new(sha256(key).digest(), AES.MODE_CFB, '\x00' * AES.block_size)

def encrypt(key, data):
    return _cipher(key).encrypt(data)

def decrypt(key, data):
    return _cipher(key).decrypt(data)

def raises(exceptions, func, *args, **kwargs):
    try:
        func(*args, **kwargs)
    except exceptions:
        return True
    return False

class EnforcementError(Exception):
    pass

def enforce(rule, msg, exception=EnforcementError):
    if not rule:
        raise exception(msg)

def enforce_false(rule, msg, exception=EnforcementError):
    if rule:
        raise exception(msg)

def enforce_clean_exit(func):
    @wraps(func)
    def wrapper(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except EnforcementError as err:
            sys.exit('ERROR: %s' % err)
    return wrapper

def which(cmd):
    os_paths = os.environ['PATH'].split(os.pathsep)
    cmd_paths = [os.path.join(i, cmd) for i in os_paths]
    return [i for i in cmd_paths if os.path.exists(i)]

def shell(cmd=None, msg=None, caller=None, strict=False, verbose=False,
          **kwargs):
    caller = caller or check_call
    msg = msg if msg else cmd
    kwargs.setdefault('shell', True)
    if kwargs['shell']:
        kwargs.setdefault('executable', '/bin/bash')
    if strict or verbose:
        if not kwargs['shell'] or not isinstance(cmd, string_types):
            raise ValueError('strict or verbose can only be used when '
                             'shell=True and cmd is a string')
    set_options = ''
    if strict:
        set_options += 'e'
    if verbose:
        set_options += 'v'
    if set_options:
        cmd = 'set -{}\n{}'.format(set_options, cmd)
    print(' {0} '.format(msg).center(60, '-'))
    start = timer()
    returncode = caller(cmd, **kwargs)
    print('duration: %s' % timedelta(seconds=timer() - start))
    return returncode

def shell_quote(s):
    return "'" + s.replace("'", "'\"'\"'") + "'"

class TimeoutError(Exception):
    pass

def _wait_base(target, timeout, delay, callable, *args, **kwargs):
    start = timer()
    while True:
        result = callable(*args, **kwargs)
        if bool(result) == target:
            return result
        duration = timer() - start
        if timeout and duration > timeout:
            raise TimeoutError('waited for %0.3fs' % duration)
        time.sleep(delay)

def wait(timeout=None, delay=0.1, callable=None, *args, **kwargs):
    return _wait_base(True, timeout, delay, callable, *args, **kwargs)

def wait_false(timeout=None, delay=0.1, callable=None, *args, **kwargs):
    return _wait_base(False, timeout, delay, callable, *args, **kwargs)

def reformat_query(query, *args, **kwargs):
    f = string.Formatter()
    parsed = list(f.parse(query))
    query = '?'.join([i[0] for i in parsed])
    fields = [i[1] for i in parsed if i[1] is not None]
    parameters = tuple(f.get_field(i, args, kwargs)[0] for i in fields)
    return query, parameters

class Arg(object):
    def __init__(self, *args, **kwargs):
        self.args = args
        self.kwargs = kwargs

class UtileArgFormatter(ArgumentDefaultsHelpFormatter,
                        RawDescriptionHelpFormatter):
        """
        Help message formatter which adds default values to argument help and
        which retains any formatting in descriptions.
        """

def arg_parser(description, *args, **kwargs):
    kwargs['description'] = description
    kwargs.setdefault('formatter_class', UtileArgFormatter)
    autocomplete = kwargs.pop('autocomplete', True)
    parser = ArgumentParser(**kwargs)
    for i in args:
        completer = i.kwargs.pop('completer', None)
        action = parser.add_argument(*i.args, **i.kwargs)
        if completer:
            action.completer = completer
    argcomplete = safe_import('argcomplete')
    if autocomplete and argcomplete:
        argcomplete.autocomplete(parser)
    return parser
<commit_msg>improved implementation of print() function<commit_after># utile.py - Collection of useful functions
# Copyright (C) 2013 Marwan Alsabbagh
# license: BSD, see LICENSE for more details.

from __future__ import print_function
import time
import re
import os
import os.path
import sys
import string
import random
from itertools import chain
from timeit import default_timer as timer
from functools import wraps
from shutil import rmtree
from hashlib import sha256
from tempfile import mkdtemp
from contextlib import contextmanager
from fcntl import flock, LOCK_EX, LOCK_NB
from datetime import timedelta
from textwrap import dedent
from operator import itemgetter
from math import log10
from subprocess import check_call, Popen, PIPE
from inspect import getargspec
from argparse import (
    ArgumentParser, ArgumentDefaultsHelpFormatter, RawDescriptionHelpFormatter
)

__version__ = '0.4.dev'
PY3 = sys.version_info[0] == 3
string_types = str if PY3 else basestring
_builtin_print = print
alpha_numeric = string.ascii_letters + string.digits

def random_text(length, characters=alpha_numeric):
    return ''.join(random.choice(characters) for i in range(length))

# print(*objects, sep=' ', end='\n', file=sys.stdout, flush=False)
def print(*args, **kwargs):
    flush = kwargs.pop('flush', False)
    _builtin_print(*args, **kwargs)
    if flush:
        kwargs.get('file', sys.stdout).flush()

def force_print(*args, **kwargs):
    print(*args, file=sys.__stdout__, **kwargs)

def countdown(length, msg='Countdown', delay=0.1):
    start = timer()
    maxlen = 0
    template = '{msg}: {remaining:.%df}' % -log10(delay)
    while timer() - start < length:
        remaining = (start + length) - timer()
        text = template.format(msg=msg, remaining=remaining)
        maxlen = max(maxlen, len(text))
        print(text.ljust(maxlen), end='\r', flush=True)
        time.sleep(delay)
    print('{msg}: done'.format(msg=msg).ljust(maxlen))

def resolve(obj):
    if callable(obj):
        return obj()
    item, module = None, []
    for i in obj.split('.'):
        module.append(i)
        try:
            item = getattr(item, i)
        except AttributeError:
            top_module = __import__('.'.join(module))
            if item:
                item = getattr(item, i)
            else:
                item = top_module
    return item

class LazyResolve(object):
    def __init__(self, lookup=None):
        self.lookup = lookup or dict()

    def __getattr__(self, name):
        obj = resolve(self.lookup.get(name, name))
        setattr(self, name, obj)
        return obj

def safe_import(name, default=None):
    try:
        return resolve(name)
    except ImportError:
        return default

def bunch_or_dict(*args, **kwargs):
    _class = safe_import('bunch.Bunch', dict)
    return _class(*args, **kwargs)

def dir_dict(obj, default=None, only_public=True):
    names = dir(obj)
    if only_public:
        names = [i for i in names if not i.startswith('_')]
    return bunch_or_dict((i, getattr(obj, i, default)) for i in names)

def requires_package(name, pypi_name=None):
    pypi_name = pypi_name or name.split('.')[0]
    msg = 'Could not import %r. Install it by running:\npip install %s'
    enforce(safe_import(name), msg % (name, pypi_name))
    return safe_import(name)

def requires_commands(commands):
    missing = [i for i in commands.split() if not which(i)]
    enforce(not missing, '%r command(s) not found' % ' '.join(missing))

def pretty_xml(xml):
    etree = requires_package('lxml.etree')
    root = etree.fromstring(xml, etree.XMLParser(remove_blank_text=True))
    return etree.tostring(root, pretty_print=True, encoding='unicode')

def element_to_dict(elem, return_tuple=False):
    children = bunch_or_dict(element_to_dict(i, True) for i in elem)
    if return_tuple:
        return elem.tag, children or elem.text
    else:
        return children

def xml_to_dict(xml, *args, **kwargs):
    etree = requires_package('lxml.etree')
    root = etree.fromstring(xml, etree.XMLParser(*args, **kwargs))
    return element_to_dict(root)

def slicer_by_size(sizes):
    s = [slice(sum(sizes[0:i]), sum(sizes[0:i+1])) for i in range(len(sizes))]
    return itemgetter(*s)

def parse_table(text, yaml=False):
    if yaml:
        decoder = requires_package('yaml', 'PyYAML').load
    else:
        decoder = lambda x: x
    lines = dedent(text).strip().splitlines()
    slicer = slicer_by_size([len(i) for i in re.findall('=+ *', lines[0])])
    keys = [i.strip() for i in slicer(lines[1])]
    rows = []
    for line in lines[3:-1]:
        values = [decoder(i.strip()) for i in slicer(line)]
        rows.append(bunch_or_dict(zip(keys, values)))
    return rows

def git_version(version):
    if 'dev' not in version:
        return version
    describe = ''
    if which('git'):
        process = Popen(['git', 'describe'], stdout=PIPE, stderr=PIPE)
        describe, _ = process.communicate()
    if describe:
        return version + describe.decode('utf8').split('-')[1]
    elif os.path.exists('PKG-INFO'):
        info = open('PKG-INFO').read()
        return re.findall(r'^Version: (.*)$', info, re.MULTILINE)[0]
    else:
        return version

def save_args(f):

    @wraps(f)
    def wrapper(self, *args, **kwargs):
        names = getargspec(f).args[1:]
        defaults = zip(reversed(names), reversed(getargspec(f).defaults))
        positional = zip(names, args)
        keyword = kwargs.items()
        for k, v in chain(defaults, positional, keyword):
            setattr(self, k, v)
        return f(self, *args, **kwargs)
    return wrapper

def flatten(data):
    return list(chain.from_iterable(data))

def process_name(pid=None, ignore_errors=False):
    pid = pid or os.getpid()
    try:
        with open('/proc/%s/cmdline' % pid) as f:
            return f.read().strip('\x00').split('\x00')
    except IOError:
        if ignore_errors:
            return []
        else:
            raise

def get_pid_list():
    return sorted(int(i) for i in os.listdir('/proc') if i.isdigit())

def mac_address(interface='eth0'):
    output, _ = Popen(['ifconfig', interface], stdout=PIPE).communicate()
    return re.search(r'HWaddr ([\w:]+)', output.decode('utf8')).group(1)

@contextmanager
def TemporaryDirectory(suffix='', prefix='tmp', dir=None):
    path = mkdtemp(suffix, prefix, dir)
    try:
        yield path
    finally:
        rmtree(path)

@contextmanager
def file_lock(path):
    try:
        f = open(path, 'w')
        flock(f, LOCK_EX | LOCK_NB)
    except IOError:
        f.close()
        raise IOError('Could not lock %r' % path)
    try:
        yield
    finally:
        f.close()
        os.remove(path)

def _cipher(key):
    AES = requires_package('Crypto.Cipher.AES', 'pycrypto')
    return AES.new(sha256(key).digest(), AES.MODE_CFB, '\x00' * AES.block_size)

def encrypt(key, data):
    return _cipher(key).encrypt(data)

def decrypt(key, data):
    return _cipher(key).decrypt(data)

def raises(exceptions, func, *args, **kwargs):
    try:
        func(*args, **kwargs)
    except exceptions:
        return True
    return False

class EnforcementError(Exception):
    pass

def enforce(rule, msg, exception=EnforcementError):
    if not rule:
        raise exception(msg)

def enforce_false(rule, msg, exception=EnforcementError):
    if rule:
        raise exception(msg)

def enforce_clean_exit(func):
    @wraps(func)
    def wrapper(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except EnforcementError as err:
            sys.exit('ERROR: %s' % err)
    return wrapper

def which(cmd):
    os_paths = os.environ['PATH'].split(os.pathsep)
    cmd_paths = [os.path.join(i, cmd) for i in os_paths]
    return [i for i in cmd_paths if os.path.exists(i)]

def shell(cmd=None, msg=None, caller=None, strict=False, verbose=False,
          **kwargs):
    caller = caller or check_call
    msg = msg if msg else cmd
    kwargs.setdefault('shell', True)
    if kwargs['shell']:
        kwargs.setdefault('executable', '/bin/bash')
    if strict or verbose:
        if not kwargs['shell'] or not isinstance(cmd, string_types):
            raise ValueError('strict or verbose can only be used when '
                             'shell=True and cmd is a string')
    set_options = ''
    if strict:
        set_options += 'e'
    if verbose:
        set_options += 'v'
    if set_options:
        cmd = 'set -{}\n{}'.format(set_options, cmd)
    print(' {0} '.format(msg).center(60, '-'))
    start = timer()
    returncode = caller(cmd, **kwargs)
    print('duration: %s' % timedelta(seconds=timer() - start))
    return returncode

def shell_quote(s):
    return "'" + s.replace("'", "'\"'\"'") + "'"

class TimeoutError(Exception):
    pass

def _wait_base(target, timeout, delay, callable, *args, **kwargs):
    start = timer()
    while True:
        result = callable(*args, **kwargs)
        if bool(result) == target:
            return result
        duration = timer() - start
        if timeout and duration > timeout:
            raise TimeoutError('waited for %0.3fs' % duration)
        time.sleep(delay)

def wait(timeout=None, delay=0.1, callable=None, *args, **kwargs):
    return _wait_base(True, timeout, delay, callable, *args, **kwargs)

def wait_false(timeout=None, delay=0.1, callable=None, *args, **kwargs):
    return _wait_base(False, timeout, delay, callable, *args, **kwargs)

def reformat_query(query, *args, **kwargs):
    f = string.Formatter()
    parsed = list(f.parse(query))
    query = '?'.join([i[0] for i in parsed])
    fields = [i[1] for i in parsed if i[1] is not None]
    parameters = tuple(f.get_field(i, args, kwargs)[0] for i in fields)
    return query, parameters

class Arg(object):
    def __init__(self, *args, **kwargs):
        self.args = args
        self.kwargs = kwargs

class UtileArgFormatter(ArgumentDefaultsHelpFormatter,
                        RawDescriptionHelpFormatter):
        """
        Help message formatter which adds default values to argument help and
        which retains any formatting in descriptions.
        """

def arg_parser(description, *args, **kwargs):
    kwargs['description'] = description
    kwargs.setdefault('formatter_class', UtileArgFormatter)
    autocomplete = kwargs.pop('autocomplete', True)
    parser = ArgumentParser(**kwargs)
    for i in args:
        completer = i.kwargs.pop('completer', None)
        action = parser.add_argument(*i.args, **i.kwargs)
        if completer:
            action.completer = completer
    argcomplete = safe_import('argcomplete')
    if autocomplete and argcomplete:
        argcomplete.autocomplete(parser)
    return parser
<|endoftext|>


<commit_before>import os
import re

import sublime
import sublime_plugin

class SplitOnCommand(sublime_plugin.TextCommand):
    def run(self, edit, **kwargs):
        selections = list(self.view.sel())
        selections.reverse()
        split_char = kwargs.get('split_char', '')
        if split_char:
            le = '\n'
            for selection in selections:
                current_line = self.view.line(selection)
                padding = ' ' * (selection.begin() - current_line.begin())
                text = self.view.substr(selection)
                elements = [x.strip() for x in text.split(split_char)]
                elements[1:] = [padding + x for x in elements[1:]]
                new_text = (split_char + le).join(elements)
                self.view.replace(edit, selection, new_text)

class Heading(object):
    def __init__(self, line):
        P = r'^(#+)[ \t]*[\d\.]*[ \t]*(.*)$'
        m = re.match(P, line)
        self.level = len(m.group(1))
        self.text = m.group(2)
        if self.level == 1:
            self.format = "{0}.0"
        else:
            self.format = '.'.join(['{{{v}}}'.format(v=i) for i in range(self.level)])
        self.value = ''

    def format_value(self, levels):
        self.value = self.format.format(*levels)

    def __str__(self):
        return ' '.join(['#' * self.level, self.value, self.text])

class MarkdownHeadings(sublime_plugin.TextCommand):

    def run(self, edit, **kwargs):
        regions = self.view.find_all(r'^(#+)[ \t]*[\d\.]*[ \t]*(.*)$')
        headings = []
        for r in regions:
            headings.append(Heading(self.view.substr(r)))
        levels = [0]
        for h in headings:
            if h.level > len(levels):
                levels.append(1)
            elif h.level == len(levels):
                levels[-1] += 1
            else:
                levels[h.level - 1] += 1
                levels[h.level: ] = [0] * len(levels[h.level: ])
            h.format_value(levels[:h.level])
        headings.reverse()
        regions.reverse()
        for i in range(len(regions)):
            self.view.replace(edit, regions[i], str(headings[i]))

    def is_enabled(self):
        return os.path.splitext(self.view.file_name())[-1].strip('.') in ['md', 'markdown']

class MarkdownHeadingsListener(sublime_plugin.EventListener):
    def on_pre_save(self, view):
        view.run_command('markdown_headings')
<commit_msg>small efficiency fix for markdown headings<commit_after>import os
import re

import sublime
import sublime_plugin

class SplitOnCommand(sublime_plugin.TextCommand):
    def run(self, edit, **kwargs):
        selections = list(self.view.sel())
        selections.reverse()
        split_char = kwargs.get('split_char', '')
        if split_char:
            le = '\n'
            for selection in selections:
                current_line = self.view.line(selection)
                padding = ' ' * (selection.begin() - current_line.begin())
                text = self.view.substr(selection)
                elements = [x.strip() for x in text.split(split_char)]
                elements[1:] = [padding + x for x in elements[1:]]
                new_text = (split_char + le).join(elements)
                self.view.replace(edit, selection, new_text)

HEADING_P = r'^(#+)[ \t]*[\d\.]*[ \t]*(.*)$'

class Heading(object):
    def __init__(self, line):
        m = re.match(HEADING_P, line)
        self.level = len(m.group(1))
        self.text = m.group(2)
        if self.level == 1:
            self.format = "{0}.0"
        else:
            self.format = '.'.join(['{{{v}}}'.format(v=i) for i in range(self.level)])
        self.value = ''

    def format_value(self, levels):
        if self.level > len(levels):
            levels.append(1)
        elif self.level == len(levels):
            levels[-1] += 1
        else:
            levels[self.level - 1] += 1
            levels[self.level: ] = [0] * len(levels[self.level: ])
        self.value = self.format.format(*levels[:self.level])
        return levels

    def __str__(self):
        return ' '.join(['#' * self.level, self.value, self.text])

class MarkdownHeadings(sublime_plugin.TextCommand):

    def run(self, edit, **kwargs):
        regions = self.view.find_all(HEADING_P)
        headings = []
        levels = [0]
        for r in regions:
            h = Heading(self.view.substr(r))
            levels = h.format_value(levels)
            headings.append(h)
        headings.reverse()
        regions.reverse()
        for i in range(len(regions)):
            self.view.replace(edit, regions[i], str(headings[i]))

    def is_enabled(self):
        return os.path.splitext(self.view.file_name())[-1].strip('.') in ['md', 'markdown']

class MarkdownHeadingsListener(sublime_plugin.EventListener):
    def on_pre_save(self, view):
        view.run_command('markdown_headings')
<|endoftext|>