<commit_before>#!/usr/bin/env python

import dataproc
import itertools
import numpy as np
import random
import sys
import tensorflow as tf
from collections import defaultdict
from ltrdnn import LTRDNN

flags = tf.flags
FLAGS = flags.FLAGS
# model related:
flags.DEFINE_integer('vocab_size', 1532783, 'vocabulary size')
flags.DEFINE_integer('emb_dim', 256, 'embedding dimension')
flags.DEFINE_integer('repr_dim', 256, 'sentence representing dimension')
flags.DEFINE_string('combiner', 'sum', 'how to combine words in a sentence')
# training related:
flags.DEFINE_integer('train_bs', 128, 'train batch size')
flags.DEFINE_integer('max_epoch', 1, 'max epoch')
flags.DEFINE_integer('max_iter', 1000, 'max iteration')
flags.DEFINE_float('eps', 1.0, 'zero-loss threshold epsilon in hinge loss')
flags.DEFINE_integer('eval_steps', 20, 'every how many steps to evaluate')

def inp_fn(data):
    def _random_choose(l): return random.sample(l, 1)[0]
    sp_feed = defaultdict(list)
    batch_size = len(data)
    seq_len = 0
    for i, inst in enumerate(data):
        flds = inst.split('\t')
        query = map(int, flds[0].split(' '))
        pos_title_num = int(flds[1])
        pos_titles = flds[2:2+pos_title_num]
        neg_title_num = int(flds[2+pos_title_num])
        neg_titles = flds[2+pos_title_num+1:]

        pos_title = _random_choose(pos_titles)
        pos_title = map(int, pos_title.split(' '))
        neg_title = _random_choose(neg_titles)
        neg_title = map(int, neg_title.split(' '))

        seq_len = max(seq_len, len(query), len(pos_title), len(neg_title))
        for j, word_id in enumerate(query):
            sp_feed['qry_idx'].append([i, j])
            sp_feed['qry_val'].append(word_id)
        for j, word_id in enumerate(pos_title):
            sp_feed['pos_idx'].append([i, j])
            sp_feed['pos_val'].append(word_id)
        for j, word_id in enumerate(neg_title):
            sp_feed['neg_idx'].append([i, j])
            sp_feed['neg_val'].append(word_id)
    return (sp_feed['qry_idx'], sp_feed['qry_val'], [batch_size, seq_len]), \
           (sp_feed['pos_idx'], sp_feed['pos_val'], [batch_size, seq_len]), \
           (sp_feed['neg_idx'], sp_feed['neg_val'], [batch_size, seq_len])

def eval_fn(inst):
    def _max_len(lst): return max([len(x) for x in lst])
    flds = inst.split('\t')
    qrys = flds[0:1]
    pos_num = int(flds[1])
    poss = flds[2:2+pos_num]
    neg_num = int(flds[2+pos_num])
    negs = flds[2+pos_num+1:]
    qrys = [map(int, x.split(' ')) for x in qrys]
    poss = [map(int, x.split(' ')) for x in poss]
    negs = [map(int, x.split(' ')) for x in negs]
    seq_len = max(_max_len(qrys), _max_len(poss), _max_len(negs))
    batch_size = len(qrys) * len(poss) * len(negs)

    sp_feed = defaultdict(list)
    for i, (qry, pos, neg) in enumerate(itertools.product(qrys, poss, negs)):
        for j, word_id in enumerate(qry):
            sp_feed['qry_idx'].append([i, j])
            sp_feed['qry_val'].append(word_id)
        for j, word_id in enumerate(pos):
            sp_feed['pos_idx'].append([i, j])
            sp_feed['pos_val'].append(word_id)
        for j, word_id in enumerate(neg):
            sp_feed['neg_idx'].append([i, j])
            sp_feed['neg_val'].append(word_id)
    return (sp_feed['qry_idx'], sp_feed['qry_val'], [batch_size, seq_len]), \
           (sp_feed['pos_idx'], sp_feed['pos_val'], [batch_size, seq_len]), \
           (sp_feed['neg_idx'], sp_feed['neg_val'], [batch_size, seq_len])

# train_file = './3.train.negtive_sampled.ids'
# test_file = './3.train.negtive_sampled.ids.pair'
# test_file = './data/3.test.negtive_sampled.ids'
train_file = './data_train_example.tsv'
valid_file = './data_test_example.tsv'
test_file = './data_test_example.tsv'
train_freader = dataproc.BatchReader(train_file, max_epoch=FLAGS.max_epoch)
with open(valid_file) as f:
    valid_data = [x.rstrip('\n') for x in f.readlines()]
valid_q, valid_pt, valid_nt = inp_fn(valid_data)

mdl = LTRDNN(
    vocab_size=FLAGS.vocab_size,
    emb_dim=FLAGS.emb_dim,
    repr_dim=FLAGS.repr_dim,
    combiner=FLAGS.combiner,
    eps=FLAGS.eps)

sess = tf.Session()
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
metrics = ['loss']
mdl_ckpt_dir = './model_ckpt/model.ckpt'
for niter in xrange(FLAGS.max_iter):
    batch_data = train_freader.get_batch(FLAGS.train_bs)
    if not batch_data:
        break
    train_q, train_pt, train_nt = inp_fn(batch_data)
    mdl.train_step(sess, train_q, train_pt, train_nt)
    train_eval = mdl.eval_step(sess, train_q, train_pt, train_nt, metrics)
    valid_eval = mdl.eval_step(sess, valid_q, valid_pt, valid_nt, metrics) \
        if niter % FLAGS.eval_steps == 0 else 'SKIP'
    print niter, 'train_loss:', train_eval, 'valid_loss:', valid_eval

feval = open(test_file)
acc = mdl.pairwise_accuracy(sess, feval, eval_fn)
print 'pairwise accuracy:', acc

save_path = mdl.saver.save(sess, mdl_ckpt_dir, global_step=mdl.global_step)
print 'model saved:', save_path

sess.close()
feval.close()
<commit_msg>fix pep8<commit_after>#!/usr/bin/env python

import dataproc
import itertools
import numpy as np
import random
import sys
import tensorflow as tf
from collections import defaultdict
from ltrdnn import LTRDNN

flags = tf.flags
FLAGS = flags.FLAGS
# model related:
flags.DEFINE_integer('vocab_size', 1532783, 'vocabulary size')
flags.DEFINE_integer('emb_dim', 256, 'embedding dimension')
flags.DEFINE_integer('repr_dim', 256, 'sentence representing dimension')
flags.DEFINE_string('combiner', 'sum', 'how to combine words in a sentence')
# training related:
flags.DEFINE_integer('train_bs', 128, 'train batch size')
flags.DEFINE_integer('max_epoch', 1, 'max epoch')
flags.DEFINE_integer('max_iter', 1000, 'max iteration')
flags.DEFINE_float('eps', 1.0, 'zero-loss threshold epsilon in hinge loss')
flags.DEFINE_integer('eval_steps', 20, 'every how many steps to evaluate')

def inp_fn(data):
    def _random_choose(l): return random.sample(l, 1)[0]
    sp_feed = defaultdict(list)
    batch_size = len(data)
    seq_len = 0
    for i, inst in enumerate(data):
        flds = inst.split('\t')
        query = map(int, flds[0].split(' '))
        pos_title_num = int(flds[1])
        pos_titles = flds[2:2+pos_title_num]
        neg_title_num = int(flds[2+pos_title_num])
        neg_titles = flds[2+pos_title_num+1:]

        pos_title = _random_choose(pos_titles)
        pos_title = map(int, pos_title.split(' '))
        neg_title = _random_choose(neg_titles)
        neg_title = map(int, neg_title.split(' '))

        seq_len = max(seq_len, len(query), len(pos_title), len(neg_title))
        for j, word_id in enumerate(query):
            sp_feed['qry_idx'].append([i, j])
            sp_feed['qry_val'].append(word_id)
        for j, word_id in enumerate(pos_title):
            sp_feed['pos_idx'].append([i, j])
            sp_feed['pos_val'].append(word_id)
        for j, word_id in enumerate(neg_title):
            sp_feed['neg_idx'].append([i, j])
            sp_feed['neg_val'].append(word_id)
    return (sp_feed['qry_idx'], sp_feed['qry_val'], [batch_size, seq_len]), \
           (sp_feed['pos_idx'], sp_feed['pos_val'], [batch_size, seq_len]), \
           (sp_feed['neg_idx'], sp_feed['neg_val'], [batch_size, seq_len])

def eval_fn(inst):
    def _max_len(lst): return max([len(x) for x in lst])
    flds = inst.split('\t')
    qrys = flds[0:1]
    pos_num = int(flds[1])
    poss = flds[2:2+pos_num]
    neg_num = int(flds[2+pos_num])
    negs = flds[2+pos_num+1:]
    qrys = [map(int, x.split(' ')) for x in qrys]
    poss = [map(int, x.split(' ')) for x in poss]
    negs = [map(int, x.split(' ')) for x in negs]
    seq_len = max(_max_len(qrys), _max_len(poss), _max_len(negs))
    batch_size = len(qrys) * len(poss) * len(negs)

    sp_feed = defaultdict(list)
    for i, (qry, pos, neg) in enumerate(itertools.product(qrys, poss, negs)):
        for j, word_id in enumerate(qry):
            sp_feed['qry_idx'].append([i, j])
            sp_feed['qry_val'].append(word_id)
        for j, word_id in enumerate(pos):
            sp_feed['pos_idx'].append([i, j])
            sp_feed['pos_val'].append(word_id)
        for j, word_id in enumerate(neg):
            sp_feed['neg_idx'].append([i, j])
            sp_feed['neg_val'].append(word_id)
    return (sp_feed['qry_idx'], sp_feed['qry_val'], [batch_size, seq_len]), \
           (sp_feed['pos_idx'], sp_feed['pos_val'], [batch_size, seq_len]), \
           (sp_feed['neg_idx'], sp_feed['neg_val'], [batch_size, seq_len])

# train_file = './3.train.negtive_sampled.ids'
# test_file = './3.train.negtive_sampled.ids.pair'
# test_file = './data/3.test.negtive_sampled.ids'
train_file = './data_train_example.tsv'
valid_file = './data_test_example.tsv'
test_file = './data_test_example.tsv'
train_freader = dataproc.BatchReader(train_file, max_epoch=FLAGS.max_epoch)
with open(valid_file) as f:
    valid_data = [x.rstrip('\n') for x in f.readlines()]
valid_q, valid_pt, valid_nt = inp_fn(valid_data)

mdl = LTRDNN(
    vocab_size=FLAGS.vocab_size,
    emb_dim=FLAGS.emb_dim,
    repr_dim=FLAGS.repr_dim,
    combiner=FLAGS.combiner,
    eps=FLAGS.eps)

sess = tf.Session()
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
metrics = ['loss']
mdl_ckpt_dir = './model_ckpt/model.ckpt'
for niter in xrange(FLAGS.max_iter):
    batch_data = train_freader.get_batch(FLAGS.train_bs)
    if not batch_data:
        break
    train_q, train_pt, train_nt = inp_fn(batch_data)
    mdl.train_step(sess, train_q, train_pt, train_nt)
    train_eval = mdl.eval_step(sess, train_q, train_pt, train_nt, metrics)
    valid_eval = mdl.eval_step(sess, valid_q, valid_pt, valid_nt, metrics) \
        if niter % FLAGS.eval_steps == 0 else 'SKIP'
    print niter, 'train_loss:', train_eval, 'valid_loss:', valid_eval

feval = open(test_file)
acc = mdl.pairwise_accuracy(sess, feval, eval_fn)
print 'pairwise accuracy:', acc

save_path = mdl.saver.save(sess, mdl_ckpt_dir, global_step=mdl.global_step)
print 'model saved:', save_path

sess.close()
feval.close()
<|endoftext|>


<commit_before>import os
import argparse
import numpy as np
from model.one_hot_model import OneHotModel
from model.augmented_model import AugmentedModel
from model.data_processor import DataProcessor
from model.setting import ProposedSetting

DATA_ROOT = os.path.join(os.path.dirname(__file__), "data")
LOG_ROOT = os.path.join(os.path.dirname(__file__), "log")
MODEL_ROOT = os.path.join(os.path.dirname(__file__), "trained_model")

def prepare_dataset(dataset_kind):
    dp = DataProcessor()
    if dataset_kind == "ptb":
        dataset = dp.get_ptb(DATA_ROOT, vocab_size=10000, force=True)
    else:
        dataset = dp.get_wiki2(DATA_ROOT, vocab_size=30000, force=True)

    return dataset

def train_baseline(network_size, dataset_kind, epochs=40, skip=3):
    # prepare the data
    setting = ProposedSetting(network_size, dataset_kind)
    dataset = prepare_dataset(dataset_kind)
    vocab_size = len(dataset.vocab_data())
    sentence_size = 35

    dp = DataProcessor()
    train_steps, train_generator = dp.make_batch_iter(dataset, sentence_size=sentence_size, skip=skip)
    valid_steps, valid_generator = dp.make_batch_iter(dataset, kind="valid", sentence_size=sentence_size, skip=skip)

    # make one hot model
    model = OneHotModel(vocab_size, sentence_size, setting, LOG_ROOT)
    model.compile()
    model.fit_generator(train_generator, train_steps, valid_generator, valid_steps, epochs=epochs)
    model.save(MODEL_ROOT)

def train_augmented(network_size, dataset_kind, tying=False, epochs=40, skip=3):
    # prepare the data
    setting = ProposedSetting(network_size, dataset_kind)
    dataset = prepare_dataset(dataset_kind)
    vocab_size = len(dataset.vocab_data())
    sentence_size = 35

    dp = DataProcessor()
    train_steps, train_generator = dp.make_batch_iter(dataset, sentence_size=sentence_size, skip=skip)
    valid_steps, valid_generator = dp.make_batch_iter(dataset, kind="valid", sentence_size=sentence_size, skip=skip)

    # make one hot model
    model = AugmentedModel(vocab_size, sentence_size, setting, tying=tying, checkpoint_path=LOG_ROOT)
    model.compile()
    model.fit_generator(train_generator, train_steps, valid_generator, valid_steps, epochs=epochs)
    model.save(MODEL_ROOT)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Experiment")
    parser.add_argument("--aug", action="store_const", const=True, default=False,
                        help="use augmented model")
    parser.add_argument("--tying", action="store_const", const=True, default=False,
                        help="use tying model")
    parser.add_argument("--nsize", default="small", help="network size (small, medium, large)")
    parser.add_argument("--dataset", default="ptb", help="dataset kind (ptb or wiki2)")
    parser.add_argument("--epochs", type=int, default=40, help="epoch to train")
    parser.add_argument("--skip", type=int, default=1, help="skip size of sentence")
    args = parser.parse_args()

    n_size = args.nsize
    dataset = args.dataset

    if not os.path.exists(LOG_ROOT):
        os.mkdir(LOG_ROOT)

    if args.aug or args.tying:
        print("Use Augmented Model (tying={})".format(args.tying))
        train_augmented(n_size, dataset, args.tying, args.epochs, args.skip)
    else:
        train_baseline(n_size, dataset, args.epochs, args.skip)
<commit_msg>avoid force<commit_after>import os
import argparse
import numpy as np
from model.one_hot_model import OneHotModel
from model.augmented_model import AugmentedModel
from model.data_processor import DataProcessor
from model.setting import ProposedSetting

DATA_ROOT = os.path.join(os.path.dirname(__file__), "data")
LOG_ROOT = os.path.join(os.path.dirname(__file__), "log")
MODEL_ROOT = os.path.join(os.path.dirname(__file__), "trained_model")

def prepare_dataset(dataset_kind):
    dp = DataProcessor()
    if dataset_kind == "ptb":
        dataset = dp.get_ptb(DATA_ROOT, vocab_size=10000)
    else:
        dataset = dp.get_wiki2(DATA_ROOT, vocab_size=30000)

    return dataset

def train_baseline(network_size, dataset_kind, epochs=40, skip=3):
    # prepare the data
    setting = ProposedSetting(network_size, dataset_kind)
    dataset = prepare_dataset(dataset_kind)
    vocab_size = len(dataset.vocab_data())
    sentence_size = 35

    dp = DataProcessor()
    train_steps, train_generator = dp.make_batch_iter(dataset, sentence_size=sentence_size, skip=skip)
    valid_steps, valid_generator = dp.make_batch_iter(dataset, kind="valid", sentence_size=sentence_size, skip=skip)

    # make one hot model
    model = OneHotModel(vocab_size, sentence_size, setting, LOG_ROOT)
    model.compile()
    model.fit_generator(train_generator, train_steps, valid_generator, valid_steps, epochs=epochs)
    model.save(MODEL_ROOT)

def train_augmented(network_size, dataset_kind, tying=False, epochs=40, skip=3):
    # prepare the data
    setting = ProposedSetting(network_size, dataset_kind)
    dataset = prepare_dataset(dataset_kind)
    vocab_size = len(dataset.vocab_data())
    sentence_size = 35

    dp = DataProcessor()
    train_steps, train_generator = dp.make_batch_iter(dataset, sentence_size=sentence_size, skip=skip)
    valid_steps, valid_generator = dp.make_batch_iter(dataset, kind="valid", sentence_size=sentence_size, skip=skip)

    # make one hot model
    model = AugmentedModel(vocab_size, sentence_size, setting, tying=tying, checkpoint_path=LOG_ROOT)
    model.compile()
    model.fit_generator(train_generator, train_steps, valid_generator, valid_steps, epochs=epochs)
    model.save(MODEL_ROOT)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Experiment")
    parser.add_argument("--aug", action="store_const", const=True, default=False,
                        help="use augmented model")
    parser.add_argument("--tying", action="store_const", const=True, default=False,
                        help="use tying model")
    parser.add_argument("--nsize", default="small", help="network size (small, medium, large)")
    parser.add_argument("--dataset", default="ptb", help="dataset kind (ptb or wiki2)")
    parser.add_argument("--epochs", type=int, default=40, help="epoch to train")
    parser.add_argument("--skip", type=int, default=1, help="skip size of sentence")
    args = parser.parse_args()

    n_size = args.nsize
    dataset = args.dataset

    if not os.path.exists(LOG_ROOT):
        os.mkdir(LOG_ROOT)

    if args.aug or args.tying:
        print("Use Augmented Model (tying={})".format(args.tying))
        train_augmented(n_size, dataset, args.tying, args.epochs, args.skip)
    else:
        train_baseline(n_size, dataset, args.epochs, args.skip)
<|endoftext|>


<commit_before># Copyright 2016 Louis Kirsch. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

import tensorflow as tf
import numpy as np
import time
import os

import vocabulary
from wav2letter_model import Wav2LetterModel
from preprocess import SpeechCorpusReader

tf.app.flags.DEFINE_float("learning_rate", 1e-3, "Learning rate.")
tf.app.flags.DEFINE_float("learning_rate_decay_factor", 0.9,
                          "Learning rate decays by this much (multiplication).")
tf.app.flags.DEFINE_float("max_gradient_norm", 5.0, "Clip gradients to this norm.")
tf.app.flags.DEFINE_bool('relu', False, 'Use ReLU activation instead of tanh')
tf.app.flags.DEFINE_integer("batch_size", 64,
                            "Batch size to use during training.")
tf.app.flags.DEFINE_string("data_dir", "data/", "Data directory")
tf.app.flags.DEFINE_string("train_dir", "train/", "Training directory")
tf.app.flags.DEFINE_string("log_dir", "log/", "Logging directory for summaries")
tf.app.flags.DEFINE_string("run_name", "", "Give this training a name to appear in tensorboard")
tf.app.flags.DEFINE_integer("limit_training_set", 0,
                            "Train on a smaller training set, limited to the specified size")
tf.app.flags.DEFINE_integer("steps_per_checkpoint", 200,
                            "How many training steps to do per checkpoint.")

FLAGS = tf.app.flags.FLAGS

N_COEFFICIENTS = 13 * 3

def extract_decoded_ids(sparse_tensor):
  ids = []
  last_batch_id = 0
  for i, index in enumerate(sparse_tensor.indices):
    batch_id, char_id = index
    if batch_id > last_batch_id:
      yield ids
      ids = []
      last_batch_id = batch_id
    ids.append(sparse_tensor.values[i])
  yield ids

def create_model(session):
  """Create speechT model and initialize or load parameters in session."""
  model = Wav2LetterModel(N_COEFFICIENTS,
                          vocabulary.SIZE + 1,
                          FLAGS.learning_rate,
                          FLAGS.learning_rate_decay_factor,
                          FLAGS.max_gradient_norm,
                          FLAGS.log_dir,
                          FLAGS.relu,
                          FLAGS.run_name)
  ckpt = tf.train.get_checkpoint_state(FLAGS.train_dir)
  if ckpt and tf.gfile.Exists(ckpt.model_checkpoint_path):
    print("Reading model parameters from %s" % ckpt.model_checkpoint_path)
    model.saver.restore(session, ckpt.model_checkpoint_path)
    model.init_session(session, init_variables=False)
  else:
    print("Created model with fresh parameters.")
    model.init_session(session, init_variables=True)
  return model

def train():
  # Create training directory if it does not exist
  if not os.path.exists(FLAGS.train_dir):
    os.makedirs(FLAGS.train_dir)

  with tf.Session() as sess:
    model = create_model(sess)

    reader = SpeechCorpusReader(FLAGS.data_dir)

    def batch(iterable, batch_size):
      args = [iter(iterable)] * batch_size
      return zip(*args)

    sample_generator = reader.load_samples('train', loop_infinitely=True, limit_count=FLAGS.limit_training_set)
    dev_sample_generator = reader.load_samples('dev', loop_infinitely=True)

    step_time, loss = 0.0, 0.0
    current_step = 0
    previous_losses = []

    for sample_batch in batch(sample_generator, FLAGS.batch_size):
      input_list, label_list = zip(*sample_batch)

      current_step += 1
      is_checkpoint_step = current_step % FLAGS.steps_per_checkpoint == 0

      start_time = time.time()
      step_result = model.step(sess, input_list, label_list, summary=is_checkpoint_step)
      avg_loss = step_result[0]
      step_time += (time.time() - start_time) / FLAGS.steps_per_checkpoint
      loss += avg_loss / FLAGS.steps_per_checkpoint

      # Once in a while, we save checkpoint, print statistics, and run evals.
      if is_checkpoint_step:
        global_step = model.global_step.eval()

        # Print statistics for the previous epoch.
        perplexity = np.exp(float(avg_loss)) if avg_loss < 300 else float("inf")
        print("global step {:d} learning rate {:.4f} step-time {:.2f} average loss {:.2f} perplexity {:.2f}"
              .format(global_step, model.learning_rate.eval(), step_time, avg_loss, perplexity))

        # Decrease learning rate if no improvement was seen over last 3 times.
        if len(previous_losses) > 2 and loss > max(previous_losses[-3:]):
          sess.run(model.learning_rate_decay_op)
        previous_losses.append(loss)

        # Save checkpoint and zero timer and loss.
        checkpoint_path = os.path.join(FLAGS.train_dir, "speechT.ckpt")
        model.saver.save(sess, checkpoint_path, global_step=model.global_step)
        step_time, loss = 0.0, 0.0

        # Retrieve and store summary
        summary = step_result[2]
        model.train_writer.add_summary(summary, global_step)

        # Validate on development set and write summary
        audio, label = next(dev_sample_generator)
        avg_loss, decoded, summary = model.step(sess, [audio], [label], update=False, decode=True, summary=True)
        model.dev_writer.add_summary(summary, global_step)
        perplexity = np.exp(float(avg_loss)) if avg_loss < 300 else float("inf")
        print("Validation average loss {:.2f} perplexity {:.2f}".format(avg_loss, perplexity))
        decoded_ids = next(extract_decoded_ids(decoded))
        decoded_str = vocabulary.ids_to_sentence(decoded_ids)
        expected_str = vocabulary.ids_to_sentence(label_list[0])
        print('Expected: {}'.format(expected_str))
        print('Decoded: {}'.format(decoded_str))

def main(_):
  train()

if __name__ == "__main__":
  tf.app.run()
<commit_msg>Decrease learning rate only after evaluation on dev set<commit_after># Copyright 2016 Louis Kirsch. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

import tensorflow as tf
import numpy as np
import time
import os

import vocabulary
from wav2letter_model import Wav2LetterModel
from preprocess import SpeechCorpusReader

tf.app.flags.DEFINE_float("learning_rate", 1e-3, "Learning rate.")
tf.app.flags.DEFINE_float("learning_rate_decay_factor", 0.9,
                          "Learning rate decays by this much (multiplication).")
tf.app.flags.DEFINE_float("max_gradient_norm", 5.0, "Clip gradients to this norm.")
tf.app.flags.DEFINE_bool('relu', False, 'Use ReLU activation instead of tanh')
tf.app.flags.DEFINE_integer("batch_size", 64,
                            "Batch size to use during training.")
tf.app.flags.DEFINE_string("data_dir", "data/", "Data directory")
tf.app.flags.DEFINE_string("train_dir", "train/", "Training directory")
tf.app.flags.DEFINE_string("log_dir", "log/", "Logging directory for summaries")
tf.app.flags.DEFINE_string("run_name", "", "Give this training a name to appear in tensorboard")
tf.app.flags.DEFINE_integer("limit_training_set", 0,
                            "Train on a smaller training set, limited to the specified size")
tf.app.flags.DEFINE_integer("steps_per_checkpoint", 200,
                            "How many training steps to do per checkpoint.")

FLAGS = tf.app.flags.FLAGS

N_COEFFICIENTS = 13 * 3

def extract_decoded_ids(sparse_tensor):
  ids = []
  last_batch_id = 0
  for i, index in enumerate(sparse_tensor.indices):
    batch_id, char_id = index
    if batch_id > last_batch_id:
      yield ids
      ids = []
      last_batch_id = batch_id
    ids.append(sparse_tensor.values[i])
  yield ids

def create_model(session):
  """Create speechT model and initialize or load parameters in session."""
  model = Wav2LetterModel(N_COEFFICIENTS,
                          vocabulary.SIZE + 1,
                          FLAGS.learning_rate,
                          FLAGS.learning_rate_decay_factor,
                          FLAGS.max_gradient_norm,
                          FLAGS.log_dir,
                          FLAGS.relu,
                          FLAGS.run_name)
  ckpt = tf.train.get_checkpoint_state(FLAGS.train_dir)
  if ckpt and tf.gfile.Exists(ckpt.model_checkpoint_path):
    print("Reading model parameters from %s" % ckpt.model_checkpoint_path)
    model.saver.restore(session, ckpt.model_checkpoint_path)
    model.init_session(session, init_variables=False)
  else:
    print("Created model with fresh parameters.")
    model.init_session(session, init_variables=True)
  return model

def train():
  # Create training directory if it does not exist
  if not os.path.exists(FLAGS.train_dir):
    os.makedirs(FLAGS.train_dir)

  with tf.Session() as sess:
    model = create_model(sess)

    reader = SpeechCorpusReader(FLAGS.data_dir)

    def batch(iterable, batch_size):
      args = [iter(iterable)] * batch_size
      return zip(*args)

    sample_generator = reader.load_samples('train', loop_infinitely=True, limit_count=FLAGS.limit_training_set)
    dev_sample_generator = reader.load_samples('dev', loop_infinitely=True)

    step_time, loss = 0.0, 0.0
    current_step = 0
    previous_losses = []

    for sample_batch in batch(sample_generator, FLAGS.batch_size):
      input_list, label_list = zip(*sample_batch)

      current_step += 1
      is_checkpoint_step = current_step % FLAGS.steps_per_checkpoint == 0

      start_time = time.time()
      step_result = model.step(sess, input_list, label_list, summary=is_checkpoint_step)
      avg_loss = step_result[0]
      step_time += (time.time() - start_time) / FLAGS.steps_per_checkpoint
      loss += avg_loss / FLAGS.steps_per_checkpoint

      # Once in a while, we save checkpoint, print statistics, and run evals.
      if is_checkpoint_step:
        global_step = model.global_step.eval()

        # Print statistics for the previous epoch.
        perplexity = np.exp(float(avg_loss)) if avg_loss < 300 else float("inf")
        print("global step {:d} learning rate {:.4f} step-time {:.2f} average loss {:.2f} perplexity {:.2f}"
              .format(global_step, model.learning_rate.eval(), step_time, avg_loss, perplexity))

        # Retrieve and store summary
        summary = step_result[2]
        model.train_writer.add_summary(summary, global_step)

        # Validate on development set and write summary
        audio, label = next(dev_sample_generator)
        avg_loss, decoded, summary = model.step(sess, [audio], [label], update=False, decode=True, summary=True)
        model.dev_writer.add_summary(summary, global_step)
        perplexity = np.exp(float(avg_loss)) if avg_loss < 300 else float("inf")
        print("validation average loss {:.2f} perplexity {:.2f}".format(avg_loss, perplexity))
        decoded_ids = next(extract_decoded_ids(decoded))
        decoded_str = vocabulary.ids_to_sentence(decoded_ids)
        expected_str = vocabulary.ids_to_sentence(label_list[0])
        print('expected: {}'.format(expected_str))
        print('decoded: {}'.format(decoded_str))

        # Decrease learning rate if no improvement was seen over last 3 times.
        if len(previous_losses) > 2 and loss > max(previous_losses[-3:]):
          sess.run(model.learning_rate_decay_op)
        previous_losses.append(loss)

        # Save checkpoint and zero timer and loss.
        checkpoint_path = os.path.join(FLAGS.train_dir, "speechT.ckpt")
        model.saver.save(sess, checkpoint_path, global_step=model.global_step)
        print('Model saved')
        step_time, loss = 0.0, 0.0

def main(_):
  train()

if __name__ == "__main__":
  tf.app.run()
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
np.random.seed(2 ** 10)

# Prevent reaching to maximum recursion depth in `theano.tensor.grad`
# import sys
# sys.setrecursionlimit(2 ** 20)

from six.moves import range

from keras.datasets import cifar10
from keras.layers import Input, Dense, Layer, merge, Activation, Flatten, Lambda
from keras.layers.convolutional import Convolution2D, AveragePooling2D
from keras.layers.normalization import BatchNormalization
from keras.models import Model
from keras.optimizers import SGD
from keras.callbacks import Callback
from keras.preprocessing.image import ImageDataGenerator
from keras.utils import np_utils
import keras.backend as K

batch_size = 16
nb_classes = 10
nb_epoch = 500
N = 18

death_mode = "lin_decay"  # or uniform
death_rate = 0.5

img_rows, img_cols = 32, 32
img_channels = 3

(X_train, y_train), (X_test, y_test) = cifar10.load_data()
print('X_train shape:', X_train.shape)
print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')

X_train = X_train.astype('float32')
X_test = X_test.astype('float32')

X_train -= X_train.mean(axis=0)
X_train *= 1 / X_train.std(axis=0)
X_test -= X_test.mean(axis=0)
X_test *= 1 / X_test.std(axis=0)

# convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)

add_tables = []

inputs = Input(shape=(img_channels, img_rows, img_cols))

net = Convolution2D(16, 3, 3, border_mode="same")(inputs)
net = BatchNormalization()(net)
net = Activation("relu")(net)

def residual_drop(x, input_shape, output_shape, strides=(1, 1)):
    global add_tables

    nb_filter = output_shape[0]
    conv = Convolution2D(nb_filter, 3, 3, subsample=strides, border_mode="same")(x)
    conv = BatchNormalization()(conv)
    conv = Activation("relu")(conv)
    conv = Convolution2D(nb_filter, 3, 3, border_mode="same")(conv)
    conv = BatchNormalization()(conv)

    if strides[0] >= 2:
        x = AveragePooling2D(strides)(x)

    if (output_shape[0] - input_shape[0]) > 0:
        pad_shape = (batch_size,
                     output_shape[0] - input_shape[0],
                     output_shape[1],
                     output_shape[2])
        padding = K.ones(pad_shape)
        x = Lambda(lambda y: K.concatenate([y, padding], axis=1),
                   output_shape=output_shape)(x)

    _death_rate = K.variable(death_rate)
    scale = K.ones_like(conv) - _death_rate
    conv = Lambda(lambda c: K.in_test_phase(scale * c, c),
                  output_shape=output_shape)(conv)

    out = merge([conv, x], mode="sum")
    out = Activation("relu")(out)

    gate = K.variable(1, dtype="uint8")
    add_tables += [{"death_rate": _death_rate, "gate": gate}]
    return Lambda(lambda tensors: K.switch(gate, tensors[0], tensors[1]),
                  output_shape=output_shape)([out, x])

for i in range(N):
    net = residual_drop(net, input_shape=(16, 32, 32), output_shape=(16, 32, 32))

net = residual_drop(
    net,
    input_shape=(16, 32, 32),
    output_shape=(32, 16, 16),
    strides=(2, 2)
)
for i in range(N - 1):
    net = residual_drop(
        net,
        input_shape=(32, 16, 16),
        output_shape=(32, 16, 16)
    )

net = residual_drop(
    net,
    input_shape=(32, 16, 16),
    output_shape=(64, 8, 8),
    strides=(2, 2)
)
for i in range(N - 1):
    net = residual_drop(
        net,
        input_shape=(64, 8, 8),
        output_shape=(64, 8, 8)
    )

pool = AveragePooling2D((8, 8))(net)
flatten = Flatten()(pool)

predictions = Dense(10, activation="softmax")(flatten)
model = Model(input=inputs, output=predictions)

sgd = SGD(lr=0.5, decay=1e-4, momentum=0.9, nesterov=True)
model.compile(optimizer=sgd, loss="categorical_crossentropy")

def open_all_gates():
    for t in add_tables:
        K.set_value(t["gate"], 1)

# setup death rate
for i, tb in enumerate(add_tables):
    if death_mode == "uniform":
        K.set_value(tb["death_rate"], death_rate)
    elif death_mode == "lin_decay":
        K.set_value(tb["death_rate"], i / len(add_tables) * death_rate)
    else:
        raise

class GatesUpdate(Callback):
    def on_batch_begin(self, batch, logs={}):
        open_all_gates()

        rands = np.random.uniform(size=len(add_tables))
        for t, rand in zip(add_tables, rands):
            if rand < K.get_value(t["death_rate"]):
                K.set_value(t["gate"], 0)

    def on_epoch_end(self, epoch, logs={}):
        open_all_gates()  # for validation

datagen = ImageDataGenerator(
    featurewise_center=False,
    samplewise_center=False,
    featurewise_std_normalization=False,
    samplewise_std_normalization=False,
    zca_whitening=False,
    rotation_range=0,
    width_shift_range=0.,
    height_shift_range=0.,
    horizontal_flip=True,
    vertical_flip=False)
datagen.fit(X_train)

test_datagen = ImageDataGenerator(
    featurewise_center=False,
    samplewise_center=False,
    featurewise_std_normalization=False,
    samplewise_std_normalization=False,
    zca_whitening=False,
    rotation_range=0,
    width_shift_range=0.,
    height_shift_range=0.,
    horizontal_flip=False,
    vertical_flip=False)
test_datagen.fit(X_test)

# fit the model on the batches generated by datagen.flow()
model.fit_generator(datagen.flow(X_train, Y_train, batch_size=batch_size),
                    samples_per_epoch=X_train.shape[0],
                    nb_epoch=nb_epoch,
                    validation_data=test_datagen.flow(X_test, Y_test, batch_size=batch_size),
                    nb_val_samples=X_test.shape[0])
<commit_msg>Avoid GpuJoin error when batch_size changes<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
np.random.seed(2 ** 10)

# Prevent reaching to maximum recursion depth in `theano.tensor.grad`
# import sys
# sys.setrecursionlimit(2 ** 20)

from six.moves import range

from keras.datasets import cifar10
from keras.layers import Input, Dense, Layer, merge, Activation, Flatten, Lambda
from keras.layers.convolutional import Convolution2D, AveragePooling2D
from keras.layers.normalization import BatchNormalization
from keras.models import Model
from keras.optimizers import SGD
from keras.callbacks import Callback
from keras.preprocessing.image import ImageDataGenerator
from keras.utils import np_utils
import keras.backend as K

batch_size = 50  # train/test data size must be divisible by batch_size
nb_classes = 10
nb_epoch = 500
N = 18

death_mode = "lin_decay"  # or uniform
death_rate = 0.5

img_rows, img_cols = 32, 32
img_channels = 3

(X_train, y_train), (X_test, y_test) = cifar10.load_data()
print('X_train shape:', X_train.shape)
print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')

X_train = X_train.astype('float32')
X_test = X_test.astype('float32')

X_train -= X_train.mean(axis=0)
X_train *= 1 / X_train.std(axis=0)
X_test -= X_test.mean(axis=0)
X_test *= 1 / X_test.std(axis=0)

# convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)

add_tables = []

inputs = Input(shape=(img_channels, img_rows, img_cols))

net = Convolution2D(16, 3, 3, border_mode="same")(inputs)
net = BatchNormalization()(net)
net = Activation("relu")(net)

def residual_drop(x, input_shape, output_shape, strides=(1, 1)):
    global add_tables

    nb_filter = output_shape[0]
    conv = Convolution2D(nb_filter, 3, 3, subsample=strides, border_mode="same")(x)
    conv = BatchNormalization()(conv)
    conv = Activation("relu")(conv)
    conv = Convolution2D(nb_filter, 3, 3, border_mode="same")(conv)
    conv = BatchNormalization()(conv)

    if strides[0] >= 2:
        x = AveragePooling2D(strides)(x)

    if (output_shape[0] - input_shape[0]) > 0:
        pad_shape = (batch_size,
                     output_shape[0] - input_shape[0],
                     output_shape[1],
                     output_shape[2])
        padding = K.ones(pad_shape)
        x = Lambda(lambda y: K.concatenate([y, padding], axis=1),
                   output_shape=output_shape)(x)

    _death_rate = K.variable(death_rate)
    scale = K.ones_like(conv) - _death_rate
    conv = Lambda(lambda c: K.in_test_phase(scale * c, c),
                  output_shape=output_shape)(conv)

    out = merge([conv, x], mode="sum")
    out = Activation("relu")(out)

    gate = K.variable(1, dtype="uint8")
    add_tables += [{"death_rate": _death_rate, "gate": gate}]
    return Lambda(lambda tensors: K.switch(gate, tensors[0], tensors[1]),
                  output_shape=output_shape)([out, x])

for i in range(N):
    net = residual_drop(net, input_shape=(16, 32, 32), output_shape=(16, 32, 32))

net = residual_drop(
    net,
    input_shape=(16, 32, 32),
    output_shape=(32, 16, 16),
    strides=(2, 2)
)
for i in range(N - 1):
    net = residual_drop(
        net,
        input_shape=(32, 16, 16),
        output_shape=(32, 16, 16)
    )

net = residual_drop(
    net,
    input_shape=(32, 16, 16),
    output_shape=(64, 8, 8),
    strides=(2, 2)
)
for i in range(N - 1):
    net = residual_drop(
        net,
        input_shape=(64, 8, 8),
        output_shape=(64, 8, 8)
    )

pool = AveragePooling2D((8, 8))(net)
flatten = Flatten()(pool)

predictions = Dense(10, activation="softmax")(flatten)
model = Model(input=inputs, output=predictions)

sgd = SGD(lr=0.5, decay=1e-4, momentum=0.9, nesterov=True)
model.compile(optimizer=sgd, loss="categorical_crossentropy")

def open_all_gates():
    for t in add_tables:
        K.set_value(t["gate"], 1)

# setup death rate
for i, tb in enumerate(add_tables):
    if death_mode == "uniform":
        K.set_value(tb["death_rate"], death_rate)
    elif death_mode == "lin_decay":
        K.set_value(tb["death_rate"], i / len(add_tables) * death_rate)
    else:
        raise

class GatesUpdate(Callback):
    def on_batch_begin(self, batch, logs={}):
        open_all_gates()

        rands = np.random.uniform(size=len(add_tables))
        for t, rand in zip(add_tables, rands):
            if rand < K.get_value(t["death_rate"]):
                K.set_value(t["gate"], 0)

    def on_epoch_end(self, epoch, logs={}):
        open_all_gates()  # for validation

datagen = ImageDataGenerator(
    featurewise_center=False,
    samplewise_center=False,
    featurewise_std_normalization=False,
    samplewise_std_normalization=False,
    zca_whitening=False,
    rotation_range=0,
    width_shift_range=0.,
    height_shift_range=0.,
    horizontal_flip=True,
    vertical_flip=False)
datagen.fit(X_train)

test_datagen = ImageDataGenerator(
    featurewise_center=False,
    samplewise_center=False,
    featurewise_std_normalization=False,
    samplewise_std_normalization=False,
    zca_whitening=False,
    rotation_range=0,
    width_shift_range=0.,
    height_shift_range=0.,
    horizontal_flip=False,
    vertical_flip=False)
test_datagen.fit(X_test)

# fit the model on the batches generated by datagen.flow()
model.fit_generator(datagen.flow(X_train, Y_train, batch_size=batch_size),
                    samples_per_epoch=X_train.shape[0],
                    nb_epoch=nb_epoch,
                    validation_data=test_datagen.flow(X_test, Y_test, batch_size=batch_size),
                    nb_val_samples=X_test.shape[0])
<|endoftext|>


<commit_before>"""Train text classifier to detect spam/ham e-mail-messages.
"""
from ml import read_split, read_one, count_class

from sklearn.naive_bayes import MultinomialNB
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.metrics import confusion_matrix, classification_report
from sklearn.pipeline import Pipeline

CLASSES = ['ham', 'spam']
TARGETS = [0, 1]
COMBINE = True

SA_SPAM = ['data/spam-assassin/spam']
SA_HAM  = ['data/spam-assassin/easy_ham', 'data/spam-assassin/hard_ham']
ENRON_SPAM = ['data/enron/spam']
ENRON_HAM  = ['data/enron/ham']

print('Read features...')
if COMBINE:
    print('Training and test data from same corpus')
    SPAM = SA_SPAM + ENRON_SPAM
    HAM = SA_HAM + ENRON_HAM
    X_train, X_test, y_train, y_test = read_split(SPAM, HAM, 19300, 19300, 0.12)
else:
    print('Training and test data from separate corpus')
    X_train, y_train = read_one(ENRON_SPAM, ENRON_HAM, 17000, 17000)
    X_test, y_test = read_one(SA_SPAM, SA_HAM, 2300, 2300)

print('%s ham training emails' % count_class(0, y_train))
print('%s spam training emails' % count_class(1, y_train))
print('%s ham test emails' % count_class(0, y_test))
print('%s spam test emails' % count_class(1, y_test))

print('Create pipeline...')
clf = Pipeline([
    ('count', CountVectorizer(
        stop_words='english',
        ngram_range=(1, 2),
    )),
    ('tfidf', TfidfTransformer()),
    ('clf', MultinomialNB()),
])

print('Training classifier on the training set...')
clf = clf.fit(X_train, y_train)

print('Evaluating classifier on the test set...')

print('Confusion matrix:')
y_pred = clf.predict(X_test)
cm = confusion_matrix(y_test, y_pred)
print(cm)

cm_tn, cm_fn = cm[0][0], cm[0][1]
cm_tp, cm_fp = cm[1][1], cm[1][0]
print('%s spams correctly classified as spam' % cm_tp)
print('%s hams correctly classified as ham' % cm_tn)
print('%s hams incorrectly classified as spam' % cm_fp)
print('%s spams incorrectly classified as ham' % cm_fn)

print(classification_report(y_test, y_pred, target_names=CLASSES))
<commit_msg>tidy up lines<commit_after>"""Train text classifier to detect spam/ham e-mail-messages.
"""
from ml import read_split, read_one, count_class

from sklearn.naive_bayes import MultinomialNB
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.metrics import confusion_matrix, classification_report
from sklearn.pipeline import Pipeline

CLASSES = ['ham', 'spam']
TARGETS = [0, 1]
COMBINE = True

SA_SPAM = ['data/spam-assassin/spam']
SA_HAM  = ['data/spam-assassin/easy_ham', 'data/spam-assassin/hard_ham']
ENRON_SPAM = ['data/enron/spam']
ENRON_HAM  = ['data/enron/ham']

print('Read features...')
if COMBINE:
    print('Training and test data from same corpus')
    SPAM = SA_SPAM + ENRON_SPAM
    HAM = SA_HAM + ENRON_HAM
    X_train, X_test, y_train, y_test = read_split(SPAM, HAM, 19300, 19300, 0.12)
else:
    print('Training and test data from separate corpus')
    X_train, y_train = read_one(ENRON_SPAM, ENRON_HAM, 17000, 17000)
    X_test, y_test = read_one(SA_SPAM, SA_HAM, 2300, 2300)

print('%s ham training emails' % count_class(0, y_train))
print('%s spam training emails' % count_class(1, y_train))
print('%s ham test emails' % count_class(0, y_test))
print('%s spam test emails' % count_class(1, y_test))

print('Create pipeline...')
clf = Pipeline([
    ('count', CountVectorizer(
        stop_words='english',
        ngram_range=(1, 2),
    )),
    ('tfidf', TfidfTransformer()),
    ('clf', MultinomialNB()),
])

print('Training classifier on the training set...')
clf = clf.fit(X_train, y_train)

print('Evaluating classifier on the test set...')
print('Confusion matrix:')
y_pred = clf.predict(X_test)
cm = confusion_matrix(y_test, y_pred)
print(cm)

cm_tn, cm_fn = cm[0][0], cm[0][1]
cm_tp, cm_fp = cm[1][1], cm[1][0]
print('%s spams correctly classified as spam' % cm_tp)
print('%s hams correctly classified as ham' % cm_tn)
print('%s hams incorrectly classified as spam' % cm_fp)
print('%s spams incorrectly classified as ham' % cm_fn)

print(classification_report(y_test, y_pred, target_names=CLASSES))
<|endoftext|>


<commit_before>import numpy as np
np.random.seed(777)
import chainer
from chainer import cuda
from chainer import optimizers
from chainer import serializers

from tqdm import tqdm
import argparse
import csv
import sys
import os

parser = argparse.ArgumentParser()
group = parser.add_mutually_exclusive_group()
group.add_argument('--original', action='store_true',
                    default=True, help='train on original MNIST')
group.add_argument('--translated', action='store_true',
                    default=False, help='train on translated MNIST')
group.add_argument('--cluttered', action='store_true',
                    default=False, help='train on translated & cluttered MNIST')
parser.add_argument('--lstm', type=bool, default=False,
                    help='use LSTM units in core layer')
parser.add_argument('-m', '--initmodel', type=str, default='',
                    help='load model weights from given file')
parser.add_argument('-g', '--gpuid', type=int, default=-1,
                    help='GPU device ID (default is CPU)')
parser.add_argument('-b', '--batchsize', type=int, default=100,
                    help='training batch size')
parser.add_argument('-v', '--variance', type=float, default=0.05,
                    help='variance of location policy')
parser.add_argument('-e', '--epoch', type=int, default=500,
                    help='iterate training given epoch times')
parser.add_argument('-f', '--filename', type=str, default='',
                    help='prefix of output filenames')
args = parser.parse_args()

# load mnist dataset
train, test = chainer.datasets.get_mnist()
train_data, train_targets = np.array(train).transpose()
test_data, test_targets = np.array(test).transpose()
train_data = np.array(list(train_data)).reshape(train_data.shape[0],1,28,28)
test_data = np.array(list(test_data)).reshape(test_data.shape[0],1,28,28)
train_data.flags.writeable = False
train_targets = np.array(train_targets).astype(np.int32)
test_targets = np.array(test_targets).astype(np.int32)

# hyper-params for each task
if args.original:
    filename = 'ram_original'
    # RAM params for original MNIST
    g_size = 8
    n_steps = 6
    n_scales = 1

    def process(batch):
        return batch

if args.translated:
    filename = 'ram_translated'
    g_size = 12
    n_steps = 8
    n_scales = 3

    # create translated MNIST
    def translate(batch):
        n, c, w_i = batch.shape[:3]
        w_o = 60
        data = np.zeros(shape=(n,c,w_o,w_o), dtype=np.float32)
        for k in range(n):
            i, j = np.random.randint(0, w_o-w_i, size=2)
            data[k, :, i:i+w_i, j:j+w_i] += batch[k]
        return data

    process = translate

if args.cluttered:
    filename = 'ram_cluttered'
    g_size = 12
    n_steps = 8
    n_scales = 3

    # create cluttered MNIST
    def clutter(batch):
        n, c, w_i = batch.shape[:3]
        w_o = 60
        data = np.zeros(shape=(n,c,w_o,w_o), dtype=np.float32)
        for k in range(n):
            i, j = np.random.randint(0, w_o-w_i, size=2)
            data[k, :, i:i+w_i, j:j+w_i] += batch[k]
            for _ in range(4):
                clt = train_data[np.random.randint(0, train_data.shape[0]-1)]
                c1, c2 = np.random.randint(0, w_i-8, size=2)
                i1, i2 = np.random.randint(0, w_o-8, size=2)
                data[k, :, i1:i1+8, i2:i2+8] += clt[:, c1:c1+8, c2:c2+8]
        data = np.clip(data, 0., 1.)
        return data

    process = clutter

# init RAM model
from ram import RAM
model = RAM(n_e=128, n_h=256, g_size=g_size, n_steps=n_steps,
            n_scales=n_scales, var=args.variance, use_lstm=args.lstm)

if not args.lstm:
    data = model.core_hh.W.data
    data[:] = np.identity(data.shape[0], dtype=np.float32)

if args.initmodel:
    print('load model from {}'.format(args.initmodel))
    serializers.load_hdf5(args.initmodel, model)

gpuid = args.gpuid
if gpuid >= 0:
    cuda.get_device(gpuid).use()
    model.to_gpu()

# set optimizer
from nesterov_ag import NesterovAG
lr_base = 1e-2
optimizer = NesterovAG(lr=lr_base)
optimizer.use_cleargrads()
optimizer.setup(model)

# get test scores

def test(x, t):
    batchsize = 1000
    sum_accuracy = sum_loss = 0
    with tqdm(total=len(t)) as pbar:
        pbar.set_description('test')
        for i in range(0, len(t), batchsize):
            pbar.update(batchsize)
            model(x[i:i+batchsize], t[i:i+batchsize], train=False)
            sum_loss += float(model.loss.data)
            sum_accuracy += float(model.accuracy.data)
    sys.stderr.flush()
    return sum_loss*batchsize / len(t), sum_accuracy*batchsize / len(t)

filename = args.filename + '_' + filename
log = open(filename+'.log', 'a')
writer = csv.writer(log, lineterminator='\n')
writer.writerow(('iter', 'loss', 'acc'))
test_data = process(test_data) # generate test data before training
test_data.flags.writeable = False
loss, acc = test(test_data, test_targets)
writer.writerow((0, loss, acc))
log.flush()
sys.stdout.write('test: loss={0:.6f}, accuracy={1:.6f}\n'.format(loss, acc))
sys.stdout.flush()

# optimize weights
batchsize = args.batchsize
n_data = len(train_targets)
n_epoch = args.epoch
lr_gamma = np.exp(-1*np.log(10) / n_epoch) # drop by 10^-1 for n_epoch

for epoch in range(n_epoch):
    sys.stdout.write('(epoch: {})\n'.format(epoch+1))
    sys.stdout.flush()

    optimizer.lr = lr_base * np.power(lr_gamma, epoch)
    print('learning rate: {:.3e}'.format(optimizer.lr))

    perm = np.random.permutation(n_data)
    with tqdm(total=n_data) as pbar:
        for i in range(0, n_data, batchsize):
            # generate train data on the fly
            x = process(train_data[perm[i:i+batchsize]])
            t = train_targets[perm[i:i+batchsize]]
            optimizer.update(model, x, t)
            pbar.set_description(
                ('train: loss={0:.1e}, b={1:.1e}, r={2:+.1e}').format(
                    float(model.loss_action.data),
                    float(model.loss_base.data), float(model.loss_reinforce.data)
                )
            )
            pbar.update(batchsize)
    sys.stderr.flush()

    # evaluate
    loss, acc = test(test_data, test_targets)
    writer.writerow((epoch+1, loss, acc))
    log.flush()
    sys.stdout.write('test: loss={0:.3f}, accuracy={1:.3f}\n'.format(loss, acc))
    sys.stdout.flush()

    # save model params
    if (epoch+1) % 100 == 0:
        model_filename = filename+'_epoch{0:d}'.format(epoch+1)
        serializers.save_hdf5(model_filename+'.chainermodel', model)

log.close()
<commit_msg>set hyper-params produce scores given in the paper<commit_after>import numpy as np
np.random.seed(777)
import chainer
from chainer import cuda
from chainer import optimizers
from chainer import serializers

from tqdm import tqdm
import argparse
import csv
import sys
import os

parser = argparse.ArgumentParser()
group = parser.add_mutually_exclusive_group()
group.add_argument('--original', action='store_true',
                    default=True, help='train on original MNIST')
group.add_argument('--translated', action='store_true',
                    default=False, help='train on translated MNIST')
group.add_argument('--cluttered', action='store_true',
                    default=False, help='train on translated & cluttered MNIST')
parser.add_argument('--lstm', type=bool, default=False,
                    help='use LSTM units in core layer')
parser.add_argument('-m', '--initmodel', type=str, default='',
                    help='load model weights from given file')
parser.add_argument('-g', '--gpuid', type=int, default=-1,
                    help='GPU device ID (default is CPU)')
parser.add_argument('-b', '--batchsize', type=int, default=100,
                    help='training batch size')
parser.add_argument('-v', '--variance', type=float, default=0.05,
                    help='variance of location policy')
parser.add_argument('-e', '--epoch', type=int, default=500,
                    help='iterate training given epoch times')
parser.add_argument('-f', '--filename', type=str, default='',
                    help='prefix of output filenames')
args = parser.parse_args()

# load mnist dataset
train, test = chainer.datasets.get_mnist()
train_data, train_targets = np.array(train).transpose()
test_data, test_targets = np.array(test).transpose()
train_data = np.array(list(train_data)).reshape(train_data.shape[0],1,28,28)
test_data = np.array(list(test_data)).reshape(test_data.shape[0],1,28,28)
train_data.flags.writeable = False
train_targets = np.array(train_targets).astype(np.int32)
test_targets = np.array(test_targets).astype(np.int32)

# hyper-params for each task
if args.original:
    filename = 'ram_original'
    # RAM params for original MNIST
    g_size = 8
    n_steps = 6
    n_scales = 1

    def process(batch):
        return batch

if args.translated:
    filename = 'ram_translated'
    g_size = 12
    n_steps = 8
    n_scales = 3

    # create translated MNIST
    def translate(batch):
        n, c, w_i = batch.shape[:3]
        w_o = 60
        data = np.zeros(shape=(n,c,w_o,w_o), dtype=np.float32)
        for k in range(n):
            i, j = np.random.randint(0, w_o-w_i, size=2)
            data[k, :, i:i+w_i, j:j+w_i] += batch[k]
        return data

    process = translate

if args.cluttered:
    filename = 'ram_cluttered'
    g_size = 12
    n_steps = 8
    n_scales = 3

    # create cluttered MNIST
    def clutter(batch):
        n, c, w_i = batch.shape[:3]
        w_o = 60
        data = np.zeros(shape=(n,c,w_o,w_o), dtype=np.float32)
        for k in range(n):
            i, j = np.random.randint(0, w_o-w_i, size=2)
            data[k, :, i:i+w_i, j:j+w_i] += batch[k]
            for _ in range(4):
                clt = train_data[np.random.randint(0, train_data.shape[0]-1)]
                c1, c2 = np.random.randint(0, w_i-8, size=2)
                i1, i2 = np.random.randint(0, w_o-8, size=2)
                data[k, :, i1:i1+8, i2:i2+8] += clt[:, c1:c1+8, c2:c2+8]
        data = np.clip(data, 0., 1.)
        return data

    process = clutter

# init RAM model
from ram import RAM
model = RAM(n_e=128, n_h=256, g_size=g_size, n_steps=n_steps,
            n_scales=n_scales, var=args.variance, use_lstm=args.lstm)

if not args.lstm:
    data = model.core_hh.W.data
    data[:] = np.identity(data.shape[0], dtype=np.float32)

if args.initmodel:
    print('load model from {}'.format(args.initmodel))
    serializers.load_hdf5(args.initmodel, model)

gpuid = args.gpuid
if gpuid >= 0:
    cuda.get_device(gpuid).use()
    model.to_gpu()

# set optimizer
from nesterov_ag import NesterovAG
lr_base = 1e-2
optimizer = NesterovAG(lr=lr_base)
optimizer.use_cleargrads()
optimizer.setup(model)

# get test scores
def test(x, t):
    batchsize = 1000
    sum_accuracy = sum_loss = 0
    with tqdm(total=len(t)) as pbar:
        pbar.set_description('test')
        for i in range(0, len(t), batchsize):
            pbar.update(batchsize)
            model(x[i:i+batchsize], t[i:i+batchsize], train=False)
            sum_loss += float(model.loss.data)
            sum_accuracy += float(model.accuracy.data)
    sys.stderr.flush()
    return sum_loss*batchsize / len(t), sum_accuracy*batchsize / len(t)

filename = args.filename + '_' + filename
log = open(filename+'.log', 'a')
writer = csv.writer(log, lineterminator='\n')
writer.writerow(('iter', 'loss', 'acc'))
test_data = process(test_data) # generate test data before training
test_data.flags.writeable = False
loss, acc = test(test_data, test_targets)
writer.writerow((0, loss, acc))
log.flush()
sys.stdout.write('test: loss={0:.6f}, accuracy={1:.6f}\n'.format(loss, acc))
sys.stdout.flush()

# optimize weights
batchsize = args.batchsize
n_data = len(train_targets)
n_epoch = args.epoch

for epoch in range(n_epoch):
    sys.stdout.write('(epoch: {})\n'.format(epoch+1))
    sys.stdout.flush()

    if args.original and epoch+1 > 400: optimizer.lr = lr_base * 0.1
    print('learning rate: {:.3e}'.format(optimizer.lr))

    perm = np.random.permutation(n_data)
    with tqdm(total=n_data) as pbar:
        for i in range(0, n_data, batchsize):
            # generate train data on the fly
            x = process(train_data[perm[i:i+batchsize]])
            t = train_targets[perm[i:i+batchsize]]
            optimizer.update(model, x, t)
            pbar.set_description(
                ('train: loss={0:.1e}, b={1:.1e}, r={2:+.1e}').format(
                    float(model.loss_action.data),
                    float(model.loss_base.data), float(model.loss_reinforce.data)
                )
            )
            pbar.update(batchsize)
    sys.stderr.flush()

    # evaluate
    loss, acc = test(test_data, test_targets)
    writer.writerow((epoch+1, loss, acc))
    log.flush()
    sys.stdout.write('test: loss={0:.3f}, accuracy={1:.3f}\n'.format(loss, acc))
    sys.stdout.flush()

    # save model params
    if (epoch+1) % 100 == 0:
        model_filename = filename+'_epoch{0:d}'.format(epoch+1)
        serializers.save_hdf5(model_filename+'.chainermodel', model)

log.close()
<|endoftext|>


<commit_before>import os

import math
import tensorflow as tf
import time
import click
import dill
import logging

from skipthought import SkipthoughtModel
from skipthought.data_utils import TextData

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

@click.command()
@click.option('--data_path', default='data/input.txt', type=str, help="Path to file with text data.")
@click.option('--save_dir', default='save/', type=str, help='Directory to store checkpointed models')
@click.option('--num_epochs', default=10, type=int, help="Number of epochs.")
@click.option('--num_layers', default=1, type=int, help="Number of layers in encoder.")
@click.option('--batch_size', default=128, type=int, help="The size of batch.")
@click.option('--num_hidden', default=512, type=int, help="Hidden size of the cell.")
@click.option('--cell_type', default='gru',
            type=click.Choice(SkipthoughtModel.SUPPORTED_CELLTYPES),
              help='Type of the RNN cell.')
@click.option('--embedding_size', default=1024, type=int, help="The size of word embeddings.")
@click.option('--max_vocab_size', default=100000, type=int, help="Size of vocabulary. Most frequent words are used.")
@click.option('--num_samples', default=512, type=int, help="Number of samples in sampled softmax.")
@click.option('--learning_rate', default=0.01, type=float, help="Initial learning rate.")
@click.option('--decay_rate', default=0.99, type=float, help="Exponential decay rate.")
@click.option('--grad_clip', default=5.0, type=float, help="Value for gradient clipping.")
@click.option('--save_every', default=1000, type=int, help="Number of batch steps before creating a model checkpoint")
@click.option('--verbose', default=100, type=int, help="How often to print batch loss and other info.")
@click.option('--init_from', default=None, help="""continue training from saved model at this path. Path must contain files saved by previous training process:
                            'config.pkl'        : configuration;
                            'chars_vocab.pkl'   : vocabulary definitions;
                            'checkpoint'        : paths to model file(s) (created by tf).
                                                  Note: this file contains absolute paths, be careful when moving files around;
                            'model.ckpt-*'      : file(s) with model definition (created by tf)
                        """)
def main(**kwargs):
    logger.info("Your params:")
    logger.info(kwargs)
    textdata = TextData(kwargs['data_path'], max_vocab_size=kwargs['max_vocab_size'])
    max_length_decoder = textdata.max_len

    # check compatibility if training is continued from previously saved model
    if kwargs['init_from'] is not None:
        logger.info("Check if I can restore model from {0}".format(kwargs['init_from']))
        # check if all necessary files exist
        assert os.path.isdir(kwargs['init_from']), "%s must be a a path" % kwargs['init_from']
        assert os.path.isfile(os.path.join(kwargs['init_from'], "config.pkl")), "config.pkl file does not exist in path %s" % kwargs['init_from']
        assert os.path.isfile(os.path.join(kwargs['init_from'], "textdata.pkl")), "textdata.pkl file does not exist in path %s" % kwargs['init_from']
        ckpt = tf.train.get_checkpoint_state(kwargs['init_from'])
        assert ckpt, "No checkpoint found"
        assert ckpt.model_checkpoint_path, "No model path found in checkpoint"

        # open old config and check if models are compatible
        with open(os.path.join(kwargs['init_from'], 'config.pkl'), 'rb') as f:
            saved_model_args = dill.load(f)
            need_be_same = ["cell_type", "num_hidden", "num_layers", "grad_clip", "num_samples",
                            "max_vocab_size"]
            for checkme in need_be_same:
                assert saved_model_args[checkme] == kwargs[checkme], "Command line argument and saved model disagree on '%s' " % checkme
        logger.info("Args checker. Load TextData")
        # open saved TextData
        textdata = TextData.load(os.path.join(kwargs['init_from'], 'textdata.pkl'))

    logger.info("Save config and textdata.")
    with open(os.path.join(kwargs['save_dir'], 'config.pkl'), 'wb') as f:
        dill.dump(kwargs, f)
    TextData.save(textdata, os.path.join(kwargs['save_dir'], 'textdata.pkl'))

    # Make triples.
    logger.info("Making triples")
    triples = textdata.make_triples(textdata.dataset)
    logger.info("Number of triples: {0}".format(len(triples[0])))
    decay_steps = len(triples[0])

    model = SkipthoughtModel(kwargs['cell_type'], kwargs['num_hidden'], kwargs['num_layers'],
                             kwargs['embedding_size'], kwargs['max_vocab_size'], kwargs['learning_rate'],
                             kwargs['decay_rate'], decay_steps, kwargs['grad_clip'],
                             kwargs['num_samples'], max_length_decoder)

    with tf.Session(config=tf.ConfigProto(log_device_placement=True)) as sess:
        init = tf.initialize_all_variables()
        sess.run(init)
        saver = tf.train.Saver(tf.all_variables(), max_to_keep=20)

        if kwargs['init_from'] is not None:
            saver.restore(sess, ckpt.model_checkpoint_path)

        num_batches = len(triples[0])//kwargs['batch_size']
        loss_history = []
        for e in range(kwargs['num_epochs']):
            it = textdata.triples_data_iterator(triples[0], triples[1], triples[2],
                                                textdata.max_len, kwargs['batch_size'])
            for b, batch in enumerate(it):
                train_op, loss, feed_dict = model.train_step(*batch)

                start_time = time.time()
                batch_loss, _ = sess.run([loss, train_op], feed_dict=feed_dict)
                batch_perplexity = math.exp(float(batch_loss)) if batch_loss < 300 else float("inf")
                end_time = time.time()

                loss_history.append(batch_loss)
                if b % kwargs['verbose'] == 0:
                            print(
                                "{}/{} (epoch {}), train_loss = {:.3f}, perplexity = {:.3f}, time/batch = {:.3f}" \
                                .format(e * num_batches + b,
                                        kwargs['num_epochs'] * num_batches,
                                        e, batch_loss, batch_perplexity, end_time - start_time))
                if (e * num_batches + b) % kwargs['save_every'] == 0 \
                        or (e == kwargs['num_epochs']-1 and b == num_batches-1): # save for the last result
                    checkpoint_path = os.path.join(kwargs['save_dir'], 'model.ckpt')
                    saver.save(sess, checkpoint_path, global_step=e * num_batches + b)
                    print("model saved to {}".format(checkpoint_path))

if __name__ == "__main__":
    main()
<commit_msg>Additional print when restore model<commit_after>import os

import math
import tensorflow as tf
import time
import click
import dill
import logging

from skipthought import SkipthoughtModel
from skipthought.data_utils import TextData

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

@click.command()
@click.option('--data_path', default='data/input.txt', type=str, help="Path to file with text data.")
@click.option('--save_dir', default='save/', type=str, help='Directory to store checkpointed models')
@click.option('--num_epochs', default=10, type=int, help="Number of epochs.")
@click.option('--num_layers', default=1, type=int, help="Number of layers in encoder.")
@click.option('--batch_size', default=128, type=int, help="The size of batch.")
@click.option('--num_hidden', default=512, type=int, help="Hidden size of the cell.")
@click.option('--cell_type', default='gru',
            type=click.Choice(SkipthoughtModel.SUPPORTED_CELLTYPES),
              help='Type of the RNN cell.')
@click.option('--embedding_size', default=1024, type=int, help="The size of word embeddings.")
@click.option('--max_vocab_size', default=100000, type=int, help="Size of vocabulary. Most frequent words are used.")
@click.option('--num_samples', default=512, type=int, help="Number of samples in sampled softmax.")
@click.option('--learning_rate', default=0.01, type=float, help="Initial learning rate.")
@click.option('--decay_rate', default=0.99, type=float, help="Exponential decay rate.")
@click.option('--grad_clip', default=5.0, type=float, help="Value for gradient clipping.")
@click.option('--save_every', default=1000, type=int, help="Number of batch steps before creating a model checkpoint")
@click.option('--verbose', default=100, type=int, help="How often to print batch loss and other info.")
@click.option('--init_from', default=None, help="""continue training from saved model at this path. Path must contain files saved by previous training process:
                            'config.pkl'        : configuration;
                            'chars_vocab.pkl'   : vocabulary definitions;
                            'checkpoint'        : paths to model file(s) (created by tf).
                                                  Note: this file contains absolute paths, be careful when moving files around;
                            'model.ckpt-*'      : file(s) with model definition (created by tf)
                        """)
def main(**kwargs):
    logger.info("Your params:")
    logger.info(kwargs)
    textdata = TextData(kwargs['data_path'], max_vocab_size=kwargs['max_vocab_size'])
    max_length_decoder = textdata.max_len

    # check compatibility if training is continued from previously saved model
    if kwargs['init_from'] is not None:
        logger.info("Check if I can restore model from {0}".format(kwargs['init_from']))
        # check if all necessary files exist
        assert os.path.isdir(kwargs['init_from']), "%s must be a a path" % kwargs['init_from']
        assert os.path.isfile(os.path.join(kwargs['init_from'], "config.pkl")), "config.pkl file does not exist in path %s" % kwargs['init_from']
        assert os.path.isfile(os.path.join(kwargs['init_from'], "textdata.pkl")), "textdata.pkl file does not exist in path %s" % kwargs['init_from']
        ckpt = tf.train.get_checkpoint_state(kwargs['init_from'])
        assert ckpt, "No checkpoint found"
        assert ckpt.model_checkpoint_path, "No model path found in checkpoint"

        # open old config and check if models are compatible
        with open(os.path.join(kwargs['init_from'], 'config.pkl'), 'rb') as f:
            saved_model_args = dill.load(f)
            need_be_same = ["cell_type", "num_hidden", "num_layers", "grad_clip", "num_samples",
                            "max_vocab_size"]
            for checkme in need_be_same:
                assert saved_model_args[checkme] == kwargs[checkme], "Command line argument and saved model disagree on '%s' " % checkme
        logger.info("Args checker. Load TextData")
        # open saved TextData
        textdata = TextData.load(os.path.join(kwargs['init_from'], 'textdata.pkl'))

    logger.info("Save config and textdata.")
    with open(os.path.join(kwargs['save_dir'], 'config.pkl'), 'wb') as f:
        dill.dump(kwargs, f)
    TextData.save(textdata, os.path.join(kwargs['save_dir'], 'textdata.pkl'))

    # Make triples.
    logger.info("Making triples")
    triples = textdata.make_triples(textdata.dataset)
    logger.info("Number of triples: {0}".format(len(triples[0])))
    decay_steps = len(triples[0])

    model = SkipthoughtModel(kwargs['cell_type'], kwargs['num_hidden'], kwargs['num_layers'],
                             kwargs['embedding_size'], kwargs['max_vocab_size'], kwargs['learning_rate'],
                             kwargs['decay_rate'], decay_steps, kwargs['grad_clip'],
                             kwargs['num_samples'], max_length_decoder)

    with tf.Session(config=tf.ConfigProto(log_device_placement=True)) as sess:
        init = tf.initialize_all_variables()
        sess.run(init)
        saver = tf.train.Saver(tf.all_variables(), max_to_keep=20)

        if kwargs['init_from'] is not None:
            saver.restore(sess, ckpt.model_checkpoint_path)
            print("Restored from {0}".format(ckpt.model_checkpoint_path))

        num_batches = len(triples[0])//kwargs['batch_size']
        loss_history = []
        for e in range(kwargs['num_epochs']):
            it = textdata.triples_data_iterator(triples[0], triples[1], triples[2],
                                                textdata.max_len, kwargs['batch_size'])
            for b, batch in enumerate(it):
                train_op, loss, feed_dict = model.train_step(*batch)

                start_time = time.time()
                batch_loss, _ = sess.run([loss, train_op], feed_dict=feed_dict)
                batch_perplexity = math.exp(float(batch_loss)) if batch_loss < 300 else float("inf")
                end_time = time.time()

                loss_history.append(batch_loss)
                if b % kwargs['verbose'] == 0:
                            print(
                                "{}/{} (epoch {}), train_loss = {:.3f}, perplexity = {:.3f}, time/batch = {:.3f}" \
                                .format(e * num_batches + b,
                                        kwargs['num_epochs'] * num_batches,
                                        e, batch_loss, batch_perplexity, end_time - start_time))
                if (e * num_batches + b) % kwargs['save_every'] == 0 \
                        or (e == kwargs['num_epochs']-1 and b == num_batches-1): # save for the last result
                    checkpoint_path = os.path.join(kwargs['save_dir'], 'model.ckpt')
                    saver.save(sess, checkpoint_path, global_step=e * num_batches + b)
                    print("model saved to {}".format(checkpoint_path))

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>"""
Program to build a Corpus object, train LDA models and finally build a config
file for the topic explorer.
"""
import multiprocessing
import os
import os.path
import sys

from vsm.corpus import Corpus
from vsm.corpus.util.corpusbuilders import coll_corpus, dir_corpus, toy_corpus
from vsm.model.ldacgsmulti import LdaCgsMulti as LDA
from vsm.viewer.ldagibbsviewer import LDAGibbsViewer

def get_corpus_filename(corpus_path, model_path, nltk_stop=True, stop_freq=1,
			context_type='document'):
    corpus_name = os.path.basename(corpus_path)
    if not corpus_name:
        corpus_name = os.path.basename(os.path.dirname(corpus_path))
    if nltk_stop and stop_freq:
        filename = '%s-nltk-en-freq%d.npz' % (corpus_name, stop_freq)
    elif stop_freq:
        filename = '%s-freq%d.npz' % (corpus_name, stop_freq)
    else:
        filename = '%s.npz' % corpus_name
    return os.path.join(model_path, filename)

def build_corpus(corpus_path, model_path, nltk_stop=True, stop_freq=1,
    context_type='document', ignore=['.json','.log','.err','.pickle','.npz']):
    if os.path.isfile(corpus_path):
        print "Constructing toy corpus, each line is a document"
        c = toy_corpus(corpus_path, is_filename=True, nltk_stop=nltk_stop,
                       stop_freq=stop_freq, context_type=context_type)
    elif os.path.isdir(corpus_path):
        contents = os.listdir(corpus_path)
        contents = [os.path.join(corpus_path,obj) for obj in contents
            if not any([obj.endswith(suffix) for suffix in ignore])]
        count_dirs = len(filter(os.path.isdir, contents))
        count_files = len(filter(os.path.isfile, contents))

        print "Detected %d folders and %d files in %s" %\
            (count_dirs, count_files, corpus_path)

        if count_files > 0 and count_dirs == 0:
            print "Constructing directory corpus, each file is a document"
            c = dir_corpus(corpus_path, nltk_stop=nltk_stop,
                           stop_freq=stop_freq, chunk_name=context_type,
                           ignore=ignore)
        elif count_dirs > 0 and count_files == 0:
            print "Constructing collection corpus, each folder is a document"
            context_type='book'
            c = coll_corpus(corpus_path, nltk_stop=nltk_stop,
                            stop_freq=stop_freq, ignore=ignore)
        else:
            raise IOError("Invalid Path: empty directory")
    else:
        raise IOError("Invalid path")

    filename = get_corpus_filename(
        corpus_path, model_path, nltk_stop, stop_freq, context_type)
    c.save(filename)
    return filename

def build_models(corpus_filename, model_path, krange, n_iterations=200,
                 n_proc=2):

    corpus = Corpus.load(corpus_filename)
    if 'book' in corpus.context_types:
        corpus_type = 'book'
    else:
        corpus_type = 'document'

    basefilename = os.path.basename(corpus_filename).replace('.npz','')
    basefilename += "-LDA-K%s-%s-%d.npz" % ('{0}', corpus_type, n_iterations)
    basefilename = os.path.join(model_path, basefilename)

    for k in krange:
        print "Training model for k={0} Topics with {1} Processes"\
            .format(k, n_proc)
        m = LDA(corpus, corpus_type, K=k)
        m.train(n_iterations=n_iterations, n_proc=n_proc)
        m.save(basefilename.format(k))

    return basefilename

if __name__ == '__main__':
    from argparse import ArgumentParser
    from ConfigParser import RawConfigParser as ConfigParser
    parser = ArgumentParser()
    parser.add_argument("corpus_path", help="Path to Corpus")
    parser.add_argument("--model-path", dest="model_path",
        help="Model Path [Default: [corpus_path]/../models]")
    parser.add_argument("--htrc", action="store_true")
    parser.add_argument("-p", "--processes", default=2, type=int,
        help="Number of CPU cores for training [Default: 2]")
    parser.add_argument("--port", default=16000, type=int,
        help="Default port [Default: 16000]")
    parser.add_argument("-k", nargs='+',
        help="K values to train upon", type=int)
    parser.add_argument("--retrain", action="store_true")
    parser.add_argument('--iter', type=int,
        help="Number of training iterations")
    parser.add_argument('--dry-run', action="store_true",
        help="Only create corpus object and config file, not models")
    args = parser.parse_args()

    if args.model_path is None:
        args.model_path = os.path.join(args.corpus_path, '../models/')
    if not os.path.exists(args.model_path):
        os.makedirs(args.model_path)

    if args.k is None:
        args.k = range(120,0,-20)

    if args.iter is None and not args.dry_run:
        while args.iter is None:
            iters = raw_input("Number of Training Iterations [Default 200]: ")
            try:
                args.iter = int(iters)
            except ValueError:
                if iters.strip() == '':
                    args.iter = 200
                else:
                    print "Enter a valid integer!"

        print "\nTIP: number of training iterations can be specified with argument '--iter N':"
        print "python train.py --iter %d %s\n" % (args.iter, args.corpus_path)

    if args.processes < 0:
        args.processes = multiprocessing.cpu_count() + args.processes

    corpus_name = os.path.basename(args.corpus_path)
    if not corpus_name:
        corpus_name = os.path.basename(os.path.dirname(args.corpus_path))

    retrain = None
    corpus_filename = get_corpus_filename(
        args.corpus_path, args.model_path, stop_freq=5)
    if not args.retrain and os.path.exists(corpus_filename):
        while retrain not in ['y', 'n']:
            retrain = raw_input("\nCorpus file found. Rebuild? [y/n] ")
            if retrain == 'y':
	        retrain = True
    else:
        retrain = True
    if retrain == True:
        try:
            corpus_filename = build_corpus(args.corpus_path, args.model_path,
                                           stop_freq=5)
        except IOError:
            print "ERROR: invalid path, please specify either:"
            print "  * a single plain-text file,"
            print "  * a folder of plain-text files, or"
            print "  * a folder of folders of plain-text files."
            print "\nExiting..."
            sys.exit(74)

    if args.dry_run:
        model_pattern = build_models(corpus_filename, args.model_path, list(),
                                     n_iterations=args.iter, n_proc=args.processes)
    else:
        model_pattern = build_models(corpus_filename, args.model_path, args.k,
                                     n_iterations=args.iter, n_proc=args.processes)

    corpus = Corpus.load(corpus_filename)
    if 'book' in corpus.context_types:
        corpus_type = 'book'
    else:
        corpus_type = 'document'

    config = ConfigParser()
    config.add_section("main")
    config.set("main", "path", args.model_path)
    config.set("main", "corpus_file", corpus_filename)
    config.set("main", "context_type", corpus_type)
    config.set("main", "model_pattern", model_pattern)
    config.set("main", "port", "16{0:03d}")
    config.set("main", "host", "0.0.0.0")
    args.k.sort()
    config.set("main", "topics", args.k)
    config.add_section("www")
    config.set("www", "corpus_name", "Deafult")
    config.set("www", "icons", "link")
    config.add_section("logging")
    config.set("logging","path","logs/%s/{0}.log" % corpus_name)

    if args.htrc:
        config.set("main","label_module","extensions.htrc")
        config.set("www","corpus_name","HTRC Data Capsule")
        config.set("www","doc_title_format",'{0}')
        config.set("www","doc_url_format", 'http://hdl.handle.net/2027/{0}')
        config.set("www", "icons", "htrc,link")

    configfile = corpus_name + ".ini"
    config_i = 0
    while os.path.exists(configfile):
        configfile = corpus_name + ".%d.ini" % config_i
        config_i += 1

    print "\nWriting configuration file", configfile
    with open(configfile, "wb") as configfh:
        config.write(configfh)

    if args.dry_run:
        print "\nTIP: Dry run, only initalizing corpus object and config file."
        print "     Next prepare the corpus using:"
        print "python corpus_prep.py", configfile
    else:
        print "\nTIP: launch the topic explorer with:"
        print "python launch.py", configfile
<commit_msg>adding command line argument for random seed<commit_after>"""
Program to build a Corpus object, train LDA models and finally build a config
file for the topic explorer.
"""
import multiprocessing
import os
import os.path
import sys

from vsm.corpus import Corpus
from vsm.corpus.util.corpusbuilders import coll_corpus, dir_corpus, toy_corpus
from vsm.model.ldacgsmulti import LdaCgsMulti as LDA
from vsm.viewer.ldagibbsviewer import LDAGibbsViewer

def get_corpus_filename(corpus_path, model_path, nltk_stop=True, stop_freq=1,
			context_type='document'):
    corpus_name = os.path.basename(corpus_path)
    if not corpus_name:
        corpus_name = os.path.basename(os.path.dirname(corpus_path))
    if nltk_stop and stop_freq:
        filename = '%s-nltk-en-freq%d.npz' % (corpus_name, stop_freq)
    elif stop_freq:
        filename = '%s-freq%d.npz' % (corpus_name, stop_freq)
    else:
        filename = '%s.npz' % corpus_name
    return os.path.join(model_path, filename)

def build_corpus(corpus_path, model_path, nltk_stop=True, stop_freq=1,
    context_type='document', ignore=['.json','.log','.err','.pickle','.npz']):
    if os.path.isfile(corpus_path):
        print "Constructing toy corpus, each line is a document"
        c = toy_corpus(corpus_path, is_filename=True, nltk_stop=nltk_stop,
                       stop_freq=stop_freq, context_type=context_type)
    elif os.path.isdir(corpus_path):
        contents = os.listdir(corpus_path)
        contents = [os.path.join(corpus_path,obj) for obj in contents
            if not any([obj.endswith(suffix) for suffix in ignore])]
        count_dirs = len(filter(os.path.isdir, contents))
        count_files = len(filter(os.path.isfile, contents))

        print "Detected %d folders and %d files in %s" %\
            (count_dirs, count_files, corpus_path)

        if count_files > 0 and count_dirs == 0:
            print "Constructing directory corpus, each file is a document"
            c = dir_corpus(corpus_path, nltk_stop=nltk_stop,
                           stop_freq=stop_freq, chunk_name=context_type,
                           ignore=ignore)
        elif count_dirs > 0 and count_files == 0:
            print "Constructing collection corpus, each folder is a document"
            context_type='book'
            c = coll_corpus(corpus_path, nltk_stop=nltk_stop,
                            stop_freq=stop_freq, ignore=ignore)
        else:
            raise IOError("Invalid Path: empty directory")
    else:
        raise IOError("Invalid path")

    filename = get_corpus_filename(
        corpus_path, model_path, nltk_stop, stop_freq, context_type)
    c.save(filename)
    return filename

def build_models(corpus_filename, model_path, krange, n_iterations=200,
                 n_proc=2, seed=None):

    corpus = Corpus.load(corpus_filename)
    if 'book' in corpus.context_types:
        corpus_type = 'book'
    else:
        corpus_type = 'document'

    basefilename = os.path.basename(corpus_filename).replace('.npz','')
    basefilename += "-LDA-K%s-%s-%d.npz" % ('{0}', corpus_type, n_iterations)
    basefilename = os.path.join(model_path, basefilename)

    if type(seed) == int:
        seeds = [seed + p for p in range(n_proc)]
        fileparts = basefilename.split('-')
        fileparts.insert(-1, str(seed))
        basefilename = '-'.join(fileparts)
    else:
        seeds = None

    for k in krange:
        print "Training model for k={0} Topics with {1} Processes"\
            .format(k, n_proc)
        m = LDA(corpus, corpus_type, K=k)
        m.train(n_iterations=n_iterations, n_proc=n_proc, seeds=seeds)
        m.save(basefilename.format(k))

    return basefilename

if __name__ == '__main__':
    from argparse import ArgumentParser
    from ConfigParser import RawConfigParser as ConfigParser
    parser = ArgumentParser()
    parser.add_argument("corpus_path", help="Path to Corpus")
    parser.add_argument("--model-path", dest="model_path",
        help="Model Path [Default: [corpus_path]/../models]")
    parser.add_argument("--htrc", action="store_true")
    parser.add_argument("-p", "--processes", default=2, type=int,
        help="Number of CPU cores for training [Default: 2]")
    parser.add_argument("--port", default=16000, type=int,
        help="Default port [Default: 16000]")
    parser.add_argument("--seed", default=None, type=int,
        help="Random seed for topic modeling [Default: None]")
    parser.add_argument("-k", nargs='+',
        help="K values to train upon", type=int)
    parser.add_argument("--retrain", action="store_true")
    parser.add_argument('--iter', type=int,
        help="Number of training iterations")
    parser.add_argument('--dry-run', action="store_true",
        help="Only create corpus object and config file, not models")
    args = parser.parse_args()

    if args.model_path is None:
        args.model_path = os.path.join(args.corpus_path, '../models/')
    if not os.path.exists(args.model_path):
        os.makedirs(args.model_path)

    if args.k is None:
        args.k = range(120,0,-20)

    if args.iter is None and not args.dry_run:
        while args.iter is None:
            iters = raw_input("Number of Training Iterations [Default 200]: ")
            try:
                args.iter = int(iters)
            except ValueError:
                if iters.strip() == '':
                    args.iter = 200
                else:
                    print "Enter a valid integer!"

        print "\nTIP: number of training iterations can be specified with argument '--iter N':"
        print "python train.py --iter %d %s\n" % (args.iter, args.corpus_path)

    if args.processes < 0:
        args.processes = multiprocessing.cpu_count() + args.processes

    corpus_name = os.path.basename(args.corpus_path)
    if not corpus_name:
        corpus_name = os.path.basename(os.path.dirname(args.corpus_path))

    retrain = None
    corpus_filename = get_corpus_filename(
        args.corpus_path, args.model_path, stop_freq=5)
    if not args.retrain and os.path.exists(corpus_filename):
        while retrain not in ['y', 'n']:
            retrain = raw_input("\nCorpus file found. Rebuild? [y/n] ")
            if retrain == 'y':
	        retrain = True
    else:
        retrain = True
    if retrain == True:
        try:
            corpus_filename = build_corpus(args.corpus_path, args.model_path,
                                           stop_freq=5)
        except IOError:
            print "ERROR: invalid path, please specify either:"
            print "  * a single plain-text file,"
            print "  * a folder of plain-text files, or"
            print "  * a folder of folders of plain-text files."
            print "\nExiting..."
            sys.exit(74)

    if args.dry_run:
        model_pattern = build_models(corpus_filename, args.model_path, list(),
                                     n_iterations=args.iter,
                                     n_proc=args.processes, seed=args.seed)
    else:
        model_pattern = build_models(corpus_filename, args.model_path, args.k,
                                     n_iterations=args.iter,
                                     n_proc=args.processes, seed=args.seed)

    corpus = Corpus.load(corpus_filename)
    if 'book' in corpus.context_types:
        corpus_type = 'book'
    else:
        corpus_type = 'document'

    config = ConfigParser()
    config.add_section("main")
    config.set("main", "path", args.model_path)
    config.set("main", "corpus_file", corpus_filename)
    config.set("main", "context_type", corpus_type)
    config.set("main", "model_pattern", model_pattern)
    config.set("main", "port", "16{0:03d}")
    config.set("main", "host", "0.0.0.0")
    args.k.sort()
    config.set("main", "topics", args.k)
    config.add_section("www")
    config.set("www", "corpus_name", "Deafult")
    config.set("www", "icons", "link")
    config.add_section("logging")
    config.set("logging","path","logs/%s/{0}.log" % corpus_name)

    if args.htrc:
        config.set("main","label_module","extensions.htrc")
        config.set("www","corpus_name","HTRC Data Capsule")
        config.set("www","doc_title_format",'{0}')
        config.set("www","doc_url_format", 'http://hdl.handle.net/2027/{0}')
        config.set("www", "icons", "htrc,link")

    configfile = corpus_name + ".ini"
    config_i = 0
    while os.path.exists(configfile):
        configfile = corpus_name + ".%d.ini" % config_i
        config_i += 1

    print "\nWriting configuration file", configfile
    with open(configfile, "wb") as configfh:
        config.write(configfh)

    if args.dry_run:
        print "\nTIP: Dry run, only initalizing corpus object and config file."
        print "     Next prepare the corpus using:"
        print "python corpus_prep.py", configfile
    else:
        print "\nTIP: launch the topic explorer with:"
        print "python launch.py", configfile
<|endoftext|>