<commit_before>import os
# os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
# os.environ["CUDA_VISIBLE_DEVICES"] = "1"

from src.python.consts import *

from src.python.preprocess import *

from src.python.geneontology import *

from src.python.baselines import *

from pymongo import MongoClient

from tqdm import tqdm

import tensorflow as tf

### Keras
from keras import optimizers
from keras.models import Model
from keras.layers import Input, Dense, Embedding, Activation
from keras.layers import Conv2D, Conv1D
from keras.layers import Dropout, BatchNormalization
from keras.layers import MaxPooling2D, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.layers import Concatenate, Flatten, Reshape
from keras.callbacks import Callback, EarlyStopping, ModelCheckpoint, LambdaCallback, LearningRateScheduler
# from keras.losses import hinge, binary_crossentropy

from keras import backend as K

from sklearn.metrics import log_loss

import math

import argparse

sess = tf.Session()
K.set_session(sess)

LR = 0.001

BATCH_SIZE = 32

t0 = datetime(2014, 1, 1, 0, 0)
t1 = datetime(2014, 9, 1, 0, 0)

MAX_LENGTH = 2000
MIN_LENGTH = 100

def get_training_and_validation_streams(db, onto, classes, limit=None):
    q_train = {'DB': 'UniProtKB',
               'Evidence': {'$in': exp_codes},
               'Date': {"$lte": t0},
               'Aspect': ASPECT}
    seq2go_trn, _ = GoAnnotationCollectionLoader(db.goa_uniprot.find(q_train), db.goa_uniprot.count(q_train), ASPECT).load()
    query = {"_id": {"$in": unique(list(seq2go_trn.keys())).tolist()}}
    count = limit if limit else db.uniprot.count(query)
    source = db.uniprot.find(query).batch_size(10)
    if limit: source = source.limit(limit)
    stream_trn = DataStream(source, count, seq2go_trn, onto, classes)

    q_valid = {'DB': 'UniProtKB',
               'Evidence': {'$in': exp_codes},
               'Date': {"$gt": t0, "$lte": t1},
               'Aspect': ASPECT}

    seq2go_tst, _ = GoAnnotationCollectionLoader(db.goa_uniprot.find(q_valid), db.goa_uniprot.count(q_valid), ASPECT).load()
    query = {"_id": {"$in": unique(list(seq2go_tst.keys())).tolist()}}
    count = limit if limit else db.uniprot.count(query)
    source = db.uniprot.find(query).batch_size(10)
    if limit: source = source.limit(limit)
    stream_tst = DataStream(source, count, seq2go_tst, onto, classes)

    return stream_trn, stream_tst

def pad_seq(seq, max_length=MAX_LENGTH):
    seq += [PAD for _ in range(max_length - len(seq))]
    return np.asarray(seq)

class DataStream(object):
    def __init__(self, source, count, seq2go, onto, classes):

        self._classes = classes
        self._count = count
        self._source = source
        self._seq2go = seq2go
        self._onto = onto

    def __iter__(self):

        classes = self._classes
        count = self._count
        source = self._source
        seq2go = self._seq2go
        onto = self._onto

        s_cls = set(classes)

        for k, seq in UniprotCollectionLoader(source, count):
            if not MIN_LENGTH // 4 <= len(seq) <= MAX_LENGTH:
                continue
            y = np.zeros(len(classes))
            for go in onto.propagate(seq2go[k], include_root=False):
                if go not in s_cls:
                    continue
                y[classes.index(go)] = 1

                x = [AA.aa2index[aa] for aa in seq]

            yield k, x, y

    def __len__(self):
        return self._count

def step_decay(epoch):
    initial_lrate = LR
    drop = 0.5
    epochs_drop = 1.0
    lrate = max(0.0001, initial_lrate * math.pow(drop, math.floor((1 + epoch) / epochs_drop)))
    return lrate

def Features(inpt):

    feats = Embedding(input_dim=26, output_dim=20, embeddings_initializer='uniform')(inpt)

    feats = Conv1D(500, 30, activation='relu', padding='valid')(feats)
    feats = Dropout(0.3)(feats)
    feats = Conv1D(250, 10, activation='relu', padding='valid')(feats)
    feats = Dropout(0.3)(feats)
    feats = Conv1D(250, 10, activation='relu', padding='valid')(feats)
    feats = Dropout(0.3)(feats)
    feats = Conv1D(250, 10, activation='relu', padding='valid')(feats)
    feats = Dropout(0.3)(feats)
    feats = Conv1D(250, 10, activation='relu', padding='valid')(feats)
    feats = Dropout(0.3)(feats)

    return feats

def Classifier(inpt, classes):
    out = GlobalMaxPooling1D()(inpt)
    out = Dense(len(classes), activation='linear')(out)
    out = BatchNormalization()(out)
    out = Activation('sigmoid')(out)
    return out

def ModelCNN(classes):
    inp = Input(shape=(None,))
    out = Classifier(Features(inp), classes)
    model = Model(inputs=[inp], outputs=[out])
    # sgd = optimizers.SGD(lr=LR, decay=1e-6, momentum=0.9, nesterov=True)
    adam = optimizers.Adam(lr=LR, beta_1=0.9, beta_2=0.999, epsilon=1e-8)
    model.compile(loss='binary_crossentropy', optimizer=adam)

    return model

def batch_generator(stream):

    def prepare(batch):
        X, Y = zip(*batch)
        b = max(MIN_LENGTH, max(map(len, X)))
        X = [pad_seq(seq, b) for seq in X]
        return np.asarray(X), np.asarray(Y)

    batch = []
    for _, x, y in stream:
        if len(batch) == BATCH_SIZE:
            yield prepare(batch)
            batch = []
        batch.append([x, y])

    yield prepare(batch)

def add_arguments(parser):
    parser.add_argument("--mongo_url", type=str, default='mongodb://localhost:27017/',
                        help="Supply the URL of MongoDB"),
    parser.add_argument("--aspect", type=str, choices=['F', 'P', 'C'],
                        default="F", help="Specify the ontology aspect.")
    parser.add_argument("--init_epoch", type=int, default=0,
                        help="Which epoch to start training the model?")
    parser.add_argument("--num_epochs", type=int, default=20,
                        help="How many epochs to train the model?")
    parser.add_argument('-r', '--resume', default='', type=str, metavar='PATH',
                        help='path to latest checkpoint (default: none)')

class LossHistory(Callback):
    def __init__(self):
        self.losses = []

    def on_batch_end(self, batch, logs={}):
        self.losses.append(logs.get('loss'))

def train(model, gen_xy, length_xy, epoch, num_epochs,
          history=LossHistory(), lrate=LearningRateScheduler(step_decay)):

    pbar = tqdm(total=length_xy)

    for X, Y in gen_xy:
        assert len(X) == len(Y)

        model.fit(x=X, y=Y,
                  batch_size=BATCH_SIZE,
                  epochs=num_epochs,
                  verbose=0,
                  validation_data=None,
                  initial_epoch=epoch,
                  callbacks=[history])
        pbar.set_description("Training Loss:%.5f" % np.mean(history.losses))
        pbar.update(len(Y))

    pbar.close()

def zeroone2oneminusone(vec):
    return np.add(np.multiply(np.array(vec), 2), -1)

def oneminusone2zeroone(vec):
    return np.divide(np.add(np.array(vec), 1), 2)

def calc_loss(y_true, y_pred, batch_size=BATCH_SIZE):
    return batch_size * np.mean([log_loss(y, y_hat) for y, y_hat in zip(y_true, y_pred) if np.any(y)])

def predict(model, gen_xy, length_xy, classes):
    pbar = tqdm(total=length_xy, desc="Predicting...")
    i, m, n = 0, length_xy, len(classes)
    y_pred, y_true = np.zeros((m, n)), np.zeros((m, n))
    for i, (X, Y) in enumerate(gen_xy):
        assert len(X) == len(Y)
        k = len(Y)
        y_hat, y = model.predict(X), Y
        y_pred[i:i + k, ], y_true[i:i + k, ] = y_hat, y
        pbar.update(k)
    pbar.close()
    return y_true, y_pred

def evaluate(y_true, y_pred, classes):

    y_pred = y_pred[~np.all(y_pred == 0, axis=1)]
    y_true = y_true[~np.all(y_true == 0, axis=1)]

    f_max = F_max(y_pred, y_true, classes, np.arange(0.1, 1, 0.1))

    return calc_loss(y_true, y_pred), f_max

if __name__ == "__main__":

    parser = argparse.ArgumentParser()
    add_arguments(parser)
    args = parser.parse_args()

    ASPECT = args.aspect  # default: Molecular Function

    client = MongoClient(args.mongo_url)

    db = client['prot2vec']

    print("Loading Ontology...")
    onto = get_ontology(ASPECT)

    classes = onto.classes
    classes.remove(onto.root)
    assert onto.root not in classes

    model = ModelCNN(classes)
    if args.resume:
        model.load_weights(args.resume)
        print("Loaded model from disk")
    model.summary()

    for epoch in range(args.init_epoch, args.num_epochs):

        trn_stream, tst_stream = get_training_and_validation_streams(db, onto, classes)

        train(model, batch_generator(trn_stream), len(trn_stream), epoch, args.num_epochs)
        y_true, y_pred = predict(model, batch_generator(tst_stream), len(tst_stream), classes)
        loss, f_max = evaluate(y_true, y_pred, classes)

        print("[Epoch %d] (Validation Loss: %.5f, F_max: %.3f)" % (epoch + 1, loss, f_max))

        model_path = 'checkpoints/deeperseq-%d-%.5f-%.2f.hdf5' % (epoch + 1, loss, f_max)
        model.save_weights(model_path)
<commit_msg>deeperseq ...<commit_after>import os
# os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
# os.environ["CUDA_VISIBLE_DEVICES"] = "1"

from src.python.consts import *

from src.python.preprocess import *

from src.python.geneontology import *

from src.python.baselines import *

from pymongo import MongoClient

from tqdm import tqdm

import tensorflow as tf

### Keras
from keras import optimizers
from keras.models import Model
from keras.layers import Input, Dense, Embedding, Activation
from keras.layers import Conv2D, Conv1D
from keras.layers import Dropout, BatchNormalization
from keras.layers import MaxPooling2D, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.layers import Concatenate, Flatten, Reshape
from keras.callbacks import Callback, EarlyStopping, ModelCheckpoint, LambdaCallback, LearningRateScheduler
# from keras.losses import hinge, binary_crossentropy

from keras import backend as K

from sklearn.metrics import log_loss

import math

import argparse

sess = tf.Session()
K.set_session(sess)

LR = 0.001

BATCH_SIZE = 32

t0 = datetime(2014, 1, 1, 0, 0)
t1 = datetime(2014, 9, 1, 0, 0)

MAX_LENGTH = 2000
MIN_LENGTH = 100

def get_training_and_validation_streams(db, onto, classes, limit=None):
    q_train = {'DB': 'UniProtKB',
               'Evidence': {'$in': exp_codes},
               'Date': {"$lte": t0},
               'Aspect': ASPECT}
    seq2go_trn, _ = GoAnnotationCollectionLoader(db.goa_uniprot.find(q_train), db.goa_uniprot.count(q_train), ASPECT).load()
    query = {"_id": {"$in": unique(list(seq2go_trn.keys())).tolist()}}
    count = limit if limit else db.uniprot.count(query)
    source = db.uniprot.find(query).batch_size(10)
    if limit: source = source.limit(limit)
    stream_trn = DataStream(source, count, seq2go_trn, onto, classes)

    q_valid = {'DB': 'UniProtKB',
               'Evidence': {'$in': exp_codes},
               'Date': {"$gt": t0, "$lte": t1},
               'Aspect': ASPECT}

    seq2go_tst, _ = GoAnnotationCollectionLoader(db.goa_uniprot.find(q_valid), db.goa_uniprot.count(q_valid), ASPECT).load()
    query = {"_id": {"$in": unique(list(seq2go_tst.keys())).tolist()}}
    count = limit if limit else db.uniprot.count(query)
    source = db.uniprot.find(query).batch_size(10)
    if limit: source = source.limit(limit)
    stream_tst = DataStream(source, count, seq2go_tst, onto, classes)

    return stream_trn, stream_tst

def pad_seq(seq, max_length=MAX_LENGTH):
    seq += [PAD for _ in range(max_length - len(seq))]
    return np.asarray(seq)

class DataStream(object):
    def __init__(self, source, count, seq2go, onto, classes):

        self._classes = classes
        self._count = count
        self._source = source
        self._seq2go = seq2go
        self._onto = onto

    def __iter__(self):

        classes = self._classes
        count = self._count
        source = self._source
        seq2go = self._seq2go
        onto = self._onto

        s_cls = set(classes)

        for k, seq in UniprotCollectionLoader(source, count):
            if not MIN_LENGTH // 4 <= len(seq) <= MAX_LENGTH:
                continue
            y = np.zeros(len(classes))
            for go in onto.propagate(seq2go[k], include_root=False):
                if go not in s_cls:
                    continue
                y[classes.index(go)] = 1

                x = [AA.aa2index[aa] for aa in seq]

            yield k, x, y

    def __len__(self):
        return self._count

def step_decay(epoch):
    initial_lrate = LR
    drop = 0.5
    epochs_drop = 1.0
    lrate = max(0.0001, initial_lrate * math.pow(drop, math.floor((1 + epoch) / epochs_drop)))
    return lrate

def Features(inpt):

    feats = Embedding(input_dim=26, output_dim=20, embeddings_initializer='uniform')(inpt)

    feats = Conv1D(250, 30, activation='relu', padding='valid')(feats)
    feats = Dropout(0.3)(feats)
    feats = Conv1D(250, 10, activation='relu', padding='valid')(feats)
    feats = Dropout(0.3)(feats)
    feats = Conv1D(250, 10, activation='relu', padding='valid')(feats)
    feats = Dropout(0.3)(feats)
    feats = Conv1D(250, 10, activation='relu', padding='valid')(feats)
    feats = Dropout(0.3)(feats)
    feats = Conv1D(250, 10, activation='relu', padding='valid')(feats)
    feats = Dropout(0.3)(feats)

    return feats

def Classifier(inpt, classes):
    out = GlobalMaxPooling1D()(inpt)
    out = Dense(len(classes), activation='linear')(out)
    out = BatchNormalization()(out)
    out = Activation('sigmoid')(out)
    return out

def ModelCNN(classes):
    inp = Input(shape=(None,))
    out = Classifier(Features(inp), classes)
    model = Model(inputs=[inp], outputs=[out])
    # sgd = optimizers.SGD(lr=LR, decay=1e-6, momentum=0.9, nesterov=True)
    adam = optimizers.Adam(lr=LR, beta_1=0.9, beta_2=0.999, epsilon=1e-8)
    model.compile(loss='binary_crossentropy', optimizer=adam)

    return model

def batch_generator(stream):

    def prepare(batch):
        X, Y = zip(*batch)
        b = max(MIN_LENGTH, max(map(len, X)))
        X = [pad_seq(seq, b) for seq in X]
        return np.asarray(X), np.asarray(Y)

    batch = []
    for _, x, y in stream:
        if len(batch) == BATCH_SIZE:
            yield prepare(batch)
            batch = []
        batch.append([x, y])

    yield prepare(batch)

def add_arguments(parser):
    parser.add_argument("--mongo_url", type=str, default='mongodb://localhost:27017/',
                        help="Supply the URL of MongoDB"),
    parser.add_argument("--aspect", type=str, choices=['F', 'P', 'C'],
                        default="F", help="Specify the ontology aspect.")
    parser.add_argument("--init_epoch", type=int, default=0,
                        help="Which epoch to start training the model?")
    parser.add_argument("--num_epochs", type=int, default=20,
                        help="How many epochs to train the model?")
    parser.add_argument('-r', '--resume', default='', type=str, metavar='PATH',
                        help='path to latest checkpoint (default: none)')

class LossHistory(Callback):
    def __init__(self):
        self.losses = []

    def on_batch_end(self, batch, logs={}):
        self.losses.append(logs.get('loss'))

def train(model, gen_xy, length_xy, epoch, num_epochs,
          history=LossHistory(), lrate=LearningRateScheduler(step_decay)):

    pbar = tqdm(total=length_xy)

    for X, Y in gen_xy:
        assert len(X) == len(Y)

        model.fit(x=X, y=Y,
                  batch_size=BATCH_SIZE,
                  epochs=num_epochs,
                  verbose=0,
                  validation_data=None,
                  initial_epoch=epoch,
                  callbacks=[history])
        pbar.set_description("Training Loss:%.5f" % np.mean(history.losses))
        pbar.update(len(Y))

    pbar.close()

def zeroone2oneminusone(vec):
    return np.add(np.multiply(np.array(vec), 2), -1)

def oneminusone2zeroone(vec):
    return np.divide(np.add(np.array(vec), 1), 2)

def calc_loss(y_true, y_pred, batch_size=BATCH_SIZE):
    return batch_size * np.mean([log_loss(y, y_hat) for y, y_hat in zip(y_true, y_pred) if np.any(y)])

def predict(model, gen_xy, length_xy, classes):
    pbar = tqdm(total=length_xy, desc="Predicting...")
    i, m, n = 0, length_xy, len(classes)
    y_pred, y_true = np.zeros((m, n)), np.zeros((m, n))
    for i, (X, Y) in enumerate(gen_xy):
        assert len(X) == len(Y)
        k = len(Y)
        y_hat, y = model.predict(X), Y
        y_pred[i:i + k, ], y_true[i:i + k, ] = y_hat, y
        pbar.update(k)
    pbar.close()
    return y_true, y_pred

def evaluate(y_true, y_pred, classes):

    y_pred = y_pred[~np.all(y_pred == 0, axis=1)]
    y_true = y_true[~np.all(y_true == 0, axis=1)]

    f_max = F_max(y_pred, y_true, classes, np.arange(0.1, 1, 0.1))

    return calc_loss(y_true, y_pred), f_max

if __name__ == "__main__":

    parser = argparse.ArgumentParser()
    add_arguments(parser)
    args = parser.parse_args()

    ASPECT = args.aspect  # default: Molecular Function

    client = MongoClient(args.mongo_url)

    db = client['prot2vec']

    print("Loading Ontology...")
    onto = get_ontology(ASPECT)

    classes = onto.classes
    classes.remove(onto.root)
    assert onto.root not in classes

    model = ModelCNN(classes)
    if args.resume:
        model.load_weights(args.resume)
        print("Loaded model from disk")
    model.summary()

    for epoch in range(args.init_epoch, args.num_epochs):

        trn_stream, tst_stream = get_training_and_validation_streams(db, onto, classes)

        train(model, batch_generator(trn_stream), len(trn_stream), epoch, args.num_epochs)
        y_true, y_pred = predict(model, batch_generator(tst_stream), len(tst_stream), classes)
        loss, f_max = evaluate(y_true, y_pred, classes)

        print("[Epoch %d] (Validation Loss: %.5f, F_max: %.3f)" % (epoch + 1, loss, f_max))

        model_path = 'checkpoints/deeperseq-%d-%.5f-%.2f.hdf5' % (epoch + 1, loss, f_max)
        model.save_weights(model_path)
<|endoftext|>


<commit_before># Copyright (c) 2005 The Regents of The University of Michigan
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Nathan Binkert

import code
import datetime
import os
import socket
import sys

__all__ = [ 'options', 'arguments', 'main' ]

usage="%prog [gem5 options] script.py [script options]"
version="%prog 2.0"
brief_copyright=\
    "gem5 is copyrighted software; use the --copyright option for details."

def parse_options():
    import config
    from options import OptionParser

    options = OptionParser(usage=usage, version=version,
                           description=brief_copyright)
    option = options.add_option
    group = options.set_group

    # Help options
    option('-B', "--build-info", action="store_true", default=False,
        help="Show build information")
    option('-C', "--copyright", action="store_true", default=False,
        help="Show full copyright information")
    option('-R', "--readme", action="store_true", default=False,
        help="Show the readme")

    # Options for configuring the base simulator
    option('-d', "--outdir", metavar="DIR", default="m5out",
        help="Set the output directory to DIR [Default: %default]")
    option('-r', "--redirect-stdout", action="store_true", default=False,
        help="Redirect stdout (& stderr, without -e) to file")
    option('-e', "--redirect-stderr", action="store_true", default=False,
        help="Redirect stderr to file")
    option("--stdout-file", metavar="FILE", default="simout",
        help="Filename for -r redirection [Default: %default]")
    option("--stderr-file", metavar="FILE", default="simerr",
        help="Filename for -e redirection [Default: %default]")
    option('-i', "--interactive", action="store_true", default=False,
        help="Invoke the interactive interpreter after running the script")
    option("--pdb", action="store_true", default=False,
        help="Invoke the python debugger before running the script")
    option('-p', "--path", metavar="PATH[:PATH]", action='append', split=':',
        help="Prepend PATH to the system path when invoking the script")
    option('-q', "--quiet", action="count", default=0,
        help="Reduce verbosity")
    option('-v', "--verbose", action="count", default=0,
        help="Increase verbosity")

    # Statistics options
    group("Statistics Options")
    option("--stats-file", metavar="FILE", default="stats.txt",
        help="Sets the output file for statistics [Default: %default]")

    # Configuration Options
    group("Configuration Options")
    option("--dump-config", metavar="FILE", default="config.ini",
        help="Dump configuration output file [Default: %default]")
    option("--json-config", metavar="FILE", default="config.json",
        help="Create JSON output of the configuration [Default: %default]")
    option("--dot-config", metavar="FILE", default="config.dot",
        help="Create DOT & pdf outputs of the configuration [Default: %default]")

    # Debugging options
    group("Debugging Options")
    option("--debug-break", metavar="TIME[,TIME]", action='append', split=',',
        help="Cycle to create a breakpoint")
    option("--debug-help", action='store_true',
        help="Print help on trace flags")
    option("--debug-flags", metavar="FLAG[,FLAG]", action='append', split=',',
        help="Sets the flags for tracing (-FLAG disables a flag)")
    option("--remote-gdb-port", type='int', default=7000,
        help="Remote gdb base port (set to 0 to disable listening)")

    # Tracing options
    group("Trace Options")
    option("--trace-start", metavar="TIME", type='int',
        help="Start tracing at TIME (must be in ticks)")
    option("--trace-file", metavar="FILE", default="cout",
        help="Sets the output file for tracing [Default: %default]")
    option("--trace-ignore", metavar="EXPR", action='append', split=':',
        help="Ignore EXPR sim objects")

    # Help options
    group("Help Options")
    option("--list-sim-objects", action='store_true', default=False,
        help="List all built-in SimObjects, their params and default values")

    # load the options.py config file to allow people to set their own
    # default options
    options_file = config.get('options.py')
    if options_file:
        scope = { 'options' : options }
        execfile(options_file, scope)

    arguments = options.parse_args()
    return options,arguments

def interact(scope):
    banner = "gem5 Interactive Console"
    try:
        from IPython.Shell import IPShellEmbed
        ipshell = IPShellEmbed(argv=[], banner=banner, user_ns=scope)
        ipshell()
    except ImportError:
        code.InteractiveConsole(scope).interact(banner)

def main(*args):
    import m5

    import core
    import debug
    import defines
    import event
    import info
    import stats
    import trace

    from util import fatal

    if len(args) == 0:
        options, arguments = parse_options()
    elif len(args) == 2:
        options, arguments = args
    else:
        raise TypeError, "main() takes 0 or 2 arguments (%d given)" % len(args)

    m5.options = options

    def check_tracing():
        if defines.TRACING_ON:
            return

        fatal("Tracing is not enabled.  Compile with TRACING_ON")

    if not os.path.isdir(options.outdir):
        os.makedirs(options.outdir)

    # These filenames are used only if the redirect_std* options are set
    stdout_file = os.path.join(options.outdir, options.stdout_file)
    stderr_file = os.path.join(options.outdir, options.stderr_file)

    # Print redirection notices here before doing any redirection
    if options.redirect_stdout and not options.redirect_stderr:
        print "Redirecting stdout and stderr to", stdout_file
    else:
        if options.redirect_stdout:
            print "Redirecting stdout to", stdout_file
        if options.redirect_stderr:
            print "Redirecting stderr to", stderr_file

    # Now redirect stdout/stderr as desired
    if options.redirect_stdout:
        redir_fd = os.open(stdout_file, os. O_WRONLY | os.O_CREAT | os.O_TRUNC)
        os.dup2(redir_fd, sys.stdout.fileno())
        if not options.redirect_stderr:
            os.dup2(redir_fd, sys.stderr.fileno())

    if options.redirect_stderr:
        redir_fd = os.open(stderr_file, os. O_WRONLY | os.O_CREAT | os.O_TRUNC)
        os.dup2(redir_fd, sys.stderr.fileno())

    done = False

    if options.build_info:
        done = True
        print 'Build information:'
        print
        print 'compiled %s' % defines.compileDate;
        print 'build options:'
        keys = defines.buildEnv.keys()
        keys.sort()
        for key in keys:
            val = defines.buildEnv[key]
            print '    %s = %s' % (key, val)
        print

    if options.copyright:
        done = True
        print info.COPYING
        print

    if options.readme:
        done = True
        print 'Readme:'
        print
        print info.README
        print

    if options.debug_help:
        done = True
        check_tracing()
        debug.help()

    if options.list_sim_objects:
        import SimObject
        done = True
        print "SimObjects:"
        objects = SimObject.allClasses.keys()
        objects.sort()
        for name in objects:
            obj = SimObject.allClasses[name]
            print "    %s" % obj
            params = obj._params.keys()
            params.sort()
            for pname in params:
                param = obj._params[pname]
                default = getattr(param, 'default', '')
                print "        %s" % pname
                if default:
                    print "            default: %s" % default
                print "            desc: %s" % param.desc
                print
            print

    if done:
        sys.exit(0)

    # setting verbose and quiet at the same time doesn't make sense
    if options.verbose > 0 and options.quiet > 0:
        options.usage(2)

    verbose = options.verbose - options.quiet
    if options.verbose >= 0:
        print "gem5 Simulator System.  http://gem5.org"
        print brief_copyright
        print

        print "gem5 compiled %s" % defines.compileDate;

        print "gem5 started %s" % \
            datetime.datetime.now().strftime("%b %e %Y %X")
        print "gem5 executing on %s" % socket.gethostname()

        print "command line:",
        for argv in sys.argv:
            print argv,
        print

    # check to make sure we can find the listed script
    if not arguments or not os.path.isfile(arguments[0]):
        if arguments and not os.path.isfile(arguments[0]):
            print "Script %s not found" % arguments[0]

        options.usage(2)

    # tell C++ about output directory
    core.setOutputDir(options.outdir)

    # update the system path with elements from the -p option
    sys.path[0:0] = options.path

    # set stats options
    stats.initText(options.stats_file)

    # set debugging options
    debug.setRemoteGDBPort(options.remote_gdb_port)
    for when in options.debug_break:
        debug.schedBreakCycle(int(when))

    if options.debug_flags:
        check_tracing()

        on_flags = []
        off_flags = []
        for flag in options.debug_flags:
            off = False
            if flag.startswith('-'):
                flag = flag[1:]
                off = True

            if flag not in debug.flags:
                print >>sys.stderr, "invalid debug flag '%s'" % flag
                sys.exit(1)

            if off:
                debug.flags[flag].disable()
            else:
                debug.flags[flag].enable()

    if options.trace_start:
        check_tracing()
        e = event.create(trace.enable, event.Event.Trace_Enable_Pri)
        event.mainq.schedule(e, options.trace_start)
    else:
        trace.enable()

    trace.output(options.trace_file)

    for ignore in options.trace_ignore:
        check_tracing()
        trace.ignore(ignore)

    sys.argv = arguments
    sys.path = [ os.path.dirname(sys.argv[0]) ] + sys.path

    filename = sys.argv[0]
    filedata = file(filename, 'r').read()
    filecode = compile(filedata, filename, 'exec')
    scope = { '__file__' : filename,
              '__name__' : '__m5_main__' }

    # we want readline if we're doing anything interactive
    if options.interactive or options.pdb:
        exec "import readline" in scope

    # if pdb was requested, execfile the thing under pdb, otherwise,
    # just do the execfile normally
    if options.pdb:
        import pdb
        import traceback

        pdb = pdb.Pdb()
        try:
            pdb.run(filecode, scope)
        except SystemExit:
            print "The program exited via sys.exit(). Exit status: ",
            print sys.exc_info()[1]
        except:
            traceback.print_exc()
            print "Uncaught exception. Entering post mortem debugging"
            t = sys.exc_info()[2]
            while t.tb_next is not None:
                t = t.tb_next
                pdb.interaction(t.tb_frame,t)
    else:
        exec filecode in scope

    # once the script is done
    if options.interactive:
        interact(scope)

if __name__ == '__main__':
    from pprint import pprint

    options, arguments = parse_options()

    print 'opts:'
    pprint(options, indent=4)
    print

    print 'args:'
    pprint(arguments, indent=4)
<commit_msg>base: Add support for newer versions of IPython<commit_after># Copyright (c) 2005 The Regents of The University of Michigan
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Nathan Binkert

import code
import datetime
import os
import socket
import sys

__all__ = [ 'options', 'arguments', 'main' ]

usage="%prog [gem5 options] script.py [script options]"
version="%prog 2.0"
brief_copyright=\
    "gem5 is copyrighted software; use the --copyright option for details."

def parse_options():
    import config
    from options import OptionParser

    options = OptionParser(usage=usage, version=version,
                           description=brief_copyright)
    option = options.add_option
    group = options.set_group

    # Help options
    option('-B', "--build-info", action="store_true", default=False,
        help="Show build information")
    option('-C', "--copyright", action="store_true", default=False,
        help="Show full copyright information")
    option('-R', "--readme", action="store_true", default=False,
        help="Show the readme")

    # Options for configuring the base simulator
    option('-d', "--outdir", metavar="DIR", default="m5out",
        help="Set the output directory to DIR [Default: %default]")
    option('-r', "--redirect-stdout", action="store_true", default=False,
        help="Redirect stdout (& stderr, without -e) to file")
    option('-e', "--redirect-stderr", action="store_true", default=False,
        help="Redirect stderr to file")
    option("--stdout-file", metavar="FILE", default="simout",
        help="Filename for -r redirection [Default: %default]")
    option("--stderr-file", metavar="FILE", default="simerr",
        help="Filename for -e redirection [Default: %default]")
    option('-i', "--interactive", action="store_true", default=False,
        help="Invoke the interactive interpreter after running the script")
    option("--pdb", action="store_true", default=False,
        help="Invoke the python debugger before running the script")
    option('-p', "--path", metavar="PATH[:PATH]", action='append', split=':',
        help="Prepend PATH to the system path when invoking the script")
    option('-q', "--quiet", action="count", default=0,
        help="Reduce verbosity")
    option('-v', "--verbose", action="count", default=0,
        help="Increase verbosity")

    # Statistics options
    group("Statistics Options")
    option("--stats-file", metavar="FILE", default="stats.txt",
        help="Sets the output file for statistics [Default: %default]")

    # Configuration Options
    group("Configuration Options")
    option("--dump-config", metavar="FILE", default="config.ini",
        help="Dump configuration output file [Default: %default]")
    option("--json-config", metavar="FILE", default="config.json",
        help="Create JSON output of the configuration [Default: %default]")
    option("--dot-config", metavar="FILE", default="config.dot",
        help="Create DOT & pdf outputs of the configuration [Default: %default]")

    # Debugging options
    group("Debugging Options")
    option("--debug-break", metavar="TIME[,TIME]", action='append', split=',',
        help="Cycle to create a breakpoint")
    option("--debug-help", action='store_true',
        help="Print help on trace flags")
    option("--debug-flags", metavar="FLAG[,FLAG]", action='append', split=',',
        help="Sets the flags for tracing (-FLAG disables a flag)")
    option("--remote-gdb-port", type='int', default=7000,
        help="Remote gdb base port (set to 0 to disable listening)")

    # Tracing options
    group("Trace Options")
    option("--trace-start", metavar="TIME", type='int',
        help="Start tracing at TIME (must be in ticks)")
    option("--trace-file", metavar="FILE", default="cout",
        help="Sets the output file for tracing [Default: %default]")
    option("--trace-ignore", metavar="EXPR", action='append', split=':',
        help="Ignore EXPR sim objects")

    # Help options
    group("Help Options")
    option("--list-sim-objects", action='store_true', default=False,
        help="List all built-in SimObjects, their params and default values")

    # load the options.py config file to allow people to set their own
    # default options
    options_file = config.get('options.py')
    if options_file:
        scope = { 'options' : options }
        execfile(options_file, scope)

    arguments = options.parse_args()
    return options,arguments

def interact(scope):
    banner = "gem5 Interactive Console"

    ipshell = None
    prompt_in1 = "gem5 \\#> "
    prompt_out = "gem5 \\#: "

    # Is IPython version 0.10 or earlier available?
    try:
        from IPython.Shell import IPShellEmbed
        ipshell = IPShellEmbed(argv=["-prompt_in1", prompt_in1,
                                     "-prompt_out", prompt_out],
                               banner=banner, user_ns=scope)
    except ImportError:
        pass

    # Is IPython version 0.11 or later available?
    if not ipshell:
        try:
            import IPython
            from IPython.config.loader import Config
            from IPython.frontend.terminal.embed import InteractiveShellEmbed

            cfg = Config()
            cfg.PromptManager.in_template = prompt_in1
            cfg.PromptManager.out_template = prompt_out
            ipshell = InteractiveShellEmbed(config=cfg, user_ns=scope,
                                            banner1=banner)
        except ImportError:
            pass

    if ipshell:
        ipshell()
    else:
        # Use the Python shell in the standard library if IPython
        # isn't available.
        code.InteractiveConsole(scope).interact(banner)

def main(*args):
    import m5

    import core
    import debug
    import defines
    import event
    import info
    import stats
    import trace

    from util import fatal

    if len(args) == 0:
        options, arguments = parse_options()
    elif len(args) == 2:
        options, arguments = args
    else:
        raise TypeError, "main() takes 0 or 2 arguments (%d given)" % len(args)

    m5.options = options

    def check_tracing():
        if defines.TRACING_ON:
            return

        fatal("Tracing is not enabled.  Compile with TRACING_ON")

    if not os.path.isdir(options.outdir):
        os.makedirs(options.outdir)

    # These filenames are used only if the redirect_std* options are set
    stdout_file = os.path.join(options.outdir, options.stdout_file)
    stderr_file = os.path.join(options.outdir, options.stderr_file)

    # Print redirection notices here before doing any redirection
    if options.redirect_stdout and not options.redirect_stderr:
        print "Redirecting stdout and stderr to", stdout_file
    else:
        if options.redirect_stdout:
            print "Redirecting stdout to", stdout_file
        if options.redirect_stderr:
            print "Redirecting stderr to", stderr_file

    # Now redirect stdout/stderr as desired
    if options.redirect_stdout:
        redir_fd = os.open(stdout_file, os. O_WRONLY | os.O_CREAT | os.O_TRUNC)
        os.dup2(redir_fd, sys.stdout.fileno())
        if not options.redirect_stderr:
            os.dup2(redir_fd, sys.stderr.fileno())

    if options.redirect_stderr:
        redir_fd = os.open(stderr_file, os. O_WRONLY | os.O_CREAT | os.O_TRUNC)
        os.dup2(redir_fd, sys.stderr.fileno())

    done = False

    if options.build_info:
        done = True
        print 'Build information:'
        print
        print 'compiled %s' % defines.compileDate;
        print 'build options:'
        keys = defines.buildEnv.keys()
        keys.sort()
        for key in keys:
            val = defines.buildEnv[key]
            print '    %s = %s' % (key, val)
        print

    if options.copyright:
        done = True
        print info.COPYING
        print

    if options.readme:
        done = True
        print 'Readme:'
        print
        print info.README
        print

    if options.debug_help:
        done = True
        check_tracing()
        debug.help()

    if options.list_sim_objects:
        import SimObject
        done = True
        print "SimObjects:"
        objects = SimObject.allClasses.keys()
        objects.sort()
        for name in objects:
            obj = SimObject.allClasses[name]
            print "    %s" % obj
            params = obj._params.keys()
            params.sort()
            for pname in params:
                param = obj._params[pname]
                default = getattr(param, 'default', '')
                print "        %s" % pname
                if default:
                    print "            default: %s" % default
                print "            desc: %s" % param.desc
                print
            print

    if done:
        sys.exit(0)

    # setting verbose and quiet at the same time doesn't make sense
    if options.verbose > 0 and options.quiet > 0:
        options.usage(2)

    verbose = options.verbose - options.quiet
    if options.verbose >= 0:
        print "gem5 Simulator System.  http://gem5.org"
        print brief_copyright
        print

        print "gem5 compiled %s" % defines.compileDate;

        print "gem5 started %s" % \
            datetime.datetime.now().strftime("%b %e %Y %X")
        print "gem5 executing on %s" % socket.gethostname()

        print "command line:",
        for argv in sys.argv:
            print argv,
        print

    # check to make sure we can find the listed script
    if not arguments or not os.path.isfile(arguments[0]):
        if arguments and not os.path.isfile(arguments[0]):
            print "Script %s not found" % arguments[0]

        options.usage(2)

    # tell C++ about output directory
    core.setOutputDir(options.outdir)

    # update the system path with elements from the -p option
    sys.path[0:0] = options.path

    # set stats options
    stats.initText(options.stats_file)

    # set debugging options
    debug.setRemoteGDBPort(options.remote_gdb_port)
    for when in options.debug_break:
        debug.schedBreakCycle(int(when))

    if options.debug_flags:
        check_tracing()

        on_flags = []
        off_flags = []
        for flag in options.debug_flags:
            off = False
            if flag.startswith('-'):
                flag = flag[1:]
                off = True

            if flag not in debug.flags:
                print >>sys.stderr, "invalid debug flag '%s'" % flag
                sys.exit(1)

            if off:
                debug.flags[flag].disable()
            else:
                debug.flags[flag].enable()

    if options.trace_start:
        check_tracing()
        e = event.create(trace.enable, event.Event.Trace_Enable_Pri)
        event.mainq.schedule(e, options.trace_start)
    else:
        trace.enable()

    trace.output(options.trace_file)

    for ignore in options.trace_ignore:
        check_tracing()
        trace.ignore(ignore)

    sys.argv = arguments
    sys.path = [ os.path.dirname(sys.argv[0]) ] + sys.path

    filename = sys.argv[0]
    filedata = file(filename, 'r').read()
    filecode = compile(filedata, filename, 'exec')
    scope = { '__file__' : filename,
              '__name__' : '__m5_main__' }

    # we want readline if we're doing anything interactive
    if options.interactive or options.pdb:
        exec "import readline" in scope

    # if pdb was requested, execfile the thing under pdb, otherwise,
    # just do the execfile normally
    if options.pdb:
        import pdb
        import traceback

        pdb = pdb.Pdb()
        try:
            pdb.run(filecode, scope)
        except SystemExit:
            print "The program exited via sys.exit(). Exit status: ",
            print sys.exc_info()[1]
        except:
            traceback.print_exc()
            print "Uncaught exception. Entering post mortem debugging"
            t = sys.exc_info()[2]
            while t.tb_next is not None:
                t = t.tb_next
                pdb.interaction(t.tb_frame,t)
    else:
        exec filecode in scope

    # once the script is done
    if options.interactive:
        interact(scope)

if __name__ == '__main__':
    from pprint import pprint

    options, arguments = parse_options()

    print 'opts:'
    pprint(options, indent=4)
    print

    print 'args:'
    pprint(arguments, indent=4)
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
##
# domains.py: module for domains of model outcomes
##
# © 2012 Chris Ferrie (csferrie@gmail.com) and
#        Christopher E. Granade (cgranade@gmail.com)
#
# This file is a part of the Qinfer project.
# Licensed under the AGPL version 3.
##
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
##

## IMPORTS ###################################################################

from __future__ import division
from __future__ import absolute_import

from builtins import range
from future.utils import with_metaclass

from scipy.special import binom
from math import factorial
from itertools import combinations_with_replacement
import numpy as np

import abc

import warnings

## EXPORTS ###################################################################

__all__ = [
    'Domain',
    'RealDomain',
    'IntegerDomain',
    'MultinomialDomain'
]

## FUNCTIONS #################################################################

## ABSTRACT CLASSES AND MIXINS ###############################################

class Domain(with_metaclass(abc.ABCMeta, object)):
    """
    Abstract base class for domains of outcomes of models.
    """

    ## ABSTRACT PROPERTIES ##

    @abc.abstractproperty
    def is_continuous(self):
        """
        Whether or not the domain has an uncountable number of values.

        :type: `bool`
        """
        pass

    @abc.abstractproperty
    def is_finite(self):
        """
        Whether or not the domain contains a finite number of points.

        :type: `bool`
        """
        pass

    @abc.abstractproperty
    def dtype(self):
        """
        The numpy dtype of a single element of the domain.

        :type: `np.dtype`
        """
        pass

    @abc.abstractproperty
    def n_members(self):
        """
        Returns the number of members in the domain if it
        `is_finite`, otherwise, returns `None`.

        :type: ``int``
        """
        pass

    @abc.abstractproperty
    def example_point(self):
        """
        Returns any single point guaranteed to be in the domain, but
        no other guarantees; useful for testing purposes.
        This is given as a size 1 ``np.array`` of type ``dtype``.

        :type: ``np.ndarray``
        """
        pass

    @abc.abstractproperty
    def values(self):
        """
        Returns an `np.array` of type `self.dtype` containing
        some values from the domain.
        For domains where ``is_finite`` is ``True``, all elements
        of the domain will be yielded exactly once.

        :rtype: `np.ndarray`
        """
        pass

    ## CONCRETE PROPERTIES ##

    @property
    def is_discrete(self):
        """
        Whether or not the domain has a countable number of values.

        :type: `bool`
        """
        return not self.is_continuous

    ## ABSTRACT METHODS ##

    @abc.abstractmethod
    def in_domain(self, points):
        """
        Returns ``True`` if all of the given points are in the domain,
        ``False`` otherwise.

        :param np.ndarray points: An `np.ndarray` of type `self.dtype`.

        :rtype: `bool`
        """
        pass

## CLASSES ###################################################################

class RealDomain(Domain):
    """
    A domain specifying a contiguous (and possibly open ended) subset
    of the real numbers.

    :param float min: A number specifying the lowest possible value of the
        domain. If left as `None`, negative infinity is assumed.
    :param float max: A number specifying the largest possible value of the
        domain. If left as `None`, positive infinity is assumed.
    """

    def __init__(self, min=None, max=None):
        self._min = min
        self._max = max

    ## PROPERTIES ##

    @property
    def min(self):
        """
        Returns the minimum value of the domain. The outcome
        None is interpreted as negative infinity.

        :rtype: `float`
        """
        return self._min
    @property
    def max(self):
        """
        Returns the maximum value of the domain. The outcome
        None is interpreted as positive infinity.

        :rtype: `float`
        """
        return self._max

    @property
    def is_continuous(self):
        """
        Whether or not the domain has an uncountable number of values.

        :type: `bool`
        """
        return True

    @property
    def is_finite(self):
        """
        Whether or not the domain contains a finite number of points.

        :type: `bool`
        """
        return False

    @property
    def dtype(self):
        """
        The numpy dtype of a single element of the domain.

        :type: `np.dtype`
        """
        return np.float

    @property
    def n_members(self):
        """
        Returns the number of members in the domain if it
        `is_finite`, otherwise, returns `None`.

        :type: ``int``
        """
        return None

    @property
    def example_point(self):
        """
        Returns any single point guaranteed to be in the domain, but
        no other guarantees; useful for testing purposes.
        This is given as a size 1 ``np.array`` of type ``dtype``.

        :type: ``np.ndarray``
        """
        if self.min is not None:
            return np.array([self.min], dtype=self.dtype)
        if self.max is not None:
            return np.array([self.max], dtype=self.dtype)
        else:
            return np.array([0], dtype=self.dtype)

    @property
    def values(self):
        """
        Returns an `np.array` of type `self.dtype` containing
        some values from the domain.
        For domains where ``is_finite`` is ``True``, all elements
        of the domain will be yielded exactly once.

        :rtype: `np.ndarray`
        """
        return self.example_point

    ## METHODS ##

    def in_domain(self, points):
        """
        Returns ``True`` if all of the given points are in the domain,
        ``False`` otherwise.

        :param np.ndarray points: An `np.ndarray` of type `self.dtype`.

        :rtype: `bool`
        """
        return np.all(points >= self._min) and np.all(points <= self._max)

class IntegerDomain(Domain):
    """
    A domain specifying a contiguous (and possibly open ended) subset
    of the integers.

    :param int min: A number specifying the lowest possible value of the
        domain. If `None`, negative infinity is assumed.
    :param int max: A number specifying the largest possible value of the
        domain. If left as `None`, positive infinity is assumed.

    Note: Yes, it is slightly unpythonic to specify `max` instead of `max`+1.
    """

    def __init__(self, min=0, max=None):
        self._min = min
        self._max = max

    ## PROPERTIES ##

    @property
    def min(self):
        """
        Returns the minimum value of the domain. The outcome
        None is interpreted as negative infinity.

        :rtype: `float`
        """
        return self._min
    @property
    def max(self):
        """
        Returns the maximum value of the domain. The outcome
        None is interpreted as positive infinity.

        :rtype: `float`
        """
        return self._max

    @property
    def is_continuous(self):
        """
        Whether or not the domain has an uncountable number of values.

        :type: `bool`
        """
        return False

    @property
    def is_finite(self):
        """
        Whether or not the domain contains a finite number of points.

        :type: `bool`
        """
        return self.min is not None and self.max is not None

    @property
    def dtype(self):
        """
        The numpy dtype of a single element of the domain.

        :type: `np.dtype`
        """
        return np.int

    @property
    def n_members(self):
        """
        Returns the number of members in the domain if it
        `is_finite`, otherwise, returns `None`.

        :type: ``int``
        """
        if self.is_finite:
            return self.max - self.min + 1
        else:
            return None

    @property
    def example_point(self):
        """
        Returns any single point guaranteed to be in the domain, but
        no other guarantees; useful for testing purposes.
        This is given as a size 1 ``np.array`` of type ``dtype``.

        :type: ``np.ndarray``
        """
        if self.min is not None:
            return np.array([self._min], dtype=self.dtype)
        if self.max is not None:
            return np.array([self._max], dtype=self.dtype)
        else:
            return np.array([0], dtype=self.dtype)

    @property
    def values(self):
        """
        Returns an `np.array` of type `self.dtype` containing
        some values from the domain.
        For domains where ``is_finite`` is ``True``, all elements
        of the domain will be yielded exactly once.

        :rtype: `np.ndarray`
        """
        if self.max is None or self.min is None:
            return self.example_point
        else:
            return np.arange(self.min, self.max + 1, dtype = self.dtype)

    ## METHODS ##

    def in_domain(self, points):
        """
        Returns ``True`` if all of the given points are in the domain,
        ``False`` otherwise.

        :param np.ndarray points: An `np.ndarray` of type `self.dtype`.

        :rtype: `bool`
        """
        return np.all(np.mod(points,1) == 0) and np.all(points >= self._min) and np.all(points <= self._max)

class MultinomialDomain(Domain):
    """
    A domain specifying k-tuples of non-negative integers which
    sum to a specific value.

    :param int n_meas: The sum of any tuple in the domain.
    :param int n_elements: The number of elements in a tuple.
    """

    def __init__(self, n_meas, n_elements=2):
        self._n_elements = n_elements
        self._n_meas = n_meas

    ## PROPERTIES ##

    @property
    def n_meas(self):
        """
        Returns the sum of any tuple in the domain.

        :rtype: `int`
        """
        return self._n_meas
    @property
    def n_elements(self):
        """
        Returns the number of elements of a tuple in the domain.

        :rtype: `int`
        """
        return self._n_elements

    @property
    def is_continuous(self):
        """
        Whether or not the domain has an uncountable number of values.

        :type: `bool`
        """
        return False

    @property
    def is_finite(self):
        """
        Whether or not the domain contains a finite number of points.

        :type: `bool`
        """
        return True

    @property
    def dtype(self):
        """
        The numpy dtype of a single element of the domain.

        :type: `np.dtype`
        """
        return np.dtype([('k', np.int, self.n_elements)])

    @property
    def n_members(self):
        """
        Returns the number of members in the domain if it
        `is_finite`, otherwise, returns `None`.

        :type: ``int``
        """
        return int(binom(self.n_meas + self.n_elements -1, self.n_elements - 1))

    @property
    def example_point(self):
        """
        Returns any single point guaranteed to be in the domain, but
        no other guarantees; useful for testing purposes.
        This is given as a size 1 ``np.array`` of type ``dtype``.

        :type: ``np.ndarray``
        """
        return np.array([([self.n_meas] + [0] * (self.n_elements-1))], dtype=self.dtype)

    @property
    def values(self):
        """
        Returns an `np.array` of type `self.dtype` containing
        some values from the domain.
        For domains where ``is_finite`` is ``True``, all elements
        of the domain will be yielded exactly once.

        :rtype: `np.ndarray`
        """

        # This code comes from Jared Goguen at http://stackoverflow.com/a/37712597/1082565
        partition_array = np.empty((self.n_members, self.n_elements), dtype=int)
        masks = np.identity(self.n_elements, dtype=int)
        for i, c in enumerate(combinations_with_replacement(masks, self.n_meas)):
            partition_array[i,:] = sum(c)

        # Convert to dtype before returning
        return self.from_regular_array(partition_array)

    ## METHODS ##

    def to_regular_array(self, A):
        """
        Converts from an array of type `self.dtype` to an array
        of type `int` with an additional index labeling the
        tuple indeces.

        :param np.ndarray A: An `np.array` of type `self.dtype`.

        :rtype: `np.ndarray`
        """
        # this could be a static method, but we choose to be consistent with
        # from_regular_array
        return A.view((int, len(A.dtype.names))).reshape(A.shape + (-1,))

    def from_regular_array(self, A):
        """
        Converts from an array of type `int` where the last index
        is assumed to have length `self.n_elements` to an array
        of type `self.d_type` with one fewer index.

        :param np.ndarray A: An `np.array` of type `int`.

        :rtype: `np.ndarray`
        """
        dims = A.shape[:-1]
        return A.reshape((np.prod(dims),-1)).view(dtype=self.dtype).squeeze(-1).reshape(dims)

    def in_domain(self, points):
        """
        Returns ``True`` if all of the given points are in the domain,
        ``False`` otherwise.

        :param np.ndarray points: An `np.ndarray` of type `self.dtype`.

        :rtype: `bool`
        """
        array_view = self.to_regular_array(points)
        print(np.all(array_view>=0))
        return np.all(array_view >= 0) and np.all(np.sum(array_view, axis=-1) == self.n_meas)

    <commit_msg>Fixed IntegerDomain in_domain bug<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
##
# domains.py: module for domains of model outcomes
##
# © 2012 Chris Ferrie (csferrie@gmail.com) and
#        Christopher E. Granade (cgranade@gmail.com)
#
# This file is a part of the Qinfer project.
# Licensed under the AGPL version 3.
##
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
##

## IMPORTS ###################################################################

from __future__ import division
from __future__ import absolute_import

from builtins import range
from future.utils import with_metaclass

from scipy.special import binom
from math import factorial
from itertools import combinations_with_replacement
import numpy as np

import abc

import warnings

## EXPORTS ###################################################################

__all__ = [
    'Domain',
    'RealDomain',
    'IntegerDomain',
    'MultinomialDomain'
]

## FUNCTIONS #################################################################

## ABSTRACT CLASSES AND MIXINS ###############################################

class Domain(with_metaclass(abc.ABCMeta, object)):
    """
    Abstract base class for domains of outcomes of models.
    """

    ## ABSTRACT PROPERTIES ##

    @abc.abstractproperty
    def is_continuous(self):
        """
        Whether or not the domain has an uncountable number of values.

        :type: `bool`
        """
        pass

    @abc.abstractproperty
    def is_finite(self):
        """
        Whether or not the domain contains a finite number of points.

        :type: `bool`
        """
        pass

    @abc.abstractproperty
    def dtype(self):
        """
        The numpy dtype of a single element of the domain.

        :type: `np.dtype`
        """
        pass

    @abc.abstractproperty
    def n_members(self):
        """
        Returns the number of members in the domain if it
        `is_finite`, otherwise, returns `None`.

        :type: ``int``
        """
        pass

    @abc.abstractproperty
    def example_point(self):
        """
        Returns any single point guaranteed to be in the domain, but
        no other guarantees; useful for testing purposes.
        This is given as a size 1 ``np.array`` of type ``dtype``.

        :type: ``np.ndarray``
        """
        pass

    @abc.abstractproperty
    def values(self):
        """
        Returns an `np.array` of type `self.dtype` containing
        some values from the domain.
        For domains where ``is_finite`` is ``True``, all elements
        of the domain will be yielded exactly once.

        :rtype: `np.ndarray`
        """
        pass

    ## CONCRETE PROPERTIES ##

    @property
    def is_discrete(self):
        """
        Whether or not the domain has a countable number of values.

        :type: `bool`
        """
        return not self.is_continuous

    ## ABSTRACT METHODS ##

    @abc.abstractmethod
    def in_domain(self, points):
        """
        Returns ``True`` if all of the given points are in the domain,
        ``False`` otherwise.

        :param np.ndarray points: An `np.ndarray` of type `self.dtype`.

        :rtype: `bool`
        """
        pass

## CLASSES ###################################################################

class RealDomain(Domain):
    """
    A domain specifying a contiguous (and possibly open ended) subset
    of the real numbers.

    :param float min: A number specifying the lowest possible value of the
        domain. If left as `None`, negative infinity is assumed.
    :param float max: A number specifying the largest possible value of the
        domain. If left as `None`, positive infinity is assumed.
    """

    def __init__(self, min=None, max=None):
        self._min = min
        self._max = max

    ## PROPERTIES ##

    @property
    def min(self):
        """
        Returns the minimum value of the domain. The outcome
        None is interpreted as negative infinity.

        :rtype: `float`
        """
        return self._min
    @property
    def max(self):
        """
        Returns the maximum value of the domain. The outcome
        None is interpreted as positive infinity.

        :rtype: `float`
        """
        return self._max

    @property
    def is_continuous(self):
        """
        Whether or not the domain has an uncountable number of values.

        :type: `bool`
        """
        return True

    @property
    def is_finite(self):
        """
        Whether or not the domain contains a finite number of points.

        :type: `bool`
        """
        return False

    @property
    def dtype(self):
        """
        The numpy dtype of a single element of the domain.

        :type: `np.dtype`
        """
        return np.float

    @property
    def n_members(self):
        """
        Returns the number of members in the domain if it
        `is_finite`, otherwise, returns `None`.

        :type: ``int``
        """
        return None

    @property
    def example_point(self):
        """
        Returns any single point guaranteed to be in the domain, but
        no other guarantees; useful for testing purposes.
        This is given as a size 1 ``np.array`` of type ``dtype``.

        :type: ``np.ndarray``
        """
        if self.min is not None:
            return np.array([self.min], dtype=self.dtype)
        if self.max is not None:
            return np.array([self.max], dtype=self.dtype)
        else:
            return np.array([0], dtype=self.dtype)

    @property
    def values(self):
        """
        Returns an `np.array` of type `self.dtype` containing
        some values from the domain.
        For domains where ``is_finite`` is ``True``, all elements
        of the domain will be yielded exactly once.

        :rtype: `np.ndarray`
        """
        return self.example_point

    ## METHODS ##

    def in_domain(self, points):
        """
        Returns ``True`` if all of the given points are in the domain,
        ``False`` otherwise.

        :param np.ndarray points: An `np.ndarray` of type `self.dtype`.

        :rtype: `bool`
        """
        return np.all(points >= self._min) and np.all(points <= self._max)

class IntegerDomain(Domain):
    """
    A domain specifying a contiguous (and possibly open ended) subset
    of the integers.

    :param int min: A number specifying the lowest possible value of the
        domain. If `None`, negative infinity is assumed.
    :param int max: A number specifying the largest possible value of the
        domain. If left as `None`, positive infinity is assumed.

    Note: Yes, it is slightly unpythonic to specify `max` instead of `max`+1.
    """

    def __init__(self, min=0, max=None):
        self._min = min
        self._max = max

    ## PROPERTIES ##

    @property
    def min(self):
        """
        Returns the minimum value of the domain. The outcome
        None is interpreted as negative infinity.

        :rtype: `float`
        """
        return self._min
    @property
    def max(self):
        """
        Returns the maximum value of the domain. The outcome
        None is interpreted as positive infinity.

        :rtype: `float`
        """
        return self._max

    @property
    def is_continuous(self):
        """
        Whether or not the domain has an uncountable number of values.

        :type: `bool`
        """
        return False

    @property
    def is_finite(self):
        """
        Whether or not the domain contains a finite number of points.

        :type: `bool`
        """
        return self.min is not None and self.max is not None

    @property
    def dtype(self):
        """
        The numpy dtype of a single element of the domain.

        :type: `np.dtype`
        """
        return np.int

    @property
    def n_members(self):
        """
        Returns the number of members in the domain if it
        `is_finite`, otherwise, returns `None`.

        :type: ``int``
        """
        if self.is_finite:
            return self.max - self.min + 1
        else:
            return None

    @property
    def example_point(self):
        """
        Returns any single point guaranteed to be in the domain, but
        no other guarantees; useful for testing purposes.
        This is given as a size 1 ``np.array`` of type ``dtype``.

        :type: ``np.ndarray``
        """
        if self.min is not None:
            return np.array([self._min], dtype=self.dtype)
        if self.max is not None:
            return np.array([self._max], dtype=self.dtype)
        else:
            return np.array([0], dtype=self.dtype)

    @property
    def values(self):
        """
        Returns an `np.array` of type `self.dtype` containing
        some values from the domain.
        For domains where ``is_finite`` is ``True``, all elements
        of the domain will be yielded exactly once.

        :rtype: `np.ndarray`
        """
        if self.max is None or self.min is None:
            return self.example_point
        else:
            return np.arange(self.min, self.max + 1, dtype = self.dtype)

    ## METHODS ##

    def in_domain(self, points):
        """
        Returns ``True`` if all of the given points are in the domain,
        ``False`` otherwise.

        :param np.ndarray points: An `np.ndarray` of type `self.dtype`.

        :rtype: `bool`
        """
        are_integer = np.all(np.mod(points,1) == 0)
        are_greater = True if self._min is None else np.all(points >= self._min)
        are_smaller = True if self._max is None else np.all(points <= self._max)
        return  are_integer and are_greater and are_smaller

class MultinomialDomain(Domain):
    """
    A domain specifying k-tuples of non-negative integers which
    sum to a specific value.

    :param int n_meas: The sum of any tuple in the domain.
    :param int n_elements: The number of elements in a tuple.
    """

    def __init__(self, n_meas, n_elements=2):
        self._n_elements = n_elements
        self._n_meas = n_meas

    ## PROPERTIES ##

    @property
    def n_meas(self):
        """
        Returns the sum of any tuple in the domain.

        :rtype: `int`
        """
        return self._n_meas
    @property
    def n_elements(self):
        """
        Returns the number of elements of a tuple in the domain.

        :rtype: `int`
        """
        return self._n_elements

    @property
    def is_continuous(self):
        """
        Whether or not the domain has an uncountable number of values.

        :type: `bool`
        """
        return False

    @property
    def is_finite(self):
        """
        Whether or not the domain contains a finite number of points.

        :type: `bool`
        """
        return True

    @property
    def dtype(self):
        """
        The numpy dtype of a single element of the domain.

        :type: `np.dtype`
        """
        return np.dtype([('k', np.int, self.n_elements)])

    @property
    def n_members(self):
        """
        Returns the number of members in the domain if it
        `is_finite`, otherwise, returns `None`.

        :type: ``int``
        """
        return int(binom(self.n_meas + self.n_elements -1, self.n_elements - 1))

    @property
    def example_point(self):
        """
        Returns any single point guaranteed to be in the domain, but
        no other guarantees; useful for testing purposes.
        This is given as a size 1 ``np.array`` of type ``dtype``.

        :type: ``np.ndarray``
        """
        return np.array([([self.n_meas] + [0] * (self.n_elements-1))], dtype=self.dtype)

    @property
    def values(self):
        """
        Returns an `np.array` of type `self.dtype` containing
        some values from the domain.
        For domains where ``is_finite`` is ``True``, all elements
        of the domain will be yielded exactly once.

        :rtype: `np.ndarray`
        """

        # This code comes from Jared Goguen at http://stackoverflow.com/a/37712597/1082565
        partition_array = np.empty((self.n_members, self.n_elements), dtype=int)
        masks = np.identity(self.n_elements, dtype=int)
        for i, c in enumerate(combinations_with_replacement(masks, self.n_meas)):
            partition_array[i,:] = sum(c)

        # Convert to dtype before returning
        return self.from_regular_array(partition_array)

    ## METHODS ##

    def to_regular_array(self, A):
        """
        Converts from an array of type `self.dtype` to an array
        of type `int` with an additional index labeling the
        tuple indeces.

        :param np.ndarray A: An `np.array` of type `self.dtype`.

        :rtype: `np.ndarray`
        """
        # this could be a static method, but we choose to be consistent with
        # from_regular_array
        return A.view((int, len(A.dtype.names))).reshape(A.shape + (-1,))

    def from_regular_array(self, A):
        """
        Converts from an array of type `int` where the last index
        is assumed to have length `self.n_elements` to an array
        of type `self.d_type` with one fewer index.

        :param np.ndarray A: An `np.array` of type `int`.

        :rtype: `np.ndarray`
        """
        dims = A.shape[:-1]
        return A.reshape((np.prod(dims),-1)).view(dtype=self.dtype).squeeze(-1).reshape(dims)

    def in_domain(self, points):
        """
        Returns ``True`` if all of the given points are in the domain,
        ``False`` otherwise.

        :param np.ndarray points: An `np.ndarray` of type `self.dtype`.

        :rtype: `bool`
        """
        array_view = self.to_regular_array(points)
        print(np.all(array_view>=0))
        return np.all(array_view >= 0) and np.all(np.sum(array_view, axis=-1) == self.n_meas)

    <|endoftext|>


<commit_before>#! /usr/bin/python
# -*- coding: utf-8 -*-

'''
A simple sample reader for a source...
'''

import collections

ChapterInfo = collections.namedtuple('ChapterInfo', ('name', 'url'))
class ImageInfo(collections.namedtuple('innerImageInfo', ('chapter_info', 'url', 'img_url', 'next_page'))):
    def __new__(cls, chapter_info, url, img_url=None, next_page=None):
      super(cls, ImageInfo).__new__(chapter_info, url, img_url, next_page)

class History(object):
  def __init__(self):
    self.dt={}
    self.dt['info'] = {}
    self.dt['info']['chapters'] = {} # chapter name -> (chapter_url, )
    self.dt['info']['images'] = {} # url -> (chapter name, img url)
    self.dt['info']['url_map'] = {} # page/image url -> next page url
    self.dt['done']['chapters'] = {} # chapter name -> local path
    self.dt['done']['images'] = {} # img url -> local file

  def save(self, dest):
    json.dump(self.dt, dest)

  def load(self, dest):
    self.dt = json.load(dh)

  def add_chapter_info(self, ci):
    self.dt['info']['chapters'][ci.name] = (ci.url, )

  def add_image_info(self, ii):
    self.dt['info']['images'][ii.url] = (ii.chapter_info.name, ii.img_url)
    self.dt['info']['url_map'][ii.url] = ii.next_page

class Reader(object):
    def __init__(self, data):
      self._data = data
      pass

    def parse_serie(self, serie_page):
      '''
      Read the serie page and return a list of ImageInfo

      The page is a BeautifulSoup document

      Best practice is to yield ImageInfo
      '''
      return []
<commit_msg>Add getter/setter for history information<commit_after>#! /usr/bin/python
# -*- coding: utf-8 -*-

'''
A simple sample reader for a source...
'''

import collections

ChapterInfo = collections.namedtuple('ChapterInfo', ('name', 'url'))
class ImageInfo(collections.namedtuple('innerImageInfo', ('chapter_info', 'url', 'img_url', 'next_page'))):
    def __new__(cls, chapter_info, url, img_url=None, next_page=None):
      super(cls, ImageInfo).__new__(chapter_info, url, img_url, next_page)

class History(object):
  def __init__(self, chapters_to_skip=tuple()):
    self.chapters_to_skip = chapters_to_skip
    self.dt={}
    self.dt['info'] = {}
    self.dt['info']['chapters'] = {} # chapter name -> (chapter_url, )
    self.dt['info']['images'] = {} # url -> (chapter name, img url)
    self.dt['info']['url_map'] = {} # page/image url -> next page url
    self.dt['done']['chapters'] = {} # chapter name -> local path
    self.dt['done']['images'] = {} # img url -> local file

  def save(self, dest):
    json.dump(self.dt, dest)

  def load(self, dest):
    self.dt = json.load(dh)

  def add_chapter_info(self, ci):
    self.dt['info']['chapters'][ci.name] = (ci.url, )

  def add_image_info(self, ii):
    self.dt['info']['images'][ii.url] = (ii.chapter_info.name, ii.img_url)
    self.dt['info']['url_map'][ii.url] = ii.next_page

  def skip_chapter(self, chapter_name):
    return self.dt['done']['chapters'].get(chapter_name) is not None or \
      chapter_name in self.chapters_to_skip

  def get_next_page(self, url):
    return self.dt['info']['url_map'].get(url)

class Reader(object):
    def __init__(self, data):
      self._data = data
      pass

    def parse_serie(self, serie_page):
      '''
      Read the serie page and return a list of ImageInfo

      The page is a BeautifulSoup document

      Best practice is to yield ImageInfo
      '''
      return []
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Author: Jonas Molina Ramirez, Kai Steinert
# Version: 0.1
# Date: 12/11/2015
#
import time
from datetime import datetime
from gensim import corpora, models, similarities
import logging
import gensim as gs
import multiprocessing as mp
import os

path = '../resrc/GoogleNews-vectors-negative300.bin'

logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

def run_parallel(knn,model,numberOfFiles,end_idx,start_idx=0):
	# create a list of locks. each lock is reserved for a file
	# and identified by the dictionary's key
	locks = dict([(id,mp.Lock()) for id in range(0,numberOfFiles)])

	# auxiliary counter for assigning locks to processes
	counter = 0

	# counter the number of instantiated processes
	processCounter = 0
	start = time.time()
	logging.info("Start computation of KNN: \n" + str(datetime.fromtimestamp(start)))
	processes = []
	for word in model.index2word[start_idx:end_idx]:
		#parallel(model,item,counter,5,locks.get(counter))
		p = mp.Process(target=parallel,args=(model,word,counter,knn,locks.get(counter)))
		p.start()
		processes.append(p)
		counter += 1
		processCounter += 1
		if(counter%numberOfFiles==0):
			#processCounter = processCounter + counter
			counter = 0
	# wait for processes to finish
	for proc in processes:
		proc.join()
	end = time.time()
	overallSize = 0
	for dirpath,dirnames,filenames in os.walk("tmp"):
		for f in filenames:
			#print f + '\n'
			fp = os.path.join(dirpath,f)
			overallSize += os.path.getsize(fp)

	logging.info("End computation of KNN: \n" + str(datetime.fromtimestamp(end)))
	logging.info("Constructed " + str(processCounter) + " processes.")
	logInfo(start,end,'Wrote files in ')
	logging.info("Overall file size: " + str(overallSize/1024) + "kb")

def logInfo(start, end,txt):
	logging.info(txt + str(end-start) + ' seconds')

def load_model():
	start = time.time()
	model = models.Word2Vec.load_word2vec_format(path, binary=True,encoding='utf8')
	end = time.time()
	logInfo(start,end,'Loaded model in ')
	return model

def parallel(model,word,id,nn,lock):
	pid = str(os.getpid())
	logging.info("Start Pid: " + pid)
	neighbours = dict(model.most_similar(word,topn=nn))
	text = [('\t' + w.encode('utf-8') + '\t' + str(s) + '\n') for (w,s) in neighbours.iteritems()]
	lock.acquire()
	with open("tmp/test_" + str(id) + ".csv", "a") as myfile:
		for line in text:
			myfile.write(word.encode('utf-8') + line)
	lock.release()
	logging.info("End Pid: " + pid)

<commit_msg>added main to sense_parallel.py<commit_after>#!/usr/bin/env python
#
# Author: Jonas Molina Ramirez, Kai Steinert
# Version: 0.1
# Date: 12/11/2015
#
import time
from datetime import datetime
from gensim import corpora, models, similarities
import logging
import gensim as gs
import multiprocessing as mp
import os

path = '../resrc/GoogleNews-vectors-negative300.bin'

logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

def run_parallel(knn,model,numberOfFiles,folder,end_idx,start_idx=0):
	# create a list of locks. each lock is reserved for a file
	# and identified by the dictionary's key
	locks = dict([(id,mp.Lock()) for id in range(0,numberOfFiles)])

	# auxiliary counter for assigning locks to processes
	counter = 0

	# counter the number of instantiated processes
	processCounter = 0
	start = time.time()
	logging.info("Start computation of KNN: \n" + str(datetime.fromtimestamp(start)))
	processes = []
	for word in model.index2word[start_idx:end_idx]:
		#parallel(model,item,counter,5,locks.get(counter))
		p = mp.Process(target=parallel,args=(model,word,counter,knn,locks.get(counter),folder))
		p.start()
		processes.append(p)
		counter += 1
		processCounter += 1
		if(counter%numberOfFiles==0):
			#processCounter = processCounter + counter
			counter = 0
	# wait for processes to finish
	for proc in processes:
		proc.join()
	end = time.time()
	overallSize = 0
	for dirpath,dirnames,filenames in os.walk(folder):
		for f in filenames:
			#print f + '\n'
			fp = os.path.join(dirpath,f)
			overallSize += os.path.getsize(fp)

	logging.info("End computation of KNN: \n" + str(datetime.fromtimestamp(end)))
	logging.info("Constructed " + str(processCounter) + " processes.")
	logInfo(start,end,'Wrote files in ')
	logging.info("Overall file size: " + str(overallSize/1024) + "kb")

def logInfo(start, end,txt):
	logging.info(txt + str(end-start) + ' seconds')

def load_model(path):
	start = time.time()
	model = models.Word2Vec.load_word2vec_format(path, binary=True,encoding='utf8')
	end = time.time()
	logInfo(start,end,'Loaded model in ')
	return model

def parallel(model,word,id,nn,lock,folder):
	pid = str(os.getpid())
	logging.info("Start Pid: " + pid)
	neighbours = dict(model.most_similar(word,topn=nn))
	text = [('\t' + w.encode('utf-8') + '\t' + str(s) + '\n') for (w,s) in neighbours.iteritems()]
	lock.acquire()
	with open(folder+"/test_" + str(id) + ".csv", "a") as myfile:
		for line in text:
			myfile.write(word.encode('utf-8') + line)
	lock.release()
	logging.info("End Pid: " + pid)

if __name__ == "__main__":
	model = load_model(path)
	run_parallel(knn=200,model=model,numberOfFiles=40,folder='tmp1',end_idx=1000)

<|endoftext|>


<commit_before>'''
Created on Jun 30, 2012

@author: dstrauss
'''
import numpy as np

def getDefaults(solverType, flavor, customDefs):
    ''' just a stub function that will allow me to produce a dictionary that
    gives default values, also populates with new preferences.'''

    if (solverType == 'splitField') & (flavor == 'TE'):
        D = {'rho':2320.27, 'xi':9.811e-4,'lmb':0.0}

    elif (solverType =='splitField') & (flavor == 'TM'):
        D = {'rho':0.019307, 'xi':1.3895e-3, 'lmb':0.0}

    elif (solverType =='sba'):
        D = {'rho':0.005, 'xi':0.9, 'lmb':0}

    elif (solverType =='contrastX') & (flavor =='TE'):
        D = {'rho':1e-3, 'xi':2e-3, 'lmb':0}

    elif (solverType =='biconvex') & (flavor=='TE'):
        D = {'rho':2069.1, 'xi':1e-5, 'lmb':0}

    elif (solverType =='contrastX') & (flavor =='TM'):
        D = {'rho':1e-3, 'xi':2e-3, 'lmb':0}

    elif (solverType =='splitField') & (flavor =='both'):
        D = {'rho':1e-3, 'xi':1e-3,'lmb':0}

    else:
        print 'somehow you did not specify a valid combination'

    D['bkgNo'] = 1
    D['outDir'] = 'testing/'
    D['uBound'] = 0.05
    D['rom'] = False
    D['freqs'] = np.array([1e3, 3e3, 13e3, 50e3])
    D['inc'] = np.array([75, -75, 45, -45])*np.pi/180
    D['maxIter'] = 1000
    D['bkgSig'] = 0.005;
    D['flavor'] = flavor
    D['numSensors'] = 10
    # D['flavor'] = 'TE' # do this by default, although it should be set explicitly

    for key,value in customDefs.iteritems():
        D[key] = value

    return D
        <commit_msg>update in solverDefaults to have a a soft Contrast X<commit_after>'''
Created on Jun 30, 2012

@author: dstrauss
'''
import numpy as np

def getDefaults(solverType, flavor, customDefs):
    ''' just a stub function that will allow me to produce a dictionary that
    gives default values, also populates with new preferences.'''

    if (solverType == 'splitField') & (flavor == 'TE'):
        D = {'rho':2320.27, 'xi':9.811e-4,'lmb':0.0}

    elif (solverType =='splitField') & (flavor == 'TM'):
        D = {'rho':0.019307, 'xi':1.3895e-3, 'lmb':0.0}

    elif (solverType =='sba'):
        D = {'rho':0.005, 'xi':0.9, 'lmb':0}

    elif (solverType =='contrastX') & (flavor =='TE'):
        D = {'rho':1e-3, 'xi':2e-3, 'lmb':0}

    elif (solverType =='biconvex') & (flavor=='TE'):
        D = {'rho':2069.1, 'xi':1e-5, 'lmb':0}

    elif (solverType =='contrastX') & (flavor =='TM'):
        D = {'rho':1e-3, 'xi':2e-3, 'lmb':0}

    elif (solverType =='splitField') & (flavor =='both'):
        D = {'rho':1e-3, 'xi':1e-3,'lmb':0}

    elif (solverType == 'contrastSoftX') & (flavor=='TE'):
        D = {'rho':1e-3, 'xi':1e-3,'lmb':0}

    else:
        print 'somehow you did not specify a valid combination'

    D['bkgNo'] = 1
    D['outDir'] = 'testing/'
    D['uBound'] = 0.05
    D['rom'] = False
    D['freqs'] = np.array([1e3, 3e3, 13e3, 50e3])
    D['inc'] = np.array([75, -75, 45, -45])*np.pi/180
    D['maxIter'] = 1000
    D['bkgSig'] = 0.005;
    D['flavor'] = flavor
    D['numSensors'] = 10
    # D['flavor'] = 'TE' # do this by default, although it should be set explicitly

    for key,value in customDefs.iteritems():
        D[key] = value

    return D
        <|endoftext|>


<commit_before>import os, os.path

def count_files( dir ):
	return len([name for name in os.listdir(dir) if os.path.isfile(os.path.join(dir, name))])

music_dir = '/media/kxstudio/Wisdom/music/From Parts Unknown'

print "folder", music_dir, "contains", count_files(music_dir), "files."
<commit_msg>added recursive funct<commit_after>import os, os.path

def count_files( dir ):
	return len([name for name in os.listdir(dir) if os.path.isfile(os.path.join(dir, name))])

def count_files_rec(dir):
	"this will count recursively in sub folders"
	path, dirs, files = os.walk("/usr/lib").next()
	return len(files)

# music_dir = '/media/kxstudio/Wisdom/music/From Parts Unknown'
music_dir = '/media/kxstudio/Wisdom/music/Audiobooks'

print "folder", music_dir, "contains", count_files_rec(music_dir), "files."
<|endoftext|>