<commit_before>import os
# os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
# os.environ["CUDA_VISIBLE_DEVICES"] = "1"

from src.python.consts import *

from src.python.preprocess import *

from src.python.geneontology import *

from src.python.baselines import *

from pymongo import MongoClient

from tqdm import tqdm

import tensorflow as tf

### Keras
from keras import optimizers
from keras.models import Model
from keras.layers import Input, Dense, Embedding, Activation
from keras.layers import Conv2D, Conv1D
from keras.layers import Dropout, BatchNormalization
from keras.layers import MaxPooling2D, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.layers import Concatenate, Flatten, Reshape
from keras.callbacks import Callback, EarlyStopping, ModelCheckpoint, LambdaCallback, LearningRateScheduler
# from keras.losses import hinge, binary_crossentropy

from keras import backend as K

from sklearn.metrics import log_loss

import math

import argparse

sess = tf.Session()
K.set_session(sess)

LR = 0.001

BATCH_SIZE = 32

t0 = datetime(2014, 1, 1, 0, 0)
t1 = datetime(2014, 9, 1, 0, 0)

MAX_LENGTH = 2000
MIN_LENGTH = 100

def get_training_and_validation_streams(db, onto, classes, limit=None):
    q_train = {'DB': 'UniProtKB',
               'Evidence': {'$in': exp_codes},
               'Date': {"$lte": t0},
               'Aspect': ASPECT}
    seq2go_trn, _ = GoAnnotationCollectionLoader(db.goa_uniprot.find(q_train), db.goa_uniprot.count(q_train), ASPECT).load()
    query = {"_id": {"$in": unique(list(seq2go_trn.keys())).tolist()}}
    count = limit if limit else db.uniprot.count(query)
    source = db.uniprot.find(query).batch_size(10)
    if limit: source = source.limit(limit)
    stream_trn = DataStream(source, count, seq2go_trn, onto, classes)

    q_valid = {'DB': 'UniProtKB',
               'Evidence': {'$in': exp_codes},
               'Date': {"$gt": t0, "$lte": t1},
               'Aspect': ASPECT}

    seq2go_tst, _ = GoAnnotationCollectionLoader(db.goa_uniprot.find(q_valid), db.goa_uniprot.count(q_valid), ASPECT).load()
    query = {"_id": {"$in": unique(list(seq2go_tst.keys())).tolist()}}
    count = limit if limit else db.uniprot.count(query)
    source = db.uniprot.find(query).batch_size(10)
    if limit: source = source.limit(limit)
    stream_tst = DataStream(source, count, seq2go_tst, onto, classes)

    return stream_trn, stream_tst

def pad_seq(seq, max_length=MAX_LENGTH):
    seq += [PAD for _ in range(max_length - len(seq))]
    return np.asarray(seq)

class DataStream(object):
    def __init__(self, source, count, seq2go, onto, classes):

        self._classes = classes
        self._count = count
        self._source = source
        self._seq2go = seq2go
        self._onto = onto

    def __iter__(self):

        classes = self._classes
        count = self._count
        source = self._source
        seq2go = self._seq2go
        onto = self._onto

        s_cls = set(classes)

        for k, seq in UniprotCollectionLoader(source, count):
            if not MIN_LENGTH // 4 <= len(seq) <= MAX_LENGTH:
                continue
            y = np.zeros(len(classes))
            for go in onto.propagate(seq2go[k], include_root=False):
                if go not in s_cls:
                    continue
                y[classes.index(go)] = 1

                x = [AA.aa2index[aa] for aa in seq]

            yield k, x, y

    def __len__(self):
        return self._count

def step_decay(epoch):
    initial_lrate = LR
    drop = 0.5
    epochs_drop = 4.0
    lrate = max(0.0001, initial_lrate * math.pow(drop, math.floor((1 + epoch) / epochs_drop)))
    return lrate

def Features(inpt):

    feats = Embedding(input_dim=26, output_dim=23, embeddings_initializer='uniform')(inpt)

    feats = Conv1D(250, 30, activation='relu', padding='valid')(feats)
    feats = Dropout(0.3)(feats)
    feats = Conv1D(250, 10, activation='relu', padding='valid')(feats)
    feats = Dropout(0.3)(feats)
    feats = Conv1D(250, 10, activation='relu', padding='valid')(feats)
    feats = Dropout(0.3)(feats)
    feats = Conv1D(250, 10, activation='relu', padding='valid')(feats)
    feats = Dropout(0.3)(feats)
    # feats = Conv1D(250, 10, activation='relu', padding='valid')(feats)
    # feats = Dropout(0.3)(feats)

    return feats

def Classifier(inpt, classes):
    out = GlobalMaxPooling1D()(inpt)
    out = Dense(len(classes), activation='linear')(out)
    out = BatchNormalization()(out)
    out = Activation('sigmoid')(out)
    return out

def ModelCNN(classes):
    inp = Input(shape=(None,))
    out = Classifier(Features(inp), classes)
    model = Model(inputs=[inp], outputs=[out])
    adam = optimizers.Adam(lr=LR, beta_1=0.9, beta_2=0.999, epsilon=1e-8)
    model.compile(loss='binary_crossentropy', optimizer=adam)

    return model

def batch_generator(stream):

    def prepare(batch):
        ids, X, Y = zip(*batch)
        b = max(MIN_LENGTH, max(map(len, X)))
        X = [pad_seq(seq, b) for seq in X]
        return ids, np.asarray(X), np.asarray(Y)

    batch = []
    for k, x, y in stream:
        if len(batch) == BATCH_SIZE:
            yield prepare(batch)
            batch = []
        batch.append([k, x, y])

    yield prepare(batch)

def add_arguments(parser):
    parser.add_argument("--mongo_url", type=str, default='mongodb://localhost:27017/',
                        help="Supply the URL of MongoDB"),
    parser.add_argument("--aspect", type=str, choices=['F', 'P', 'C'],
                        default="F", help="Specify the ontology aspect.")
    parser.add_argument("--init_epoch", type=int, default=0,
                        help="Which epoch to start training the model?")
    parser.add_argument("--num_epochs", type=int, default=20,
                        help="How many epochs to train the model?")
    parser.add_argument('-r', '--resume', default='', type=str, metavar='PATH',
                        help='path to latest checkpoint (default: none)')

class LossHistory(Callback):
    def __init__(self):
        self.losses = []

    def on_batch_end(self, batch, logs={}):
        self.losses.append(logs.get('loss'))

def train(model, gen_xy, length_xy, epoch, num_epochs,
          history=LossHistory(), lrate=LearningRateScheduler(step_decay)):

    pbar = tqdm(total=length_xy)

    for _, X, Y in gen_xy:
        assert len(X) == len(Y)

        model.fit(x=X, y=Y,
                  batch_size=BATCH_SIZE,
                  epochs=num_epochs,
                  verbose=0,
                  validation_data=None,
                  initial_epoch=epoch,
                  callbacks=[history, ])
        pbar.set_description("Training Loss:%.5f" % np.mean(history.losses))
        pbar.update(len(Y))

    pbar.close()

def zeroone2oneminusone(vec):
    return np.add(np.multiply(np.array(vec), 2), -1)

def oneminusone2zeroone(vec):
    return np.divide(np.add(np.array(vec), 1), 2)

def calc_loss(y_true, y_pred, batch_size=BATCH_SIZE):
    return batch_size * np.mean([log_loss(y, y_hat) for y, y_hat in zip(y_true, y_pred) if np.any(y)])

def predict(model, gen_xy, length_xy, classes):
    pbar = tqdm(total=length_xy, desc="Predicting...")
    i, m, n = 0, length_xy, len(classes)
    y_pred, y_true = np.zeros((m, n)), np.zeros((m, n))
    ids = list()

    for i, (k, X, Y) in enumerate(gen_xy):
        assert len(X) == len(Y)
        k = len(Y)
        y_hat, y = model.predict(X), Y
        y_pred[i:i + k, ], y_true[i:i + k, ] = y_hat, y
        pbar.update(k)
    pbar.close()
    return ids, y_true, y_pred

def evaluate(y_true, y_pred, classes):
    y_pred = y_pred[~np.all(y_pred == 0, axis=1)]
    y_true = y_true[~np.all(y_true == 0, axis=1)]
    prs, rcs, f1s = performance(y_pred, y_true, classes)
    return calc_loss(y_true, y_pred), prs, rcs, f1s

if __name__ == "__main__":

    parser = argparse.ArgumentParser()
    add_arguments(parser)
    args = parser.parse_args()

    ASPECT = args.aspect  # default: Molecular Function

    client = MongoClient(args.mongo_url)

    db = client['prot2vec']

    print("Loading Ontology...")
    onto = get_ontology(ASPECT)

    classes = onto.classes
    classes.remove(onto.root)
    assert onto.root not in classes

    model = ModelCNN(classes)
    if args.resume:
        model.load_weights(args.resume)
        print("Loaded model from disk")

    model.summary()

    for epoch in range(args.init_epoch, args.num_epochs):

        trn_stream, tst_stream = get_training_and_validation_streams(db, onto, classes)

        train(model, batch_generator(trn_stream), len(trn_stream), epoch, args.num_epochs)
        _, y_true, y_pred = predict(model, batch_generator(tst_stream), len(tst_stream), classes)
        loss, prs, rcs, f1s = evaluate(y_true, y_pred, classes)
        i = np.argmax(f1s)

        print("[Epoch %d] (Validation Loss: %.5f, F_max: %.3f, precision: %.3f, recall: %.3f)"
              % (epoch + 1, loss, f1s[i], prs[i], rcs[i]))

        model_str = 'deeperseq-%d-%.5f-%.2f.hdf5' % (epoch + 1, loss, f1s[i])
        model.save_weights("checkpoints/%s.hdf5" % model_str)
        with open("checkpoints/%s.json" % model_str, "w+") as f:
            f.write(model.to_json())
<commit_msg>deeperseq ... (5 layers)<commit_after>import os
# os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
# os.environ["CUDA_VISIBLE_DEVICES"] = "1"

from src.python.consts import *

from src.python.preprocess import *

from src.python.geneontology import *

from src.python.baselines import *

from pymongo import MongoClient

from tqdm import tqdm

import tensorflow as tf

### Keras
from keras import optimizers
from keras.models import Model
from keras.layers import Input, Dense, Embedding, Activation
from keras.layers import Conv2D, Conv1D
from keras.layers import Dropout, BatchNormalization
from keras.layers import MaxPooling2D, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.layers import Concatenate, Flatten, Reshape
from keras.callbacks import Callback, EarlyStopping, ModelCheckpoint, LambdaCallback, LearningRateScheduler
# from keras.losses import hinge, binary_crossentropy

from keras import backend as K

from sklearn.metrics import log_loss

import math

import argparse

sess = tf.Session()
K.set_session(sess)

LR = 0.001

BATCH_SIZE = 32

t0 = datetime(2014, 1, 1, 0, 0)
t1 = datetime(2014, 9, 1, 0, 0)

MAX_LENGTH = 2000
MIN_LENGTH = 100

def get_training_and_validation_streams(db, onto, classes, limit=None):
    q_train = {'DB': 'UniProtKB',
               'Evidence': {'$in': exp_codes},
               'Date': {"$lte": t0},
               'Aspect': ASPECT}
    seq2go_trn, _ = GoAnnotationCollectionLoader(db.goa_uniprot.find(q_train), db.goa_uniprot.count(q_train), ASPECT).load()
    query = {"_id": {"$in": unique(list(seq2go_trn.keys())).tolist()}}
    count = limit if limit else db.uniprot.count(query)
    source = db.uniprot.find(query).batch_size(10)
    if limit: source = source.limit(limit)
    stream_trn = DataStream(source, count, seq2go_trn, onto, classes)

    q_valid = {'DB': 'UniProtKB',
               'Evidence': {'$in': exp_codes},
               'Date': {"$gt": t0, "$lte": t1},
               'Aspect': ASPECT}

    seq2go_tst, _ = GoAnnotationCollectionLoader(db.goa_uniprot.find(q_valid), db.goa_uniprot.count(q_valid), ASPECT).load()
    query = {"_id": {"$in": unique(list(seq2go_tst.keys())).tolist()}}
    count = limit if limit else db.uniprot.count(query)
    source = db.uniprot.find(query).batch_size(10)
    if limit: source = source.limit(limit)
    stream_tst = DataStream(source, count, seq2go_tst, onto, classes)

    return stream_trn, stream_tst

def pad_seq(seq, max_length=MAX_LENGTH):
    seq += [PAD for _ in range(max_length - len(seq))]
    return np.asarray(seq)

class DataStream(object):
    def __init__(self, source, count, seq2go, onto, classes):

        self._classes = classes
        self._count = count
        self._source = source
        self._seq2go = seq2go
        self._onto = onto

    def __iter__(self):

        classes = self._classes
        count = self._count
        source = self._source
        seq2go = self._seq2go
        onto = self._onto

        s_cls = set(classes)

        for k, seq in UniprotCollectionLoader(source, count):
            if not MIN_LENGTH // 4 <= len(seq) <= MAX_LENGTH:
                continue
            y = np.zeros(len(classes))
            for go in onto.propagate(seq2go[k], include_root=False):
                if go not in s_cls:
                    continue
                y[classes.index(go)] = 1

                x = [AA.aa2index[aa] for aa in seq]

            yield k, x, y

    def __len__(self):
        return self._count

def step_decay(epoch):
    initial_lrate = LR
    drop = 0.5
    epochs_drop = 4.0
    lrate = max(0.0001, initial_lrate * math.pow(drop, math.floor((1 + epoch) / epochs_drop)))
    return lrate

def Features(inpt):

    feats = Embedding(input_dim=26, output_dim=23, embeddings_initializer='uniform')(inpt)

    feats = Conv1D(250, 30, activation='relu', padding='valid')(feats)
    feats = Dropout(0.3)(feats)
    feats = Conv1D(250, 10, activation='relu', padding='valid')(feats)
    feats = Dropout(0.3)(feats)
    feats = Conv1D(250, 10, activation='relu', padding='valid')(feats)
    feats = Dropout(0.3)(feats)
    feats = Conv1D(250, 10, activation='relu', padding='valid')(feats)
    feats = Dropout(0.3)(feats)
    feats = Conv1D(250, 10, activation='relu', padding='valid')(feats)
    feats = Dropout(0.3)(feats)

    return feats

def Classifier(inpt, classes):
    out = GlobalMaxPooling1D()(inpt)
    out = Dense(len(classes), activation='linear')(out)
    out = BatchNormalization()(out)
    out = Activation('sigmoid')(out)
    return out

def ModelCNN(classes):
    inp = Input(shape=(None,))
    out = Classifier(Features(inp), classes)
    model = Model(inputs=[inp], outputs=[out])
    adam = optimizers.Adam(lr=LR, beta_1=0.9, beta_2=0.999, epsilon=1e-8)
    model.compile(loss='binary_crossentropy', optimizer=adam)

    return model

def batch_generator(stream):

    def prepare(batch):
        ids, X, Y = zip(*batch)
        b = max(MIN_LENGTH, max(map(len, X)))
        X = [pad_seq(seq, b) for seq in X]
        return ids, np.asarray(X), np.asarray(Y)

    batch = []
    for k, x, y in stream:
        if len(batch) == BATCH_SIZE:
            yield prepare(batch)
            batch = []
        batch.append([k, x, y])

    yield prepare(batch)

def add_arguments(parser):
    parser.add_argument("--mongo_url", type=str, default='mongodb://localhost:27017/',
                        help="Supply the URL of MongoDB"),
    parser.add_argument("--aspect", type=str, choices=['F', 'P', 'C'],
                        default="F", help="Specify the ontology aspect.")
    parser.add_argument("--init_epoch", type=int, default=0,
                        help="Which epoch to start training the model?")
    parser.add_argument("--num_epochs", type=int, default=20,
                        help="How many epochs to train the model?")
    parser.add_argument('-r', '--resume', default='', type=str, metavar='PATH',
                        help='path to latest checkpoint (default: none)')

class LossHistory(Callback):
    def __init__(self):
        self.losses = []

    def on_batch_end(self, batch, logs={}):
        self.losses.append(logs.get('loss'))

def train(model, gen_xy, length_xy, epoch, num_epochs,
          history=LossHistory(), lrate=LearningRateScheduler(step_decay)):

    pbar = tqdm(total=length_xy)

    for _, X, Y in gen_xy:
        assert len(X) == len(Y)

        model.fit(x=X, y=Y,
                  batch_size=BATCH_SIZE,
                  epochs=num_epochs,
                  verbose=0,
                  validation_data=None,
                  initial_epoch=epoch,
                  callbacks=[history, ])
        pbar.set_description("Training Loss:%.5f" % np.mean(history.losses))
        pbar.update(len(Y))

    pbar.close()

def zeroone2oneminusone(vec):
    return np.add(np.multiply(np.array(vec), 2), -1)

def oneminusone2zeroone(vec):
    return np.divide(np.add(np.array(vec), 1), 2)

def calc_loss(y_true, y_pred, batch_size=BATCH_SIZE):
    return batch_size * np.mean([log_loss(y, y_hat) for y, y_hat in zip(y_true, y_pred) if np.any(y)])

def predict(model, gen_xy, length_xy, classes):
    pbar = tqdm(total=length_xy, desc="Predicting...")
    i, m, n = 0, length_xy, len(classes)
    y_pred, y_true = np.zeros((m, n)), np.zeros((m, n))
    ids = list()

    for i, (k, X, Y) in enumerate(gen_xy):
        assert len(X) == len(Y)
        k = len(Y)
        y_hat, y = model.predict(X), Y
        y_pred[i:i + k, ], y_true[i:i + k, ] = y_hat, y
        pbar.update(k)
    pbar.close()
    return ids, y_true, y_pred

def evaluate(y_true, y_pred, classes):
    y_pred = y_pred[~np.all(y_pred == 0, axis=1)]
    y_true = y_true[~np.all(y_true == 0, axis=1)]
    prs, rcs, f1s = performance(y_pred, y_true, classes)
    return calc_loss(y_true, y_pred), prs, rcs, f1s

if __name__ == "__main__":

    parser = argparse.ArgumentParser()
    add_arguments(parser)
    args = parser.parse_args()

    ASPECT = args.aspect  # default: Molecular Function

    client = MongoClient(args.mongo_url)

    db = client['prot2vec']

    print("Loading Ontology...")
    onto = get_ontology(ASPECT)

    classes = onto.classes
    classes.remove(onto.root)
    assert onto.root not in classes

    model = ModelCNN(classes)
    if args.resume:
        model.load_weights(args.resume)
        print("Loaded model from disk")

    model.summary()

    for epoch in range(args.init_epoch, args.num_epochs):

        trn_stream, tst_stream = get_training_and_validation_streams(db, onto, classes)

        train(model, batch_generator(trn_stream), len(trn_stream), epoch, args.num_epochs)
        _, y_true, y_pred = predict(model, batch_generator(tst_stream), len(tst_stream), classes)
        loss, prs, rcs, f1s = evaluate(y_true, y_pred, classes)
        i = np.argmax(f1s)

        print("[Epoch %d] (Validation Loss: %.5f, F_max: %.3f, precision: %.3f, recall: %.3f)"
              % (epoch + 1, loss, f1s[i], prs[i], rcs[i]))

        model_str = 'deeperseq-%d-%.5f-%.2f.hdf5' % (epoch + 1, loss, f1s[i])
        model.save_weights("checkpoints/%s.hdf5" % model_str)
        with open("checkpoints/%s.json" % model_str, "w+") as f:
            f.write(model.to_json())
<|endoftext|>


<commit_before># Copyright (c) 2005 The Regents of The University of Michigan
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Nathan Binkert

import code
import datetime
import os
import socket
import sys

from util import attrdict
import config
from options import OptionParser

__all__ = [ 'options', 'arguments', 'main' ]

usage="%prog [m5 options] script.py [script options]"
version="%prog 2.0"
brief_copyright='''
Copyright (c) 2001-2008
The Regents of The University of Michigan
All Rights Reserved
'''

options = OptionParser(usage=usage, version=version,
                       description=brief_copyright)
add_option = options.add_option
set_group = options.set_group
usage = options.usage

# Help options
add_option('-A', "--authors", action="store_true", default=False,
    help="Show author information")
add_option('-B', "--build-info", action="store_true", default=False,
    help="Show build information")
add_option('-C', "--copyright", action="store_true", default=False,
    help="Show full copyright information")
add_option('-R', "--readme", action="store_true", default=False,
    help="Show the readme")
add_option('-N', "--release-notes", action="store_true", default=False,
    help="Show the release notes")

# Options for configuring the base simulator
add_option('-d', "--outdir", metavar="DIR", default="m5out",
    help="Set the output directory to DIR [Default: %default]")
add_option('-r', "--redirect-stdout", action="store_true", default=False,
           help="Redirect stdout (& stderr, without -e) to file")
add_option('-e', "--redirect-stderr", action="store_true", default=False,
           help="Redirect stderr to file")
add_option("--stdout-file", metavar="FILE", default="simout",
           help="Filename for -r redirection [Default: %default]")
add_option("--stderr-file", metavar="FILE", default="simerr",
           help="Filename for -e redirection [Default: %default]")
add_option('-i', "--interactive", action="store_true", default=False,
    help="Invoke the interactive interpreter after running the script")
add_option("--pdb", action="store_true", default=False,
    help="Invoke the python debugger before running the script")
add_option('-p', "--path", metavar="PATH[:PATH]", action='append', split=':',
    help="Prepend PATH to the system path when invoking the script")
add_option('-q', "--quiet", action="count", default=0,
    help="Reduce verbosity")
add_option('-v', "--verbose", action="count", default=0,
    help="Increase verbosity")

# Statistics options
set_group("Statistics Options")
add_option("--stats-file", metavar="FILE", default="stats.txt",
    help="Sets the output file for statistics [Default: %default]")

# Configuration Options
set_group("Configuration Options")
add_option("--dump-config", metavar="FILE", default="config.ini",
    help="Dump configuration output file [Default: %default]")

# Debugging options
set_group("Debugging Options")
add_option("--debug-break", metavar="TIME[,TIME]", action='append', split=',',
    help="Cycle to create a breakpoint")
add_option("--remote-gdb-port", type='int', default=7000,
    help="Remote gdb base port")

# Tracing options
set_group("Trace Options")
add_option("--trace-help", action='store_true',
    help="Print help on trace flags")
add_option("--trace-flags", metavar="FLAG[,FLAG]", action='append', split=',',
    help="Sets the flags for tracing (-FLAG disables a flag)")
add_option("--trace-start", metavar="TIME", type='int',
    help="Start tracing at TIME (must be in ticks)")
add_option("--trace-file", metavar="FILE", default="cout",
    help="Sets the output file for tracing [Default: %default]")
add_option("--trace-ignore", metavar="EXPR", action='append', split=':',
    help="Ignore EXPR sim objects")

# Help options
set_group("Help Options")
add_option("--list-sim-objects", action='store_true', default=False,
    help="List all built-in SimObjects, their parameters and default values")

def main():
    import core
    import debug
    import defines
    import event
    import info
    import stats
    import trace

    def check_tracing():
        if defines.TRACING_ON:
            return

        fatal("Tracing is not enabled.  Compile with TRACING_ON")

    # load the options.py config file to allow people to set their own
    # default options
    options_file = config.get('options.py')
    if options_file:
        scope = { 'options' : options }
        execfile(options_file, scope)

    arguments = options.parse_args()

    if not os.path.isdir(options.outdir):
        os.makedirs(options.outdir)

    # These filenames are used only if the redirect_std* options are set
    stdout_file = os.path.join(options.outdir, options.stdout_file)
    stderr_file = os.path.join(options.outdir, options.stderr_file)

    # Print redirection notices here before doing any redirection
    if options.redirect_stdout and not options.redirect_stderr:
        print "Redirecting stdout and stderr to", stdout_file
    else:
        if options.redirect_stdout:
            print "Redirecting stdout to", stdout_file
        if options.redirect_stderr:
            print "Redirecting stderr to", stderr_file

    # Now redirect stdout/stderr as desired
    if options.redirect_stdout:
        redir_fd = os.open(stdout_file, os. O_WRONLY | os.O_CREAT | os.O_TRUNC)
        os.dup2(redir_fd, sys.stdout.fileno())
        if not options.redirect_stderr:
            os.dup2(redir_fd, sys.stderr.fileno())

    if options.redirect_stderr:
        redir_fd = os.open(stderr_file, os. O_WRONLY | os.O_CREAT | os.O_TRUNC)
        os.dup2(redir_fd, sys.stderr.fileno())

    done = False

    if options.build_info:
        done = True
        print 'Build information:'
        print
        print 'compiled %s' % defines.compileDate;
        print "revision %s" % defines.hgRev
        print 'build options:'
        keys = defines.buildEnv.keys()
        keys.sort()
        for key in keys:
            val = defines.buildEnv[key]
            print '    %s = %s' % (key, val)
        print

    if options.copyright:
        done = True
        print info.LICENSE
        print

    if options.authors:
        done = True
        print 'Author information:'
        print
        print info.AUTHORS
        print

    if options.readme:
        done = True
        print 'Readme:'
        print
        print info.README
        print

    if options.release_notes:
        done = True
        print 'Release Notes:'
        print
        print info.RELEASE_NOTES
        print

    if options.trace_help:
        done = True
        check_tracing()
        trace.help()

    if options.list_sim_objects:
        import SimObject
        done = True
        print "SimObjects:"
        objects = SimObject.allClasses.keys()
        objects.sort()
        for name in objects:
            obj = SimObject.allClasses[name]
            print "    %s" % obj
            params = obj._params.keys()
            params.sort()
            for pname in params:
                param = obj._params[pname]
                default = getattr(param, 'default', '')
                print "        %s" % pname
                if default:
                    print "            default: %s" % default
                print "            desc: %s" % param.desc
                print
            print

    if done:
        sys.exit(0)

    # setting verbose and quiet at the same time doesn't make sense
    if options.verbose > 0 and options.quiet > 0:
        options.usage(2)

    verbose = options.verbose - options.quiet
    if options.verbose >= 0:
        print "M5 Simulator System"
        print brief_copyright
        print

        print "M5 compiled %s" % defines.compileDate;
        print "M5 revision %s" % defines.hgRev

        print "M5 started %s" % datetime.datetime.now().strftime("%b %e %Y %X")
        print "M5 executing on %s" % socket.gethostname()

        print "command line:",
        for argv in sys.argv:
            print argv,
        print

    # check to make sure we can find the listed script
    if not arguments or not os.path.isfile(arguments[0]):
        if arguments and not os.path.isfile(arguments[0]):
            print "Script %s not found" % arguments[0]

        options.usage(2)

    # tell C++ about output directory
    core.setOutputDir(options.outdir)

    # update the system path with elements from the -p option
    sys.path[0:0] = options.path

    # set stats options
    stats.initText(options.stats_file)

    # set debugging options
    debug.setRemoteGDBPort(options.remote_gdb_port)
    for when in options.debug_break:
        debug.schedBreakCycle(int(when))

    if options.trace_flags:
        check_tracing()

        on_flags = []
        off_flags = []
        for flag in options.trace_flags:
            off = False
            if flag.startswith('-'):
                flag = flag[1:]
                off = True
            if flag not in trace.flags.all and flag != "All":
                print >>sys.stderr, "invalid trace flag '%s'" % flag
                sys.exit(1)

            if off:
                off_flags.append(flag)
            else:
                on_flags.append(flag)

        for flag in on_flags:
            trace.set(flag)

        for flag in off_flags:
            trace.clear(flag)

    if options.trace_start:
        check_tracing()
        e = event.create(trace.enable, event.Event.Trace_Enable_Pri)
        event.mainq.schedule(e, options.trace_start)
    else:
        trace.enable()

    trace.output(options.trace_file)

    for ignore in options.trace_ignore:
        check_tracing()
        trace.ignore(ignore)

    sys.argv = arguments
    sys.path = [ os.path.dirname(sys.argv[0]) ] + sys.path

    filename = sys.argv[0]
    filedata = file(filename, 'r').read()
    filecode = compile(filedata, filename, 'exec')
    scope = { '__file__' : filename,
              '__name__' : '__m5_main__' }

    # we want readline if we're doing anything interactive
    if options.interactive or options.pdb:
        exec "import readline" in scope

    # if pdb was requested, execfile the thing under pdb, otherwise,
    # just do the execfile normally
    if options.pdb:
        import pdb
        import traceback

        pdb = pdb.Pdb()
        try:
            pdb.run(filecode, scope)
        except SystemExit:
            print "The program exited via sys.exit(). Exit status: ",
            print sys.exc_info()[1]
        except:
            traceback.print_exc()
            print "Uncaught exception. Entering post mortem debugging"
            t = sys.exc_info()[2]
            while t.tb_next is not None:
                t = t.tb_next
                pdb.interaction(t.tb_frame,t)
    else:
        exec filecode in scope

    # once the script is done
    if options.interactive:
        interact = code.InteractiveConsole(scope)
        interact.interact("M5 Interactive Console")

if __name__ == '__main__':
    from pprint import pprint

    # load the options.py config file to allow people to set their own
    # default options
    options_file = config.get('options.py')
    if options_file:
        scope = { 'options' : options }
        execfile(options_file, scope)

    arguments = options.parse_args()

    print 'opts:'
    pprint(options, indent=4)
    print

    print 'args:'
    pprint(arguments, indent=4)
<commit_msg>python: pull out common code from main that processes arguments<commit_after># Copyright (c) 2005 The Regents of The University of Michigan
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Nathan Binkert

import code
import datetime
import os
import socket
import sys

from util import attrdict
import config
from options import OptionParser

__all__ = [ 'options', 'arguments', 'main' ]

usage="%prog [m5 options] script.py [script options]"
version="%prog 2.0"
brief_copyright='''
Copyright (c) 2001-2008
The Regents of The University of Michigan
All Rights Reserved
'''

options = OptionParser(usage=usage, version=version,
                       description=brief_copyright)
add_option = options.add_option
set_group = options.set_group
usage = options.usage

# Help options
add_option('-A', "--authors", action="store_true", default=False,
    help="Show author information")
add_option('-B', "--build-info", action="store_true", default=False,
    help="Show build information")
add_option('-C', "--copyright", action="store_true", default=False,
    help="Show full copyright information")
add_option('-R', "--readme", action="store_true", default=False,
    help="Show the readme")
add_option('-N', "--release-notes", action="store_true", default=False,
    help="Show the release notes")

# Options for configuring the base simulator
add_option('-d', "--outdir", metavar="DIR", default="m5out",
    help="Set the output directory to DIR [Default: %default]")
add_option('-r', "--redirect-stdout", action="store_true", default=False,
           help="Redirect stdout (& stderr, without -e) to file")
add_option('-e', "--redirect-stderr", action="store_true", default=False,
           help="Redirect stderr to file")
add_option("--stdout-file", metavar="FILE", default="simout",
           help="Filename for -r redirection [Default: %default]")
add_option("--stderr-file", metavar="FILE", default="simerr",
           help="Filename for -e redirection [Default: %default]")
add_option('-i', "--interactive", action="store_true", default=False,
    help="Invoke the interactive interpreter after running the script")
add_option("--pdb", action="store_true", default=False,
    help="Invoke the python debugger before running the script")
add_option('-p', "--path", metavar="PATH[:PATH]", action='append', split=':',
    help="Prepend PATH to the system path when invoking the script")
add_option('-q', "--quiet", action="count", default=0,
    help="Reduce verbosity")
add_option('-v', "--verbose", action="count", default=0,
    help="Increase verbosity")

# Statistics options
set_group("Statistics Options")
add_option("--stats-file", metavar="FILE", default="stats.txt",
    help="Sets the output file for statistics [Default: %default]")

# Configuration Options
set_group("Configuration Options")
add_option("--dump-config", metavar="FILE", default="config.ini",
    help="Dump configuration output file [Default: %default]")

# Debugging options
set_group("Debugging Options")
add_option("--debug-break", metavar="TIME[,TIME]", action='append', split=',',
    help="Cycle to create a breakpoint")
add_option("--remote-gdb-port", type='int', default=7000,
    help="Remote gdb base port")

# Tracing options
set_group("Trace Options")
add_option("--trace-help", action='store_true',
    help="Print help on trace flags")
add_option("--trace-flags", metavar="FLAG[,FLAG]", action='append', split=',',
    help="Sets the flags for tracing (-FLAG disables a flag)")
add_option("--trace-start", metavar="TIME", type='int',
    help="Start tracing at TIME (must be in ticks)")
add_option("--trace-file", metavar="FILE", default="cout",
    help="Sets the output file for tracing [Default: %default]")
add_option("--trace-ignore", metavar="EXPR", action='append', split=':',
    help="Ignore EXPR sim objects")

# Help options
set_group("Help Options")
add_option("--list-sim-objects", action='store_true', default=False,
    help="List all built-in SimObjects, their parameters and default values")

# load the options.py config file to allow people to set their own
# default options
options_file = config.get('options.py')
if options_file:
    scope = { 'options' : options }
    execfile(options_file, scope)

arguments = options.parse_args()

def main():
    import core
    import debug
    import defines
    import event
    import info
    import stats
    import trace

    def check_tracing():
        if defines.TRACING_ON:
            return

        fatal("Tracing is not enabled.  Compile with TRACING_ON")

    if not os.path.isdir(options.outdir):
        os.makedirs(options.outdir)

    # These filenames are used only if the redirect_std* options are set
    stdout_file = os.path.join(options.outdir, options.stdout_file)
    stderr_file = os.path.join(options.outdir, options.stderr_file)

    # Print redirection notices here before doing any redirection
    if options.redirect_stdout and not options.redirect_stderr:
        print "Redirecting stdout and stderr to", stdout_file
    else:
        if options.redirect_stdout:
            print "Redirecting stdout to", stdout_file
        if options.redirect_stderr:
            print "Redirecting stderr to", stderr_file

    # Now redirect stdout/stderr as desired
    if options.redirect_stdout:
        redir_fd = os.open(stdout_file, os. O_WRONLY | os.O_CREAT | os.O_TRUNC)
        os.dup2(redir_fd, sys.stdout.fileno())
        if not options.redirect_stderr:
            os.dup2(redir_fd, sys.stderr.fileno())

    if options.redirect_stderr:
        redir_fd = os.open(stderr_file, os. O_WRONLY | os.O_CREAT | os.O_TRUNC)
        os.dup2(redir_fd, sys.stderr.fileno())

    done = False

    if options.build_info:
        done = True
        print 'Build information:'
        print
        print 'compiled %s' % defines.compileDate;
        print "revision %s" % defines.hgRev
        print 'build options:'
        keys = defines.buildEnv.keys()
        keys.sort()
        for key in keys:
            val = defines.buildEnv[key]
            print '    %s = %s' % (key, val)
        print

    if options.copyright:
        done = True
        print info.LICENSE
        print

    if options.authors:
        done = True
        print 'Author information:'
        print
        print info.AUTHORS
        print

    if options.readme:
        done = True
        print 'Readme:'
        print
        print info.README
        print

    if options.release_notes:
        done = True
        print 'Release Notes:'
        print
        print info.RELEASE_NOTES
        print

    if options.trace_help:
        done = True
        check_tracing()
        trace.help()

    if options.list_sim_objects:
        import SimObject
        done = True
        print "SimObjects:"
        objects = SimObject.allClasses.keys()
        objects.sort()
        for name in objects:
            obj = SimObject.allClasses[name]
            print "    %s" % obj
            params = obj._params.keys()
            params.sort()
            for pname in params:
                param = obj._params[pname]
                default = getattr(param, 'default', '')
                print "        %s" % pname
                if default:
                    print "            default: %s" % default
                print "            desc: %s" % param.desc
                print
            print

    if done:
        sys.exit(0)

    # setting verbose and quiet at the same time doesn't make sense
    if options.verbose > 0 and options.quiet > 0:
        options.usage(2)

    verbose = options.verbose - options.quiet
    if options.verbose >= 0:
        print "M5 Simulator System"
        print brief_copyright
        print

        print "M5 compiled %s" % defines.compileDate;
        print "M5 revision %s" % defines.hgRev

        print "M5 started %s" % datetime.datetime.now().strftime("%b %e %Y %X")
        print "M5 executing on %s" % socket.gethostname()

        print "command line:",
        for argv in sys.argv:
            print argv,
        print

    # check to make sure we can find the listed script
    if not arguments or not os.path.isfile(arguments[0]):
        if arguments and not os.path.isfile(arguments[0]):
            print "Script %s not found" % arguments[0]

        options.usage(2)

    # tell C++ about output directory
    core.setOutputDir(options.outdir)

    # update the system path with elements from the -p option
    sys.path[0:0] = options.path

    # set stats options
    stats.initText(options.stats_file)

    # set debugging options
    debug.setRemoteGDBPort(options.remote_gdb_port)
    for when in options.debug_break:
        debug.schedBreakCycle(int(when))

    if options.trace_flags:
        check_tracing()

        on_flags = []
        off_flags = []
        for flag in options.trace_flags:
            off = False
            if flag.startswith('-'):
                flag = flag[1:]
                off = True
            if flag not in trace.flags.all and flag != "All":
                print >>sys.stderr, "invalid trace flag '%s'" % flag
                sys.exit(1)

            if off:
                off_flags.append(flag)
            else:
                on_flags.append(flag)

        for flag in on_flags:
            trace.set(flag)

        for flag in off_flags:
            trace.clear(flag)

    if options.trace_start:
        check_tracing()
        e = event.create(trace.enable, event.Event.Trace_Enable_Pri)
        event.mainq.schedule(e, options.trace_start)
    else:
        trace.enable()

    trace.output(options.trace_file)

    for ignore in options.trace_ignore:
        check_tracing()
        trace.ignore(ignore)

    sys.argv = arguments
    sys.path = [ os.path.dirname(sys.argv[0]) ] + sys.path

    filename = sys.argv[0]
    filedata = file(filename, 'r').read()
    filecode = compile(filedata, filename, 'exec')
    scope = { '__file__' : filename,
              '__name__' : '__m5_main__' }

    # we want readline if we're doing anything interactive
    if options.interactive or options.pdb:
        exec "import readline" in scope

    # if pdb was requested, execfile the thing under pdb, otherwise,
    # just do the execfile normally
    if options.pdb:
        import pdb
        import traceback

        pdb = pdb.Pdb()
        try:
            pdb.run(filecode, scope)
        except SystemExit:
            print "The program exited via sys.exit(). Exit status: ",
            print sys.exc_info()[1]
        except:
            traceback.print_exc()
            print "Uncaught exception. Entering post mortem debugging"
            t = sys.exc_info()[2]
            while t.tb_next is not None:
                t = t.tb_next
                pdb.interaction(t.tb_frame,t)
    else:
        exec filecode in scope

    # once the script is done
    if options.interactive:
        interact = code.InteractiveConsole(scope)
        interact.interact("M5 Interactive Console")

if __name__ == '__main__':
    from pprint import pprint

    print 'opts:'
    pprint(options, indent=4)
    print

    print 'args:'
    pprint(arguments, indent=4)
<|endoftext|>


<commit_before>from twisted.plugin import IPlugin
from twisted.words.protocols import irc
from txircd.module_interface import Command, ICommand, IModuleData, ModuleData
from txircd.utils import ModeType, timestamp
from zope.interface import implements

irc.RPL_CREATIONTIME = "329"

class ModeCommand(ModuleData):
    implements(IPlugin, IModuleData)

    name = "ModeCommand"
    core = True
    minLevel = 100

    def hookIRCd(self, ircd):
        self.ircd = ircd

    def actions(self):
        return [ ("modemessage-channel", 1, self.sendChannelModesToUsers),
                ("modechanges-channel", 1, self.sendChannelModesToServers),
                ("modemessage-user", 1, self.sendUserModesToUsers),
                ("modechanges-user", 1, self.sendUserModesToServers),
                ("commandpermission-MODE", 1, self.restrictUse) ]

    def userCommands(self):
        return [ ("MODE", 1, UserMode(self.ircd)) ]

    def serverCommands(self):
        return [ ("MODE", 1, ServerMode(self.ircd)) ]

    def load(self):
        self.rehash()

    def rehash(self):
        newLevel = self.ircd.config.getWithDefault("channel_minimum_level_mode", 100)
        try:
            self.minLevel = int(newLevel)
        except ValueError:
            try:
                self.minLevel = self.ircd.channelStatuses[newLevel[0]][1]
            except KeyError:
                self.minLevel = 100

    def getOutputModes(self, modes):
        addInStr = None
        modeStrList = []
        params = []
        modeLists = []
        modeLen = 0
        for modeData in modes:
            adding, mode, param = modeData
            paramLen = 0
            if param is not None:
                paramLen = len(param)
            if modeLen + paramLen + 3 > 300: # Don't let the mode output get too long
                modeLists.append(["".join(modeStrList)] + params)
                addInStr = None
                modeStrList = []
                params = []
                modeLen = 0
            if adding != addInStr:
                if adding:
                    modeStrList.append("+")
                else:
                    modeStrList.append("-")
                addInStr = adding
                modeLen += 1
            modeStrList.append(mode)
            modeLen += 1
            if param is not None:
                params.append(param)
                modeLen += 1 + paramLen
        modeLists.append(["".join(modeStrList)] + params)
        return modeLists

    def sendChannelModesToUsers(self, users, channel, source, sourceName, modes):
        modeOuts = self.getOutputModes(modes)
        for modeOut in modeOuts:
            modeStr = modeOut[0]
            params = modeOut[1:]
            for user in users:
                user.sendMessage("MODE", modeStr, *params, prefix=sourceName, to=channel.name)
        del users[:]

    def sendChannelModesToServers(self, channel, source, sourceName, modes):
        modeOuts = self.getOutputModes(modes)

        if source[:3] == self.ircd.serverID:
            fromServer = None
        else:
            fromServer = self.ircd.servers[source[:3]]
            while fromServer.nextClosest != self.ircd.serverID:
                fromServer = self.ircd.servers[fromServer.nextClosest]
        for modeOut in modeOuts:
            modeStr = modeOut[0]
            params = modeOut[1:]
            for server in self.ircd.servers.itervalues():
                if server.nextClosest == self.ircd.serverID and server != fromServer:
                    server.sendMessage("MODE", channel.name, str(timestamp(channel.existedSince)), modeStr, *params, prefix=source)

    def sendUserModesToUsers(self, users, user, source, sourceName, modes):
        modeOuts = self.getOutputModes(modes)
        for modeOut in modeOuts:
            modeStr = modeOut[0]
            params = modeOut[1:]
            for u in users:
                u.sendMessage("MODE", modeStr, *params, prefix=sourceName, to=user.nick)
        del users[:]

    def sendUserModesToServers(self, user, source, sourceName, modes):
        modeOuts = self.getOutputModes(modes)

        if source[:3] == self.ircd.serverID:
            fromServer = None
        else:
            fromServer = self.ircd.servers[source[:3]]
            while fromServer.nextClosest != self.ircd.serverID:
                fromServer = self.ircd.servers[fromServer.nextClosest]
        for modeOut in modeOuts:
            modeStr = modeOut[0]
            params = modeOut[1:]
            for server in self.ircd.servers.itervalues():
                if server.nextClosest == self.ircd.serverID and server != fromServer:
                    server.sendMessage("MODE", user.uuid, str(timestamp(user.connectedSince)), modeStr, *params, prefix=source)

    def restrictUse(self, user, command, data):
        if "channel" not in data or "modes" not in data:
            return None
        if not data["params"]:
            for mode in data["modes"]:
                if mode != "+" and mode != "-" and (mode not in self.ircd.channelModeTypes or self.ircd.channelModeTypes[mode] != ModeType.List):
                    break
            else:
                return None # All the modes are list modes, and there are no parameters, so we're listing list mode parameters
        channel = data["channel"]
        if channel.userRank(user) < self.minLevel:
            user.sendSingleError("ModeCmd", irc.ERR_CHANOPRIVSNEEDED, channel.name, ":You do not have access to set channel modes")
            return False
        return None

class UserMode(Command):
    implements(ICommand)

    def __init__(self, ircd):
        self.ircd = ircd

    def parseParams(self, user, params, prefix, tags):
        if not params or not params[0]:
            user.sendSingleError("ModeCmd", irc.ERR_NEEDMOREPARAMS, "MODE", ":Not enough parameters")
            return None
        channel = None
        if params[0] in self.ircd.channels:
            channel = self.ircd.channels[params[0]]
        elif params[0] in self.ircd.userNicks:
            if self.ircd.userNicks[params[0]] != user.uuid:
                user.sendSingleError("ModeCmd", irc.ERR_USERSDONTMATCH, ":Can't operate on modes for other users")
                return None
        else:
            user.sendSingleError("ModeCmd", irc.ERR_NOSUCHNICK, params[0], ":No such nick/channel")
            return None
        if len(params) == 1:
            if channel:
                return {
                    "channel": channel
                }
            return {}
        modeStr = params[1]
        modeParams = params[2:]
        if channel:
            return {
                "channel": channel,
                "modes": modeStr,
                "params": modeParams
            }
        return {
            "modes": modeStr,
            "params": modeParams
        }

    def affectedChannels(self, user, data):
        if "channel" in data:
            return [ data["channel"] ]
        return []

    def execute(self, user, data):
        if "modes" not in data:
            if "channel" in data:
                channel = data["channel"]
                user.sendMessage(irc.RPL_CHANNELMODEIS, channel.name, channel.modeString(user))
                user.sendMessage(irc.RPL_CREATIONTIME, channel.name, str(timestamp(channel.existedSince)))
                return True
            user.sendMessage(irc.RPL_UMODEIS, user.modeString(user))
            return True
        if "channel" in data:
            channel = data["channel"]
            channel.setModes(user.uuid, data["modes"], data["params"])
            return True
        user.setModes(user.uuid, data["modes"], data["params"])
        return True

class ServerMode(Command):
    implements(ICommand)

    def __init__(self, ircd):
        self.ircd = ircd

    def parseParams(self, server, params, prefix, tags):
        if len(params) < 3:
            return None
        if prefix not in self.ircd.users and prefix not in self.ircd.servers:
            return None # It's safe to say other servers shouldn't be sending modes sourced from us. That's our job!
        if params[0] not in self.ircd.users and params[0] not in self.ircd.channels:
            return None
        try:
            return {
                "source": prefix,
                "target": params[0],
                "timestamp": int(params[1]),
                "modes": params[2],
                "params": params[3:]
            }
        except ValueError:
            return None

    def execute(self, server, data):
        source = data["source"]
        target = data["target"]
        targTS = data["timestamp"]
        if target in self.ircd.channels:
            channel = self.ircd.channels[target]
            if targTS > timestamp(channel.existedSince):
                return True
            if targTS < timestamp(channel.existedSince):
                # We need to remove all of the channel's modes
                while True: # Make a point to continue back to
                    modeStrList = []
                    params = []
                    for mode, param in channel.modes.iteritems():
                        if len(modeStrList) >= 20:
                            break
                        if self.ircd.channelModeTypes[mode] == ModeType.List:
                            for paramData in param:
                                modeStrList.append(mode)
                                params.append(paramData[0])
                                if len(modeStrList) >= 20:
                                    break
                        else:
                            modeStrList.append(mode)
                            if param is not None:
                                params.append(param)
                    for user, statusList in channel.users.iteritems():
                        for status in statusList:
                            if len(modeStrList) >= 20:
                                break
                            modeStrList.append(status)
                            params.append(user.nick)
                    if modeStrList:
                        channel.setModes(source, "-{}".format("".join(modeStrList)), params)
                    if channel.modes: # More processing is to be done
                        continue
                    for status in channel.users.itervalues():
                        if status:
                            break
                    else:
                        break # This one aborts the while True loop when we're done with modes
            channel.setModes(source, data["modes"], data["params"])
            return True
        user = self.ircd.users[target]
        if targTS > timestamp(user.connectedSince):
            return True
        if targTS < timestamp(user.connectedSince):
            while True:
                modeStrList = []
                params = []
                for mode, param in channel.modes.iteritems():
                    if len(modeStrList) >= 20:
                        break
                    if self.ircd.userModeTypes[mode] == ModeType.List:
                        for paramData in param:
                            modeStrList.append(mode)
                            params.append(paramData[0])
                            if len(modeStrList) >= 20:
                                break
                    else:
                        modeStrList.append(mode)
                        if param is not None:
                            params.append(param)
                if modeStrList:
                    user.setModes(source, "-{}".format("".join(modeStrList)), params)
                if not user.modes:
                    break
        user.setModes(source, data["modes"], data["params"])
        return True

modeCommand = ModeCommand()<commit_msg>Fix duplicate MODE messages when user sets modes on self<commit_after>from twisted.plugin import IPlugin
from twisted.words.protocols import irc
from txircd.module_interface import Command, ICommand, IModuleData, ModuleData
from txircd.utils import ModeType, timestamp
from zope.interface import implements

irc.RPL_CREATIONTIME = "329"

class ModeCommand(ModuleData):
    implements(IPlugin, IModuleData)

    name = "ModeCommand"
    core = True
    minLevel = 100

    def hookIRCd(self, ircd):
        self.ircd = ircd

    def actions(self):
        return [ ("modemessage-channel", 1, self.sendChannelModesToUsers),
                ("modechanges-channel", 1, self.sendChannelModesToServers),
                ("modemessage-user", 1, self.sendUserModesToUsers),
                ("modechanges-user", 1, self.sendUserModesToServers),
                ("commandpermission-MODE", 1, self.restrictUse) ]

    def userCommands(self):
        return [ ("MODE", 1, UserMode(self.ircd)) ]

    def serverCommands(self):
        return [ ("MODE", 1, ServerMode(self.ircd)) ]

    def load(self):
        self.rehash()

    def rehash(self):
        newLevel = self.ircd.config.getWithDefault("channel_minimum_level_mode", 100)
        try:
            self.minLevel = int(newLevel)
        except ValueError:
            try:
                self.minLevel = self.ircd.channelStatuses[newLevel[0]][1]
            except KeyError:
                self.minLevel = 100

    def getOutputModes(self, modes):
        addInStr = None
        modeStrList = []
        params = []
        modeLists = []
        modeLen = 0
        for modeData in modes:
            adding, mode, param = modeData
            paramLen = 0
            if param is not None:
                paramLen = len(param)
            if modeLen + paramLen + 3 > 300: # Don't let the mode output get too long
                modeLists.append(["".join(modeStrList)] + params)
                addInStr = None
                modeStrList = []
                params = []
                modeLen = 0
            if adding != addInStr:
                if adding:
                    modeStrList.append("+")
                else:
                    modeStrList.append("-")
                addInStr = adding
                modeLen += 1
            modeStrList.append(mode)
            modeLen += 1
            if param is not None:
                params.append(param)
                modeLen += 1 + paramLen
        modeLists.append(["".join(modeStrList)] + params)
        return modeLists

    def sendChannelModesToUsers(self, users, channel, source, sourceName, modes):
        modeOuts = self.getOutputModes(modes)
        for modeOut in modeOuts:
            modeStr = modeOut[0]
            params = modeOut[1:]
            for user in users:
                user.sendMessage("MODE", modeStr, *params, prefix=sourceName, to=channel.name)
        del users[:]

    def sendChannelModesToServers(self, channel, source, sourceName, modes):
        modeOuts = self.getOutputModes(modes)

        if source[:3] == self.ircd.serverID:
            fromServer = None
        else:
            fromServer = self.ircd.servers[source[:3]]
            while fromServer.nextClosest != self.ircd.serverID:
                fromServer = self.ircd.servers[fromServer.nextClosest]
        for modeOut in modeOuts:
            modeStr = modeOut[0]
            params = modeOut[1:]
            for server in self.ircd.servers.itervalues():
                if server.nextClosest == self.ircd.serverID and server != fromServer:
                    server.sendMessage("MODE", channel.name, str(timestamp(channel.existedSince)), modeStr, *params, prefix=source)

    def sendUserModesToUsers(self, users, user, source, sourceName, modes):
        modeOuts = self.getOutputModes(modes)
        for modeOut in modeOuts:
            modeStr = modeOut[0]
            params = modeOut[1:]
            for u in set(users):
                u.sendMessage("MODE", modeStr, *params, prefix=sourceName, to=user.nick)
        del users[:]

    def sendUserModesToServers(self, user, source, sourceName, modes):
        modeOuts = self.getOutputModes(modes)

        if source[:3] == self.ircd.serverID:
            fromServer = None
        else:
            fromServer = self.ircd.servers[source[:3]]
            while fromServer.nextClosest != self.ircd.serverID:
                fromServer = self.ircd.servers[fromServer.nextClosest]
        for modeOut in modeOuts:
            modeStr = modeOut[0]
            params = modeOut[1:]
            for server in self.ircd.servers.itervalues():
                if server.nextClosest == self.ircd.serverID and server != fromServer:
                    server.sendMessage("MODE", user.uuid, str(timestamp(user.connectedSince)), modeStr, *params, prefix=source)

    def restrictUse(self, user, command, data):
        if "channel" not in data or "modes" not in data:
            return None
        if not data["params"]:
            for mode in data["modes"]:
                if mode != "+" and mode != "-" and (mode not in self.ircd.channelModeTypes or self.ircd.channelModeTypes[mode] != ModeType.List):
                    break
            else:
                return None # All the modes are list modes, and there are no parameters, so we're listing list mode parameters
        channel = data["channel"]
        if channel.userRank(user) < self.minLevel:
            user.sendSingleError("ModeCmd", irc.ERR_CHANOPRIVSNEEDED, channel.name, ":You do not have access to set channel modes")
            return False
        return None

class UserMode(Command):
    implements(ICommand)

    def __init__(self, ircd):
        self.ircd = ircd

    def parseParams(self, user, params, prefix, tags):
        if not params or not params[0]:
            user.sendSingleError("ModeCmd", irc.ERR_NEEDMOREPARAMS, "MODE", ":Not enough parameters")
            return None
        channel = None
        if params[0] in self.ircd.channels:
            channel = self.ircd.channels[params[0]]
        elif params[0] in self.ircd.userNicks:
            if self.ircd.userNicks[params[0]] != user.uuid:
                user.sendSingleError("ModeCmd", irc.ERR_USERSDONTMATCH, ":Can't operate on modes for other users")
                return None
        else:
            user.sendSingleError("ModeCmd", irc.ERR_NOSUCHNICK, params[0], ":No such nick/channel")
            return None
        if len(params) == 1:
            if channel:
                return {
                    "channel": channel
                }
            return {}
        modeStr = params[1]
        modeParams = params[2:]
        if channel:
            return {
                "channel": channel,
                "modes": modeStr,
                "params": modeParams
            }
        return {
            "modes": modeStr,
            "params": modeParams
        }

    def affectedChannels(self, user, data):
        if "channel" in data:
            return [ data["channel"] ]
        return []

    def execute(self, user, data):
        if "modes" not in data:
            if "channel" in data:
                channel = data["channel"]
                user.sendMessage(irc.RPL_CHANNELMODEIS, channel.name, channel.modeString(user))
                user.sendMessage(irc.RPL_CREATIONTIME, channel.name, str(timestamp(channel.existedSince)))
                return True
            user.sendMessage(irc.RPL_UMODEIS, user.modeString(user))
            return True
        if "channel" in data:
            channel = data["channel"]
            channel.setModes(user.uuid, data["modes"], data["params"])
            return True
        user.setModes(user.uuid, data["modes"], data["params"])
        return True

class ServerMode(Command):
    implements(ICommand)

    def __init__(self, ircd):
        self.ircd = ircd

    def parseParams(self, server, params, prefix, tags):
        if len(params) < 3:
            return None
        if prefix not in self.ircd.users and prefix not in self.ircd.servers:
            return None # It's safe to say other servers shouldn't be sending modes sourced from us. That's our job!
        if params[0] not in self.ircd.users and params[0] not in self.ircd.channels:
            return None
        try:
            return {
                "source": prefix,
                "target": params[0],
                "timestamp": int(params[1]),
                "modes": params[2],
                "params": params[3:]
            }
        except ValueError:
            return None

    def execute(self, server, data):
        source = data["source"]
        target = data["target"]
        targTS = data["timestamp"]
        if target in self.ircd.channels:
            channel = self.ircd.channels[target]
            if targTS > timestamp(channel.existedSince):
                return True
            if targTS < timestamp(channel.existedSince):
                # We need to remove all of the channel's modes
                while True: # Make a point to continue back to
                    modeStrList = []
                    params = []
                    for mode, param in channel.modes.iteritems():
                        if len(modeStrList) >= 20:
                            break
                        if self.ircd.channelModeTypes[mode] == ModeType.List:
                            for paramData in param:
                                modeStrList.append(mode)
                                params.append(paramData[0])
                                if len(modeStrList) >= 20:
                                    break
                        else:
                            modeStrList.append(mode)
                            if param is not None:
                                params.append(param)
                    for user, statusList in channel.users.iteritems():
                        for status in statusList:
                            if len(modeStrList) >= 20:
                                break
                            modeStrList.append(status)
                            params.append(user.nick)
                    if modeStrList:
                        channel.setModes(source, "-{}".format("".join(modeStrList)), params)
                    if channel.modes: # More processing is to be done
                        continue
                    for status in channel.users.itervalues():
                        if status:
                            break
                    else:
                        break # This one aborts the while True loop when we're done with modes
            channel.setModes(source, data["modes"], data["params"])
            return True
        user = self.ircd.users[target]
        if targTS > timestamp(user.connectedSince):
            return True
        if targTS < timestamp(user.connectedSince):
            while True:
                modeStrList = []
                params = []
                for mode, param in channel.modes.iteritems():
                    if len(modeStrList) >= 20:
                        break
                    if self.ircd.userModeTypes[mode] == ModeType.List:
                        for paramData in param:
                            modeStrList.append(mode)
                            params.append(paramData[0])
                            if len(modeStrList) >= 20:
                                break
                    else:
                        modeStrList.append(mode)
                        if param is not None:
                            params.append(param)
                if modeStrList:
                    user.setModes(source, "-{}".format("".join(modeStrList)), params)
                if not user.modes:
                    break
        user.setModes(source, data["modes"], data["params"])
        return True

modeCommand = ModeCommand()<|endoftext|>


<commit_before>"""
Compute k-nearest neighbors using brute force search in parallel
via scipy.spatial.distance.cdist and multiprocessing.Pool

psutil is used to evaluate available memory and minimize the number
of parallel jobs for the available resources
"""

import numpy as np
from scipy.spatial.distance import cdist
from multiprocessing import Pool
from contextlib import closing
from functools import partial
import psutil

def process_chunk(chunk, data, k, metric):
    d = cdist(chunk, data, metric=metric).astype('float32')
    p = np.argpartition(d, k).astype('int32')[:, :k]
    rows = np.arange(chunk.shape[0])[:, None]
    d = d[rows, p]
    i = np.argsort(d)
    return d[rows, i], p[rows, i]

def determine_n_chunks(n, k):
    """Assuming 32 bit representations for distances and indices"""

    # available memory
    available = psutil.virtual_memory().available

    # memory needed to store final knn data (d, idx)
    final = 2 * (n * k * 32) / 8

    # total memory usable for subprocesses
    usable = available - final

    # usable per subprocess
    usable_per_subprocess = usable / psutil.cpu_count()

    # chunk size - number of n-dimensional distance arrays that can be held in memory by each subprocess simultaneously
    chunk_size = usable_per_subprocess // (n * 32)

    return int(n // chunk_size)

def knnsearch(data, k, metric):
    """k-nearest neighbor search via parallelized brute force

    Parameters
    ----------
    data : ndarray
        n observations in d dimensions
    k : int
        number of neighbors (including self)
    metric : str
        see cdist documentation http://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.distance.cdist.html

    Returns
    -------
    d : ndarray
        distances to k nearest neighbors
    idx : ndarray
        indices of k nearest neighbors

    Notes
    -----
    This implementation uses np.array_split to pass the data to subprocesses. This uses views and does not copy the data
    in the subprocesses
    """

    f = partial(process_chunk, **{'data': data, 'k': k, 'metric': metric})

    n_chunks = determine_n_chunks(len(data), k)

    with closing(Pool()) as pool:
        result = pool.map(f, np.array_split(data, n_chunks))

    d, idx = zip(*result)

    return np.vstack(d), np.vstack(idx)
<commit_msg>Add edge case to bruteforce_nn: when data is not worth breaking up, process in single chunk.<commit_after>"""
Compute k-nearest neighbors using brute force search in parallel
via scipy.spatial.distance.cdist and multiprocessing.Pool

psutil is used to evaluate available memory and minimize the number
of parallel jobs for the available resources
"""

import numpy as np
from scipy.spatial.distance import cdist
from multiprocessing import Pool
from contextlib import closing
from functools import partial
import psutil

def process_chunk(chunk, data, k, metric):
    d = cdist(chunk, data, metric=metric).astype('float32')
    p = np.argpartition(d, k).astype('int32')[:, :k]
    rows = np.arange(chunk.shape[0])[:, None]
    d = d[rows, p]
    i = np.argsort(d)
    return d[rows, i], p[rows, i]

def determine_n_chunks(n, k):
    """Assuming 32 bit representations for distances and indices"""

    # available memory
    available = psutil.virtual_memory().available

    # memory needed to store final knn data (d, idx)
    final = 2 * (n * k * 32) / 8

    # total memory usable for subprocesses
    usable = available - final

    # usable per subprocess
    usable_per_subprocess = usable / psutil.cpu_count()

    # chunk size - number of n-dimensional distance arrays that can be held in memory by each subprocess simultaneously
    chunk_size = usable_per_subprocess // (n * 32)

    return int(n // chunk_size)

def knnsearch(data, k, metric):
    """k-nearest neighbor search via parallelized brute force

    Parameters
    ----------
    data : ndarray
        n observations in d dimensions
    k : int
        number of neighbors (including self)
    metric : str
        see cdist documentation http://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.distance.cdist.html

    Returns
    -------
    d : ndarray
        distances to k nearest neighbors
    idx : ndarray
        indices of k nearest neighbors

    Notes
    -----
    This implementation uses np.array_split to pass the data to subprocesses. This uses views and does not copy the data
    in the subprocesses
    """

    f = partial(process_chunk, **{'data': data, 'k': k, 'metric': metric})

    n_chunks = determine_n_chunks(len(data), k)

    if n_chunks > 2:

        with closing(Pool()) as pool:
            result = pool.map(f, np.array_split(data, n_chunks))

        d, idx = zip(*result)

        d, idx = np.vstack(d), np.vstack(idx)

    else:

        d, idx = process_chunk(data, data, k, metric)

    return d, idx
<|endoftext|>


<commit_before>"""A dialog window to show some informations on the current image.
"""

from PySide import QtCore, QtGui

class ImageInfoDialog(QtGui.QDialog):

    def __init__(self):
        super(ImageInfoDialog, self).__init__()

        infoLayout = QtGui.QGridLayout()
        filenameLabel = QtGui.QLabel("File name:")
        self.filename = QtGui.QLabel()
        self.filename.setTextFormat(QtCore.Qt.PlainText)
        infoLayout.addWidget(filenameLabel, 0, 0)
        infoLayout.addWidget(self.filename, 0, 1)
        createDateLabel = QtGui.QLabel("Create date:")
        self.createDate = QtGui.QLabel()
        self.createDate.setTextFormat(QtCore.Qt.PlainText)
        infoLayout.addWidget(createDateLabel, 1, 0)
        infoLayout.addWidget(self.createDate, 1, 1)
        orientationLabel = QtGui.QLabel("Orientation:")
        self.orientation = QtGui.QLabel()
        self.orientation.setTextFormat(QtCore.Qt.PlainText)
        infoLayout.addWidget(orientationLabel, 2, 0)
        infoLayout.addWidget(self.orientation, 2, 1)
        gpsPositionLabel = QtGui.QLabel("GPS position:")
        self.gpsPosition = QtGui.QLabel()
        self.gpsPosition.setTextFormat(QtCore.Qt.RichText)
        self.gpsPosition.setOpenExternalLinks(True)
        infoLayout.addWidget(gpsPositionLabel, 3, 0)
        infoLayout.addWidget(self.gpsPosition, 3, 1)

        buttonBox = QtGui.QDialogButtonBox(QtGui.QDialogButtonBox.Ok)
        buttonBox.accepted.connect(self.accept)

        mainLayout = QtGui.QVBoxLayout()
        mainLayout.addLayout(infoLayout)
        mainLayout.addWidget(buttonBox, alignment=QtCore.Qt.AlignHCenter)
        self.setLayout(mainLayout)
        self.setWindowTitle("Image Info")

    def setinfo(self, item):
        self.filename.setText(item.filename)
        if item.createDate:
            self.createDate.setText(item.createDate.strftime("%a, %x %X"))
        else:
            self.createDate.setText(None)
        if item.orientation:
            self.orientation.setText(item.orientation)
        else:
            self.orientation.setText(None)
        pos = item.gpsPosition
        if pos:
            link = "%s" % (pos.as_osmurl(), unicode(pos))
            self.gpsPosition.setText(link)
        else:
            self.gpsPosition.setText(None)
<commit_msg>Add information from the exif data to the info window.<commit_after>"""A dialog window to show some informations on the current image.
"""

from PySide import QtCore, QtGui
from photo.exif import Exif

class ImageInfoDialog(QtGui.QDialog):

    def __init__(self):
        super(ImageInfoDialog, self).__init__()

        infoLayout = QtGui.QGridLayout()
        cameraModelLabel = QtGui.QLabel("Camera model:")
        self.cameraModel = QtGui.QLabel()
        self.cameraModel.setTextFormat(QtCore.Qt.PlainText)
        infoLayout.addWidget(cameraModelLabel, 0, 0)
        infoLayout.addWidget(self.cameraModel, 0, 1)
        filenameLabel = QtGui.QLabel("File name:")
        self.filename = QtGui.QLabel()
        self.filename.setTextFormat(QtCore.Qt.PlainText)
        infoLayout.addWidget(filenameLabel, 1, 0)
        infoLayout.addWidget(self.filename, 1, 1)
        createDateLabel = QtGui.QLabel("Create date:")
        self.createDate = QtGui.QLabel()
        self.createDate.setTextFormat(QtCore.Qt.PlainText)
        infoLayout.addWidget(createDateLabel, 2, 0)
        infoLayout.addWidget(self.createDate, 2, 1)
        orientationLabel = QtGui.QLabel("Orientation:")
        self.orientation = QtGui.QLabel()
        self.orientation.setTextFormat(QtCore.Qt.PlainText)
        infoLayout.addWidget(orientationLabel, 3, 0)
        infoLayout.addWidget(self.orientation, 3, 1)
        gpsPositionLabel = QtGui.QLabel("GPS position:")
        self.gpsPosition = QtGui.QLabel()
        self.gpsPosition.setTextFormat(QtCore.Qt.RichText)
        self.gpsPosition.setOpenExternalLinks(True)
        infoLayout.addWidget(gpsPositionLabel, 4, 0)
        infoLayout.addWidget(self.gpsPosition, 4, 1)
        exposureTimeLabel = QtGui.QLabel("Exposure time:")
        self.exposureTime = QtGui.QLabel()
        self.exposureTime.setTextFormat(QtCore.Qt.PlainText)
        infoLayout.addWidget(exposureTimeLabel, 5, 0)
        infoLayout.addWidget(self.exposureTime, 5, 1)
        apertureLabel = QtGui.QLabel("F-number:")
        self.aperture = QtGui.QLabel()
        self.aperture.setTextFormat(QtCore.Qt.PlainText)
        infoLayout.addWidget(apertureLabel, 6, 0)
        infoLayout.addWidget(self.aperture, 6, 1)
        isoLabel = QtGui.QLabel("ISO speed rating:")
        self.iso = QtGui.QLabel()
        self.iso.setTextFormat(QtCore.Qt.PlainText)
        infoLayout.addWidget(isoLabel, 7, 0)
        infoLayout.addWidget(self.iso, 7, 1)
        focalLengthLabel = QtGui.QLabel("Focal length:")
        self.focalLength = QtGui.QLabel()
        self.focalLength.setTextFormat(QtCore.Qt.PlainText)
        infoLayout.addWidget(focalLengthLabel, 8, 0)
        infoLayout.addWidget(self.focalLength, 8, 1)

        buttonBox = QtGui.QDialogButtonBox(QtGui.QDialogButtonBox.Ok)
        buttonBox.accepted.connect(self.accept)

        mainLayout = QtGui.QVBoxLayout()
        mainLayout.addLayout(infoLayout)
        mainLayout.addWidget(buttonBox, alignment=QtCore.Qt.AlignHCenter)
        self.setLayout(mainLayout)
        self.setWindowTitle("Image Info")

    def setinfo(self, item):
        exifdata = Exif(item.filename)
        self.cameraModel.setText(str(exifdata.cameraModel))
        self.filename.setText(item.filename)
        if item.createDate:
            self.createDate.setText(item.createDate.strftime("%a, %x %X"))
        else:
            self.createDate.setText(None)
        if item.orientation:
            self.orientation.setText(item.orientation)
        else:
            self.orientation.setText(None)
        pos = item.gpsPosition
        if pos:
            link = "%s" % (pos.as_osmurl(), unicode(pos))
            self.gpsPosition.setText(link)
        else:
            self.gpsPosition.setText(None)
        self.exposureTime.setText(str(exifdata.exposureTime))
        self.aperture.setText(str(exifdata.aperture))
        self.iso.setText(str(exifdata.iso))
        self.focalLength.setText(str(exifdata.focalLength))
<|endoftext|>


<commit_before><commit_msg>TEST: updated flag<commit_after><|endoftext|>