<commit_before>import torch
from torch.autograd import Variable
from torch.optim import Adam, lr_scheduler
import copy
import numpy as np
import logging
import pickle
import datetime
import time
import sys; print('here!!!!!', file=sys.stderr)
import os
import signal
import argparse
import shutil
import gc

from utils import ExperimentHandler

from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_curve
from sklearn.metrics import roc_auc_score
from sklearn.preprocessing import RobustScaler

from architectures.preprocessing import rewrite_content
from architectures.preprocessing import permute_by_pt
from architectures.preprocessing import extract
from architectures.preprocessing import wrap
from architectures.preprocessing import unwrap
from architectures.preprocessing import wrap_X
from architectures.preprocessing import unwrap_X

from constants import *

from losses import log_loss

from architectures import PredictFromParticleEmbedding

from analysis.rocs import inv_fpr_at_tpr_equals_half
from analysis.reports import report_score

from loggers import StatsLogger

from loading import load_data
from loading import load_tf
from loading import crop

''' ARGUMENTS '''
'''----------------------------------------------------------------------- '''
parser = argparse.ArgumentParser(description='Jets')

# data args
parser.add_argument("-f", "--filename", type=str, default='antikt-kt')
parser.add_argument("--data_dir", type=str, default=DATA_DIR)
parser.add_argument("-n", "--n_train", type=int, default=-1)
parser.add_argument("--n_valid", type=int, default=27000)
parser.add_argument("--dont_add_cropped", action='store_true', default=False)

# general model args
parser.add_argument("-m", "--model_type", help="index of the model you want to train - look in the code for the model list", type=int, default=0)
parser.add_argument("--features", type=int, default=7)
parser.add_argument("--hidden", type=int, default=40)

# logging args
parser.add_argument("-s", "--silent", action='store_true', default=False)
parser.add_argument("-v", "--verbose", action='store_true', default=False)
parser.add_argument("--eval_every", type=int, default=25)

# loading previous models args
parser.add_argument("-l", "--load", help="model directory from which we load a state_dict", type=str, default=None)
parser.add_argument("-r", "--restart", help="restart a loaded model from where it left off", action='store_true', default=False)

# training args
parser.add_argument("-e", "--epochs", type=int, default=25)
parser.add_argument("-b", "--batch_size", type=int, default=64)
parser.add_argument("-a", "--step_size", type=float, default=0.001)
parser.add_argument("-d", "--decay", type=float, default=.912)

# computing args
parser.add_argument("--seed", help="Random seed used in torch and numpy", type=int, default=1)
parser.add_argument("-g", "--gpu", type=str, default="")

# MPNN
parser.add_argument("--not_leaves", action='store_true')
parser.add_argument("-i", "--iters", type=int, default=1)

# email
parser.add_argument("--sender", type=str, default="results74207281@gmail.com")
parser.add_argument("--password", type=str, default="deeplearning")

# debugging
parser.add_argument("--debug", help="sets everything small for fast model debugging. use in combination with ipdb", action='store_true', default=False)

args = parser.parse_args()

if args.debug:
    args.hidden = 1
    args.batch_size = 9
    args.verbose = True
    args.epochs = 3
    args.n_train = 1000
    args.seed = 1
    args.eval_every = 10

os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu

if args.n_train <= 5 * args.n_valid and args.n_train > 0:
    args.n_valid = args.n_train // 5
args.recipient = RECIPIENT
args.leaves = not args.not_leaves
def train(args):

    _, Transform, model_type = TRANSFORMS[args.model_type]
    eh = ExperimentHandler(args, os.path.join(MODELS_DIR,model_type))
    signal_handler = eh.signal_handler

    ''' DATA '''
    '''----------------------------------------------------------------------- '''
    logging.warning("Loading data...")
    if args.debug and False:
        for k in range(15):
            time.sleep(1)
            logging.info('before load tf: {} seconds'.format(k+1))

    tf = load_tf(args.data_dir, "{}-train.pickle".format(args.filename))

    if args.debug and False:
        for k in range(15):
            time.sleep(1)
            logging.info('before load_data: {} seconds'.format(k+1))

    X, y = load_data(args.data_dir, "{}-train.pickle".format(args.filename))

    logging.warning("Memory usage = {}".format(0))
    if args.debug and False:
        for k in range(15):
            time.sleep(1)
            logging.info('before tf transform: {} seconds'.format(k+1))

    #for ij, jet in enumerate(X):
    #    jet["content"] = tf.transform(jet["content"])

    logging.warning("After transform: memory usage = {}".format(eh.usage()))

    if args.n_train > 0:
        indices = torch.randperm(len(X)).numpy()[:args.n_train]
        X = [X[i] for i in indices]
        y = y[indices]

    logging.warning("Splitting into train and validation...")

    X_train, X_valid_uncropped, y_train, y_valid_uncropped = train_test_split(X, y, test_size=args.n_valid, random_state=0)
    logging.warning("\traw train size = %d" % len(X_train))
    logging.warning("\traw valid size = %d" % len(X_valid_uncropped))

    X_valid, y_valid, cropped_indices, w_valid = crop(X_valid_uncropped, y_valid_uncropped, return_cropped_indices=True)

    # add cropped indices to training data
    if not args.dont_add_cropped:
        X_train.extend([x for i, x in enumerate(X_valid_uncropped) if i in cropped_indices])
        y_train = [y for y in y_train]
        y_train.extend([y for i, y in enumerate(y_valid_uncropped) if i in cropped_indices])
        y_train = np.array(y_train)
    logging.warning("\tfinal train size = %d" % len(X_train))
    logging.warning("\tfinal valid size = %d" % len(X_valid))

    ''' MODEL '''
    '''----------------------------------------------------------------------- '''
    # Initialization
    logging.info("Initializing model...")
    Predict = PredictFromParticleEmbedding
    if args.load is None:
        model_kwargs = {
            'features': args.features,
            'hidden': args.hidden,
            'iters': args.iters,
            'leaves': args.leaves,
        }
        model = Predict(Transform, **model_kwargs)
        settings = {
            "transform": Transform,
            "predict": Predict,
            "model_kwargs": model_kwargs,
            "step_size": args.step_size,
            "args": args,
            }
    else:
        with open(os.path.join(args.load, 'settings.pickle'), "rb") as f:
            settings = pickle.load(f, encoding='latin-1')
            Transform = settings["transform"]
            Predict = settings["predict"]
            model_kwargs = settings["model_kwargs"]

        with open(os.path.join(args.load, 'model_state_dict.pt'), 'rb') as f:
            state_dict = torch.load(f)
            model = PredictFromParticleEmbedding(Transform, **model_kwargs)
            model.load_state_dict(state_dict)

        if args.restart:
            args.step_size = settings["step_size"]

    logging.warning(model)
    out_str = 'Number of parameters: {}'.format(sum(np.prod(p.data.numpy().shape) for p in model.parameters()))
    logging.warning(out_str)

    if torch.cuda.is_available():
        logging.warning("Moving model to GPU")
        if args.debug and False:
            for k in range(30):
                time.sleep(1)
                logging.info('{} seconds'.format(k+1))
        model.cuda()
        logging.warning("Moved model to GPU")

    signal_handler.set_model(model)

    ''' OPTIMIZER AND LOSS '''
    '''----------------------------------------------------------------------- '''
    logging.info("Building optimizer...")
    optimizer = Adam(model.parameters(), lr=args.step_size)
    scheduler = lr_scheduler.ExponentialLR(optimizer, gamma=args.decay)
    #scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.5)

    n_batches = int(len(X_train) // args.batch_size)
    best_score = [-np.inf]  # yuck, but works
    best_model_state_dict = copy.deepcopy(model.state_dict())

    def loss(y_pred, y):
        l = log_loss(y, y_pred.squeeze(1)).mean()
        return l

        ''' VALIDATION '''
    '''----------------------------------------------------------------------- '''
    def callback(epoch, iteration, model):

        if iteration % n_batches == 0:
            t0 = time.time()
            model.eval()

            offset = 0; train_loss = []; valid_loss = []
            yy, yy_pred = [], []
            for i in range(len(X_valid) // args.batch_size):
                idx = slice(offset, offset+args.batch_size)
                Xt, yt = X_train[idx], y_train[idx]
                X_var = wrap_X(Xt); y_var = wrap(yt)
                tl = unwrap(loss(model(X_var), y_var)); train_loss.append(tl)
                X = unwrap_X(X_var); y = unwrap(y_var)

                Xv, yv = X_valid[idx], y_valid[idx]
                X_var = wrap_X(Xv); y_var = wrap(yv)
                y_pred = model(X_var)
                vl = unwrap(loss(y_pred, y_var)); valid_loss.append(vl)
                Xv = unwrap_X(X_var); yv = unwrap(y_var); y_pred = unwrap(y_pred)
                yy.append(yv); yy_pred.append(y_pred)

                offset+=args.batch_size

            train_loss = np.mean(np.array(train_loss))
            valid_loss = np.mean(np.array(valid_loss))
            yy = np.concatenate(yy, 0)
            yy_pred = np.concatenate(yy_pred, 0)

            t1=time.time()
            logging.info("Modeling validation data took {}s".format(t1-t0))
            logdict = dict(
                epoch=epoch,
                iteration=iteration,
                yy=yy,
                yy_pred=yy_pred,
                w_valid=w_valid,
                train_loss=train_loss,
                valid_loss=valid_loss,
                settings=settings,
                model=model
            )
            eh.log(**logdict)

            scheduler.step(valid_loss)
            model.train()

    ''' TRAINING '''
    '''----------------------------------------------------------------------- '''
    eh.save(model, settings)
    logging.warning("Training...")
    iteration=1
    for i in range(args.epochs):
        logging.info("epoch = %d" % i)
        logging.info("step_size = %.8f" % settings['step_size'])
        t0 = time.time()

        for _ in range(n_batches):
            iteration += 1
            print(1)
            model.train()
            print(2)
            optimizer.zero_grad()
            print(3)
            start = torch.round(torch.rand(1) * (len(X_train) - args.batch_size)).numpy()[0].astype(np.int32)
            print(4)
            idx = slice(start, start+args.batch_size)
            print(5)
            X, y = X_train[idx], y_train[idx]
            print(6)
            X_var = wrap_X(X); y_var = wrap(y)
            print(7)
            l = loss(model(X_var), y_var)
            print(8)
            l.backward()
            print(9)
            optimizer.step()
            print(10)
            X = unwrap_X(X_var); y = unwrap(y_var)
            callback(i, iteration, model)
        t1 = time.time()
        logging.info("Epoch took {} seconds".format(t1-t0))

        scheduler.step()
        settings['step_size'] = args.step_size * (args.decay) ** (i + 1)

    eh.finished()

if __name__ == "__main__":
    train(args)
<commit_msg>debug<commit_after>import torch
from torch.autograd import Variable
from torch.optim import Adam, lr_scheduler
import copy
import numpy as np
import logging
import pickle
import datetime
import time
import sys; print('here!!!!!', file=sys.stderr)
import os
import signal
import argparse
import shutil
import gc

from utils import ExperimentHandler

from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_curve
from sklearn.metrics import roc_auc_score
from sklearn.preprocessing import RobustScaler

from architectures.preprocessing import rewrite_content
from architectures.preprocessing import permute_by_pt
from architectures.preprocessing import extract
from architectures.preprocessing import wrap
from architectures.preprocessing import unwrap
from architectures.preprocessing import wrap_X
from architectures.preprocessing import unwrap_X

from constants import *

from losses import log_loss

from architectures import PredictFromParticleEmbedding

from analysis.rocs import inv_fpr_at_tpr_equals_half
from analysis.reports import report_score

from loggers import StatsLogger

from loading import load_data
from loading import load_tf
from loading import crop

''' ARGUMENTS '''
'''----------------------------------------------------------------------- '''
parser = argparse.ArgumentParser(description='Jets')

# data args
parser.add_argument("-f", "--filename", type=str, default='antikt-kt')
parser.add_argument("--data_dir", type=str, default=DATA_DIR)
parser.add_argument("-n", "--n_train", type=int, default=-1)
parser.add_argument("--n_valid", type=int, default=27000)
parser.add_argument("--dont_add_cropped", action='store_true', default=False)

# general model args
parser.add_argument("-m", "--model_type", help="index of the model you want to train - look in the code for the model list", type=int, default=0)
parser.add_argument("--features", type=int, default=7)
parser.add_argument("--hidden", type=int, default=40)

# logging args
parser.add_argument("-s", "--silent", action='store_true', default=False)
parser.add_argument("-v", "--verbose", action='store_true', default=False)
parser.add_argument("--eval_every", type=int, default=25)

# loading previous models args
parser.add_argument("-l", "--load", help="model directory from which we load a state_dict", type=str, default=None)
parser.add_argument("-r", "--restart", help="restart a loaded model from where it left off", action='store_true', default=False)

# training args
parser.add_argument("-e", "--epochs", type=int, default=25)
parser.add_argument("-b", "--batch_size", type=int, default=64)
parser.add_argument("-a", "--step_size", type=float, default=0.001)
parser.add_argument("-d", "--decay", type=float, default=.912)

# computing args
parser.add_argument("--seed", help="Random seed used in torch and numpy", type=int, default=1)
parser.add_argument("-g", "--gpu", type=str, default="")

# MPNN
parser.add_argument("--not_leaves", action='store_true')
parser.add_argument("-i", "--iters", type=int, default=1)

# email
parser.add_argument("--sender", type=str, default="results74207281@gmail.com")
parser.add_argument("--password", type=str, default="deeplearning")

# debugging
parser.add_argument("--debug", help="sets everything small for fast model debugging. use in combination with ipdb", action='store_true', default=False)

args = parser.parse_args()

if args.debug:
    args.hidden = 1
    args.batch_size = 9
    args.verbose = True
    args.epochs = 3
    args.n_train = 1000
    args.seed = 1
    args.eval_every = 10

os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu

if args.n_train <= 5 * args.n_valid and args.n_train > 0:
    args.n_valid = args.n_train // 5
args.recipient = RECIPIENT
args.leaves = not args.not_leaves
def train(args):

    _, Transform, model_type = TRANSFORMS[args.model_type]
    eh = ExperimentHandler(args, os.path.join(MODELS_DIR,model_type))
    signal_handler = eh.signal_handler

    ''' DATA '''
    '''----------------------------------------------------------------------- '''
    logging.warning("Loading data...")
    if args.debug and False:
        for k in range(15):
            time.sleep(1)
            logging.info('before load tf: {} seconds'.format(k+1))

    tf = load_tf(args.data_dir, "{}-train.pickle".format(args.filename))

    if args.debug and False:
        for k in range(15):
            time.sleep(1)
            logging.info('before load_data: {} seconds'.format(k+1))

    X, y = load_data(args.data_dir, "{}-train.pickle".format(args.filename))

    logging.warning("Memory usage = {}".format(0))
    if args.debug and False:
        for k in range(15):
            time.sleep(1)
            logging.info('before tf transform: {} seconds'.format(k+1))

    #for ij, jet in enumerate(X):
    #    jet["content"] = tf.transform(jet["content"])

    logging.warning("After transform: memory usage = {}".format(eh.usage()))

    if args.n_train > 0:
        indices = torch.randperm(len(X)).numpy()[:args.n_train]
        X = [X[i] for i in indices]
        y = y[indices]

    logging.warning("Splitting into train and validation...")

    X_train, X_valid_uncropped, y_train, y_valid_uncropped = train_test_split(X, y, test_size=args.n_valid, random_state=0)
    logging.warning("\traw train size = %d" % len(X_train))
    logging.warning("\traw valid size = %d" % len(X_valid_uncropped))

    X_valid, y_valid, cropped_indices, w_valid = crop(X_valid_uncropped, y_valid_uncropped, return_cropped_indices=True)

    # add cropped indices to training data
    if not args.dont_add_cropped:
        X_train.extend([x for i, x in enumerate(X_valid_uncropped) if i in cropped_indices])
        y_train = [y for y in y_train]
        y_train.extend([y for i, y in enumerate(y_valid_uncropped) if i in cropped_indices])
        y_train = np.array(y_train)
    logging.warning("\tfinal train size = %d" % len(X_train))
    logging.warning("\tfinal valid size = %d" % len(X_valid))

    ''' MODEL '''
    '''----------------------------------------------------------------------- '''
    # Initialization
    logging.info("Initializing model...")
    Predict = PredictFromParticleEmbedding
    if args.load is None:
        model_kwargs = {
            'features': args.features,
            'hidden': args.hidden,
            'iters': args.iters,
            'leaves': args.leaves,
        }
        model = Predict(Transform, **model_kwargs)
        settings = {
            "transform": Transform,
            "predict": Predict,
            "model_kwargs": model_kwargs,
            "step_size": args.step_size,
            "args": args,
            }
    else:
        with open(os.path.join(args.load, 'settings.pickle'), "rb") as f:
            settings = pickle.load(f, encoding='latin-1')
            Transform = settings["transform"]
            Predict = settings["predict"]
            model_kwargs = settings["model_kwargs"]

        with open(os.path.join(args.load, 'model_state_dict.pt'), 'rb') as f:
            state_dict = torch.load(f)
            model = PredictFromParticleEmbedding(Transform, **model_kwargs)
            model.load_state_dict(state_dict)

        if args.restart:
            args.step_size = settings["step_size"]

    logging.warning(model)
    out_str = 'Number of parameters: {}'.format(sum(np.prod(p.data.numpy().shape) for p in model.parameters()))
    logging.warning(out_str)

    if torch.cuda.is_available():
        logging.warning("Moving model to GPU")
        if args.debug and False:
            for k in range(30):
                time.sleep(1)
                logging.info('{} seconds'.format(k+1))
        model.cuda()
        logging.warning("Moved model to GPU")

    signal_handler.set_model(model)

    ''' OPTIMIZER AND LOSS '''
    '''----------------------------------------------------------------------- '''
    logging.info("Building optimizer...")
    optimizer = Adam(model.parameters(), lr=args.step_size)
    scheduler = lr_scheduler.ExponentialLR(optimizer, gamma=args.decay)
    #scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.5)

    n_batches = int(len(X_train) // args.batch_size)
    best_score = [-np.inf]  # yuck, but works
    best_model_state_dict = copy.deepcopy(model.state_dict())

    def loss(y_pred, y):
        l = log_loss(y, y_pred.squeeze(1)).mean()
        return l

        ''' VALIDATION '''
    '''----------------------------------------------------------------------- '''
    def callback(epoch, iteration, model):

        if iteration % n_batches == 0:
            t0 = time.time()
            model.eval()

            offset = 0; train_loss = []; valid_loss = []
            yy, yy_pred = [], []
            for i in range(len(X_valid) // args.batch_size):
                idx = slice(offset, offset+args.batch_size)
                Xt, yt = X_train[idx], y_train[idx]
                X_var = wrap_X(Xt); y_var = wrap(yt)
                tl = unwrap(loss(model(X_var), y_var)); train_loss.append(tl)
                X = unwrap_X(X_var); y = unwrap(y_var)

                Xv, yv = X_valid[idx], y_valid[idx]
                X_var = wrap_X(Xv); y_var = wrap(yv)
                y_pred = model(X_var)
                vl = unwrap(loss(y_pred, y_var)); valid_loss.append(vl)
                Xv = unwrap_X(X_var); yv = unwrap(y_var); y_pred = unwrap(y_pred)
                yy.append(yv); yy_pred.append(y_pred)

                offset+=args.batch_size

            train_loss = np.mean(np.array(train_loss))
            valid_loss = np.mean(np.array(valid_loss))
            yy = np.concatenate(yy, 0)
            yy_pred = np.concatenate(yy_pred, 0)

            t1=time.time()
            logging.info("Modeling validation data took {}s".format(t1-t0))
            logdict = dict(
                epoch=epoch,
                iteration=iteration,
                yy=yy,
                yy_pred=yy_pred,
                w_valid=w_valid,
                train_loss=train_loss,
                valid_loss=valid_loss,
                settings=settings,
                model=model
            )
            eh.log(**logdict)

            scheduler.step(valid_loss)
            model.train()

    ''' TRAINING '''
    '''----------------------------------------------------------------------- '''
    eh.save(model, settings)
    logging.warning("Training...")
    iteration=1
    for i in range(args.epochs):
        logging.info("epoch = %d" % i)
        logging.info("step_size = %.8f" % settings['step_size'])
        t0 = time.time()
        print(n_batches)
        for _ in range(n_batches):
            iteration += 1
            print(1)
            model.train()
            print(2)
            optimizer.zero_grad()
            print(3)
            start = torch.round(torch.rand(1) * (len(X_train) - args.batch_size)).numpy()[0].astype(np.int32)
            print(4)
            idx = slice(start, start+args.batch_size)
            print(5)
            X, y = X_train[idx], y_train[idx]
            print(6)
            X_var = wrap_X(X); y_var = wrap(y)
            print(7)
            l = loss(model(X_var), y_var)
            print(8)
            l.backward()
            print(9)
            optimizer.step()
            print(10)
            X = unwrap_X(X_var); y = unwrap(y_var)
            callback(i, iteration, model)
        t1 = time.time()
        logging.info("Epoch took {} seconds".format(t1-t0))

        scheduler.step()
        settings['step_size'] = args.step_size * (args.decay) ** (i + 1)

    eh.finished()

if __name__ == "__main__":
    train(args)
<|endoftext|>


<commit_before>import torch.backends.cudnn as cudnn
cudnn.benchmark=False

import numpy as np
import time
import os
from models import dist_model as dm
from data import data_loader as dl
import argparse
from util.visualizer import Visualizer
from IPython import embed

parser = argparse.ArgumentParser()
parser.add_argument('--datasets', type=str, nargs='+', default=['train/traditional','train/cnn','train/mix'], help='datasets to train on: [train/traditional],[train/cnn],[train/mix],[val/traditional],[val/cnn],[val/color],[val/deblur],[val/frameinterp],[val/superres]')
parser.add_argument('--model', type=str, default='net-lin', help='distance model type [net-lin] for linearly calibrated net, [net] for off-the-shelf network, [l2] for euclidean distance, [ssim] for Structured Similarity Image Metric')
parser.add_argument('--net', type=str, default='alex', help='[squeeze], [alex], or [vgg] for network architectures')
parser.add_argument('--batch_size', type=int, default=50, help='batch size to test image patches in')
parser.add_argument('--use_gpu', action='store_true', help='turn on flag to use GPU')
parser.add_argument('--nepoch', type=int, default=5, help='# epochs at base learning rate')
parser.add_argument('--nepoch_decay', type=int, default=5, help='# additional epochs at linearly learning rate')
parser.add_argument('--display_freq', type=int, default=5000, help='frequency (in instances) of showing training results on screen')
parser.add_argument('--print_freq', type=int, default=5000, help='frequency (in instances) of showing training results on console')
parser.add_argument('--save_latest_freq', type=int, default=10000, help='frequency (in instances) of saving the latest results')
parser.add_argument('--save_epoch_freq', type=int, default=1, help='frequency of saving checkpoints at the end of epochs')
parser.add_argument('--display_id', type=int, default=0, help='window id of the visdom display, [0] for no displaying')
parser.add_argument('--display_winsize', type=int, default=256,  help='display window size')
parser.add_argument('--display_port', type=int, default=8001,  help='visdom display port')
parser.add_argument('--use_html', action='store_true', help='save off html pages')
parser.add_argument('--checkpoints_dir', type=str, default='checkpoints', help='checkpoints directory')
parser.add_argument('--name', type=str, default='tmp', help='directory name for training')

parser.add_argument('--from_scratch', action='store_true', help='model was initialized from scratch')
parser.add_argument('--train_trunk', action='store_true', help='model trunk was trained/tuned')

opt = parser.parse_args()
opt.train_plot = True
opt.save_dir = os.path.join(opt.checkpoints_dir,opt.name)
if(not os.path.exists(opt.save_dir)):
    os.mkdir(opt.save_dir)

# initialize model
model = dm.DistModel()
model.initialize(model=opt.model, net=opt.net, use_gpu=opt.use_gpu, is_train=True,
    pnet_rand=opt.from_scratch, pnet_tune=opt.train_trunk)

# load data from all training sets
data_loader = dl.CreateDataLoader(opt.datasets,dataset_mode='2afc', batch_size=opt.batch_size, serial_batches=False)
dataset = data_loader.load_data()
dataset_size = len(data_loader)
D = len(dataset)
print('Loading %i instances from'%dataset_size,opt.datasets)
visualizer = Visualizer(opt)

total_steps = 0
fid = open(os.path.join(opt.checkpoints_dir,opt.name,'train_log.txt'),'w+')
for epoch in range(1, opt.nepoch + opt.nepoch_decay + 1):
    epoch_start_time = time.time()
    for i, data in enumerate(dataset):
        iter_start_time = time.time()
        total_steps += opt.batch_size
        epoch_iter = total_steps - dataset_size * (epoch - 1)

        model.set_input(data)
        model.optimize_parameters()

        if total_steps % opt.display_freq == 0:
            visualizer.display_current_results(model.get_current_visuals(), epoch)

        if total_steps % opt.print_freq == 0:
            errors = model.get_current_errors()
            t = (time.time()-iter_start_time)/opt.batch_size
            t2o = (time.time()-epoch_start_time)/3600.
            t2 = t2o*D/(i+.0001)
            visualizer.print_current_errors(epoch, epoch_iter, errors, t, t2=t2, t2o=t2o, fid=fid)

            for key in errors.keys():
                visualizer.plot_current_errors_save(epoch, float(epoch_iter)/dataset_size, opt, errors, keys=[key,], name=key, to_plot=opt.train_plot)

            if opt.display_id > 0:
                visualizer.plot_current_errors(epoch, float(epoch_iter)/dataset_size, opt, errors)

        if total_steps % opt.save_latest_freq == 0:
            print('saving the latest model (epoch %d, total_steps %d)' %
                  (epoch, total_steps))
            model.save(opt.save_dir, 'latest')

    if epoch % opt.save_epoch_freq == 0:
        print('saving the model at the end of epoch %d, iters %d' %
              (epoch, total_steps))
        model.save(opt.save_dir, 'latest')
        model.save(opt.save_dir, epoch)

    print('End of epoch %d / %d \t Time Taken: %d sec' %
          (epoch, opt.nepoch + opt.nepoch_decay, time.time() - epoch_start_time))

    if epoch > opt.nepoch:
        model.update_learning_rate(opt.nepoch_decay)

# model.save_done(True)
fid.close()
<commit_msg>plot saving flag<commit_after>import torch.backends.cudnn as cudnn
cudnn.benchmark=False

import numpy as np
import time
import os
from models import dist_model as dm
from data import data_loader as dl
import argparse
from util.visualizer import Visualizer
from IPython import embed

parser = argparse.ArgumentParser()
parser.add_argument('--datasets', type=str, nargs='+', default=['train/traditional','train/cnn','train/mix'], help='datasets to train on: [train/traditional],[train/cnn],[train/mix],[val/traditional],[val/cnn],[val/color],[val/deblur],[val/frameinterp],[val/superres]')
parser.add_argument('--model', type=str, default='net-lin', help='distance model type [net-lin] for linearly calibrated net, [net] for off-the-shelf network, [l2] for euclidean distance, [ssim] for Structured Similarity Image Metric')
parser.add_argument('--net', type=str, default='alex', help='[squeeze], [alex], or [vgg] for network architectures')
parser.add_argument('--batch_size', type=int, default=50, help='batch size to test image patches in')
parser.add_argument('--use_gpu', action='store_true', help='turn on flag to use GPU')
parser.add_argument('--nepoch', type=int, default=5, help='# epochs at base learning rate')
parser.add_argument('--nepoch_decay', type=int, default=5, help='# additional epochs at linearly learning rate')
parser.add_argument('--display_freq', type=int, default=5000, help='frequency (in instances) of showing training results on screen')
parser.add_argument('--print_freq', type=int, default=5000, help='frequency (in instances) of showing training results on console')
parser.add_argument('--save_latest_freq', type=int, default=10000, help='frequency (in instances) of saving the latest results')
parser.add_argument('--save_epoch_freq', type=int, default=1, help='frequency of saving checkpoints at the end of epochs')
parser.add_argument('--display_id', type=int, default=0, help='window id of the visdom display, [0] for no displaying')
parser.add_argument('--display_winsize', type=int, default=256,  help='display window size')
parser.add_argument('--display_port', type=int, default=8001,  help='visdom display port')
parser.add_argument('--use_html', action='store_true', help='save off html pages')
parser.add_argument('--checkpoints_dir', type=str, default='checkpoints', help='checkpoints directory')
parser.add_argument('--name', type=str, default='tmp', help='directory name for training')

parser.add_argument('--from_scratch', action='store_true', help='model was initialized from scratch')
parser.add_argument('--train_trunk', action='store_true', help='model trunk was trained/tuned')
parser.add_argument('--train_plot', action='store_true', help='plot saving')

opt = parser.parse_args()
opt.save_dir = os.path.join(opt.checkpoints_dir,opt.name)
if(not os.path.exists(opt.save_dir)):
    os.mkdir(opt.save_dir)

# initialize model
model = dm.DistModel()
model.initialize(model=opt.model, net=opt.net, use_gpu=opt.use_gpu, is_train=True,
    pnet_rand=opt.from_scratch, pnet_tune=opt.train_trunk)

# load data from all training sets
data_loader = dl.CreateDataLoader(opt.datasets,dataset_mode='2afc', batch_size=opt.batch_size, serial_batches=False)
dataset = data_loader.load_data()
dataset_size = len(data_loader)
D = len(dataset)
print('Loading %i instances from'%dataset_size,opt.datasets)
visualizer = Visualizer(opt)

total_steps = 0
fid = open(os.path.join(opt.checkpoints_dir,opt.name,'train_log.txt'),'w+')
for epoch in range(1, opt.nepoch + opt.nepoch_decay + 1):
    epoch_start_time = time.time()
    for i, data in enumerate(dataset):
        iter_start_time = time.time()
        total_steps += opt.batch_size
        epoch_iter = total_steps - dataset_size * (epoch - 1)

        model.set_input(data)
        model.optimize_parameters()

        if total_steps % opt.display_freq == 0:
            visualizer.display_current_results(model.get_current_visuals(), epoch)

        if total_steps % opt.print_freq == 0:
            errors = model.get_current_errors()
            t = (time.time()-iter_start_time)/opt.batch_size
            t2o = (time.time()-epoch_start_time)/3600.
            t2 = t2o*D/(i+.0001)
            visualizer.print_current_errors(epoch, epoch_iter, errors, t, t2=t2, t2o=t2o, fid=fid)

            for key in errors.keys():
                visualizer.plot_current_errors_save(epoch, float(epoch_iter)/dataset_size, opt, errors, keys=[key,], name=key, to_plot=opt.train_plot)

            if opt.display_id > 0:
                visualizer.plot_current_errors(epoch, float(epoch_iter)/dataset_size, opt, errors)

        if total_steps % opt.save_latest_freq == 0:
            print('saving the latest model (epoch %d, total_steps %d)' %
                  (epoch, total_steps))
            model.save(opt.save_dir, 'latest')

    if epoch % opt.save_epoch_freq == 0:
        print('saving the model at the end of epoch %d, iters %d' %
              (epoch, total_steps))
        model.save(opt.save_dir, 'latest')
        model.save(opt.save_dir, epoch)

    print('End of epoch %d / %d \t Time Taken: %d sec' %
          (epoch, opt.nepoch + opt.nepoch_decay, time.time() - epoch_start_time))

    if epoch > opt.nepoch:
        model.update_learning_rate(opt.nepoch_decay)

# model.save_done(True)
fid.close()
<|endoftext|>


<commit_before># coding: utf-8
'''
Created on 2014年8月5日

@author: lunatic
'''
import json
import os
import logging
import sys
from settings import COMPILER, OUTPUT, INPUT, TARGET

'''
编译所有更新或受到更新影响的文件
编译编译依赖且不存在.o的文件
更详细的说：
        编译所有存在更新的源文件和受引用更新文件的源文件以及编译依赖不存在.o文件的源文件
        如果文件已经编译并且没有再更新过  什么也不做
        如果文件已经编译但有新的更新 重新编译
        如果文件没有编译过 （即.o文件不存在），不管是否更新 编译

处理步骤：
    1.提取所有原始文件时间戳信息
    2.和当前目录的文件时间戳进行对比找到更新的文件
    3.通过引用关系推断出直接或间接受到影响的文件
    4.合并编译所需的依赖文件和受到影响的文件
    5.生成编译器命令
    6.执行生成的命令
    7.将最新的时间戳信息保存到文件
'''

MAKE_FILE = '.mk'

FILE_PINS = {}
# HEAD_REGEX = r'^#include "(\w+.h)"$'

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
ch.setLevel(logging.DEBUG)
logger.addHandler(ch)
#
# class Config:
#     parser = ConfigParser.ConfigParser()
#     parser.read('build.cfg')
#     @classmethod
#     def get(cls, section, option):
#         return cls.parser.get(section, option)
#     @classmethod
#     def sections(cls):
#         return cls.parser.sections()
#     @classmethod
#     def options(cls, section):
#         return cls.parser.options(section)

class Recorder(object):

    def write(self, data={}):
        with open(MAKE_FILE, "w") as f:
            f.write(json.dumps(data));

    def update(self, data={}):
        _data = self.read()
        _data.update(data)
        self.write(_data)

    def read(self):
        if not os.path.exists(MAKE_FILE):
            return dict()
        with open(MAKE_FILE, "r") as f:
            return json.loads(f.read())

def get_timestamp(fn):
    return os.path.getmtime(fn)

def get_content(fn):
    with open(fn, "r") as f:
        content = f.read()
    return content

class HeadSet(object):
    '''
     头文件特殊处理：
    为每个头文件建立一个set set里面放入所有包含了该头文件的文件 如果其中有头文件包含了该头文件 递归的求包含该头文件的源文件 代替头文件本身返回
    '''
    def __init__(self):
        self.table = dict()

    def add_refs(self, fn, relfns_iter):
        self.table[fn] = set(relfns_iter)

    def adjust_refs(self):
        '''
        调整该结构使得所有的set都不包含头文件
        '''
        self.passed = set()
        for doth in self.table.keys():
            self.passed.clear()
            self.table[doth] = self.__adjust(doth)

    def __adjust(self, doth):
        '''
        获取包含该头文件的所有源文件
        '''
        # check processed return empty list
        if doth in self.passed:  # 循环引用
            return list()
        refs = self.search_refs(doth)
        # maybe sytanx error!
        if not refs:
            return list()
        dotcc = filter(lambda it: it.endswith('.cc'), refs)
        doths = filter(lambda it: it.endswith('.h'), refs)
        # add passed
        self.passed.add(doth)
        if doths:
            for ls in [self.__adjust(doth) for doth in doths]:
                dotcc.extend(ls)
        # return set(dotcc)  # filter distinct
        return set(dotcc)  # 间接引用

    def search_refs(self, fn):
        refs = self.table.get(fn)
        return refs if refs else set()

def get_diffs(origin={}, _dir=INPUT):
    '''
     获取指定目录下所有更新的源文件以及包含更新源文件的源文件
    '''
    logger.debug('Dir: %s', ' '.join(os.listdir(_dir)))

    files = [name for name in os.listdir(_dir) if name.endswith('.h') or name.endswith('.cc')]
    logger.debug('List of files: %s' % ' '.join(files))

    # name_conts = [(name, get_content(name)) for name in files]
    prints = {name: get_timestamp(os.path.join(_dir, name)) for name in files}
    global FILE_PINS
    FILE_PINS.update(prints)
    # 比对原始和最新的时间戳 过滤出所有变化的文件
    diff_pins = filter(lambda (name, pin): origin.get(name, 0.0) < pin, prints.items())
    if not diff_pins:
        logger.warn('No files changed!')
        return []

    diff_files = zip(*diff_pins)[0]
    logger.debug('List of changed files: %s', ' '.join(diff_files))

    compiles = list(filter(lambda name: name.endswith('.cc'), diff_files))

    hs = HeadSet()
    diff_hfiles = filter(lambda name: name.endswith('.h'), diff_files)
    # 找出所有更新的头文件的所有关联文件
    file_conts = [get_content(os.path.join(_dir, name)) for name in files]
    for hf in diff_hfiles:
        include = '#include "%s"' % hf
        rels = filter(lambda cont: cont.find(include) != -1, file_conts)
        if not rels:
            continue
        relfs = zip(*rels)[0]
        # logger.debug('rels', relfs
        hs.add_refs(hf, relfs)
    hs.adjust_refs()
    logger.debug('adjust: %s' % str(hs.table))

    for difls in [hs.search_refs(hf) for hf in diff_hfiles]:
        compiles.extend(difls)

    logger.debug('List of affected files: %s', ''.join(set(compiles)))
    return {}.fromkeys(compiles).keys()

class CommandBuilder(object):
    # g++ -O0 -g3 -Wall -c -fmessage-length=0
    COMPILE_CMD = '%(name)s -O%(opt)s -g%(debug)s %(warn)s -c %(other)s -o %(out)s %(input)s'
    LINK_CMD = '%(name)s -o %(target)s %(inputs)s'
    def __init__(self):
        self.command = {}
        self.command['name'] = COMPILER['name']

        opt = COMPILER.get('optimization')
        deb = COMPILER.get('debugging')
        warn = COMPILER.get('warning')
        other = COMPILER.get('other')

        self.command['opt'] = opt if opt else 0
        self.command['debug'] = deb if deb else 0
        self.command['warn'] = '-Wall' if warn and warn == True else ''
        self.command['other'] = other if other else ''

    def build_commands(self, diffs, target):
        comps = self.__merge(diffs, target)
        out = lambda arg: os.path.join(OUTPUT, arg.split('.')[0] + '.o')
        _in = lambda arg: os.path.join(INPUT, arg)
        tasks = []
        for fn in comps:
            dots = _in(fn)
            if not os.path.exists(dots):
                raise Exception('File Not Found "%s"' % dots)
            doto = out(fn)
            self.command['out'] = doto
            self.command['input'] = dots
            tasks.append(self.COMPILE_CMD % self.command)

        for fn, deps in target.items():
            self.command['inputs'] = ' '.join(map(out, deps))
            self.command['target'] = fn
            tasks.append(self.LINK_CMD % self.command)

        return tasks;

    def __merge(self, diffs, target):
        '''
        合并受更新影响的文件和编译依赖文件
        依赖文件编译要求： 如果依赖文件没有受到更新影响 而且.o文件存在则什么也不做 否则编译依赖文件
        '''
        comps = diffs
        logger.debug('merge diff and depends')
        depends = set(reduce(lambda x, y: x + y, target.values()))
        logger.debug('depends: %s', ' '.join(depends))
        out = lambda arg: os.path.join(OUTPUT, arg.split('.')[0] + '.o')
        logger.debug('comps: %s', ' '.join(comps))
        #comps.extend()
        comps.extend([dep for dep in depends if dep not in diffs and not os.path.exists(out(dep))])
        logger.debug('extend comps: %s', ' '.join(comps))
        return comps

def execute(cmds=[]):
    # doit = lambda task: os.popen(cmd).read()
    doit = lambda x:x
    return '\n'.join(map(doit, cmds))

def say(x):
    print x
    return x

def main(*args, **kwargs):

    recorder = Recorder()
    builder = CommandBuilder()

    origin = recorder.read()
    diffs = get_diffs(origin)

    cmds = builder.build_commands(diffs, TARGET)
    # tasks, upfiles = CommandBuilder().__get_comand(diffs, *TARGET['hello.exe'], target='hello.exe')
    rv = execute(cmds)
    print 'rv:\n', rv, '\n'
    recorder.update(FILE_PINS)

if __name__ == '__main__':
    logger.debug('start')
    main()
    logger.debug('end')

<commit_msg>update __doc__<commit_after># coding: utf-8
'''
Created on 2014年8月5日

@author: lunatic
'''
import json
import os
import logging
import sys
from settings import COMPILER, OUTPUT, INPUT, TARGET

'''
编译所有更新或受到更新影响的文件
编译编译依赖且.o的文件
更详细的说：
        编译所有存在更新的源文件和受引用更新文件的源文件以及编译依赖不存在.o文件的源文件

处理步骤：
    1.提取所有原始文件时间戳信息
    2.和当前目录的文件时间戳进行对比找到更新的文件
    3.通过引用关系推断出直接或间接受到影响的文件
    4.合并编译所需的依赖文件和受到影响的文件
    5.生成编译器命令
    6.执行生成的命令
    7.将最新的时间戳信息保存到文件
'''

MAKE_FILE = '.mk'

FILE_PINS = {}
# HEAD_REGEX = r'^#include "(\w+.h)"$'

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
ch.setLevel(logging.DEBUG)
logger.addHandler(ch)
#
# class Config:
#     parser = ConfigParser.ConfigParser()
#     parser.read('build.cfg')
#     @classmethod
#     def get(cls, section, option):
#         return cls.parser.get(section, option)
#     @classmethod
#     def sections(cls):
#         return cls.parser.sections()
#     @classmethod
#     def options(cls, section):
#         return cls.parser.options(section)

class Recorder(object):

    def write(self, data={}):
        with open(MAKE_FILE, "w") as f:
            f.write(json.dumps(data));

    def update(self, data={}):
        _data = self.read()
        _data.update(data)
        self.write(_data)

    def read(self):
        if not os.path.exists(MAKE_FILE):
            return dict()
        with open(MAKE_FILE, "r") as f:
            return json.loads(f.read())

def get_timestamp(fn):
    return os.path.getmtime(fn)

def get_content(fn):
    with open(fn, "r") as f:
        content = f.read()
    return content

class HeadSet(object):
    '''
     头文件特殊处理：
    为每个头文件建立一个set set里面放入所有包含了该头文件的文件 如果其中有头文件包含了该头文件 递归的求包含该头文件的源文件 代替头文件本身返回
    '''
    def __init__(self):
        self.table = dict()

    def add_refs(self, fn, relfns_iter):
        self.table[fn] = set(relfns_iter)

    def adjust_refs(self):
        '''
        调整该结构使得所有的set都不包含头文件
        '''
        self.passed = set()
        for doth in self.table.keys():
            self.passed.clear()
            self.table[doth] = self.__adjust(doth)

    def __adjust(self, doth):
        '''
        获取包含该头文件的所有源文件
        '''
        # check processed return empty list
        if doth in self.passed:  # 循环引用
            return list()
        refs = self.search_refs(doth)
        # maybe sytanx error!
        if not refs:
            return list()
        dotcc = filter(lambda it: it.endswith('.cc'), refs)
        doths = filter(lambda it: it.endswith('.h'), refs)
        # add passed
        self.passed.add(doth)
        if doths:
            for ls in [self.__adjust(doth) for doth in doths]:
                dotcc.extend(ls)
        # return set(dotcc)  # filter distinct
        return set(dotcc)  # 间接引用

    def search_refs(self, fn):
        refs = self.table.get(fn)
        return refs if refs else set()

def get_diffs(origin={}, _dir=INPUT):
    '''
     获取指定目录下所有更新的源文件以及包含更新源文件的源文件
    '''
    logger.debug('Dir: %s', ' '.join(os.listdir(_dir)))

    files = [name for name in os.listdir(_dir) if name.endswith('.h') or name.endswith('.cc')]
    logger.debug('List of files: %s' % ' '.join(files))

    # name_conts = [(name, get_content(name)) for name in files]
    prints = {name: get_timestamp(os.path.join(_dir, name)) for name in files}
    global FILE_PINS
    FILE_PINS.update(prints)
    # 比对原始和最新的时间戳 过滤出所有变化的文件
    diff_pins = filter(lambda (name, pin): origin.get(name, 0.0) < pin, prints.items())
    if not diff_pins:
        logger.warn('No files changed!')
        return []

    diff_files = zip(*diff_pins)[0]
    logger.debug('List of changed files: %s', ' '.join(diff_files))

    compiles = list(filter(lambda name: name.endswith('.cc'), diff_files))

    hs = HeadSet()
    diff_hfiles = filter(lambda name: name.endswith('.h'), diff_files)
    # 找出所有更新的头文件的所有关联文件
    file_conts = [get_content(os.path.join(_dir, name)) for name in files]
    for hf in diff_hfiles:
        include = '#include "%s"' % hf
        rels = filter(lambda cont: cont.find(include) != -1, file_conts)
        if not rels:
            continue
        relfs = zip(*rels)[0]
        # logger.debug('rels', relfs
        hs.add_refs(hf, relfs)
    hs.adjust_refs()
    logger.debug('adjust: %s' % str(hs.table))

    for difls in [hs.search_refs(hf) for hf in diff_hfiles]:
        compiles.extend(difls)

    logger.debug('List of affected files: %s', ''.join(set(compiles)))
    return {}.fromkeys(compiles).keys()

class CommandBuilder(object):
    # g++ -O0 -g3 -Wall -c -fmessage-length=0
    COMPILE_CMD = '%(name)s -O%(opt)s -g%(debug)s %(warn)s -c %(other)s -o %(out)s %(input)s'
    LINK_CMD = '%(name)s -o %(target)s %(inputs)s'
    def __init__(self):
        self.command = {}
        self.command['name'] = COMPILER['name']

        opt = COMPILER.get('optimization')
        deb = COMPILER.get('debugging')
        warn = COMPILER.get('warning')
        other = COMPILER.get('other')

        self.command['opt'] = opt if opt else 0
        self.command['debug'] = deb if deb else 0
        self.command['warn'] = '-Wall' if warn and warn == True else ''
        self.command['other'] = other if other else ''

    def build_commands(self, diffs, target):
        comps = self.__merge(diffs, target)
        out = lambda arg: os.path.join(OUTPUT, arg.split('.')[0] + '.o')
        _in = lambda arg: os.path.join(INPUT, arg)
        tasks = []
        for fn in comps:
            dots = _in(fn)
            if not os.path.exists(dots):
                raise Exception('File Not Found "%s"' % dots)
            doto = out(fn)
            self.command['out'] = doto
            self.command['input'] = dots
            tasks.append(self.COMPILE_CMD % self.command)

        for fn, deps in target.items():
            self.command['inputs'] = ' '.join(map(out, deps))
            self.command['target'] = fn
            tasks.append(self.LINK_CMD % self.command)

        return tasks;

    def __merge(self, diffs, target):
        '''
        合并受更新影响的文件和编译依赖文件
        依赖文件编译要求： 如果依赖文件没有受到更新影响 而且.o文件存在则什么也不做 否则编译依赖文件
        '''
        comps = diffs
        logger.debug('merge diff and depends')
        depends = set(reduce(lambda x, y: x + y, target.values()))
        logger.debug('depends: %s', ' '.join(depends))
        out = lambda arg: os.path.join(OUTPUT, arg.split('.')[0] + '.o')
        logger.debug('comps: %s', ' '.join(comps))
        #comps.extend()
        comps.extend([dep for dep in depends if dep not in diffs and not os.path.exists(out(dep))])
        logger.debug('extend comps: %s', ' '.join(comps))
        return comps

def execute(cmds=[]):
    # doit = lambda task: os.popen(cmd).read()
    doit = lambda x:x
    return '\n'.join(map(doit, cmds))

def say(x):
    print x
    return x

def main(*args, **kwargs):

    recorder = Recorder()
    builder = CommandBuilder()

    origin = recorder.read()
    diffs = get_diffs(origin)

    cmds = builder.build_commands(diffs, TARGET)
    # tasks, upfiles = CommandBuilder().__get_comand(diffs, *TARGET['hello.exe'], target='hello.exe')
    rv = execute(cmds)
    print 'rv:\n', rv, '\n'
    recorder.update(FILE_PINS)

if __name__ == '__main__':
    logger.debug('start')
    main()
    logger.debug('end')

<|endoftext|>


<commit_before>import os
import time
import traceback
from threading import Thread

import praw
from bitcoinrpc.authproxy import AuthServiceProxy
from praw.models import Message, Comment

import bot_command
import bot_logger
import crypto
import lang
import user_function
import utils
from config import rpc_config, bot_config, DATA_PATH

class SoDogeTip():
    def __init__(self):
        self.reddit = praw.Reddit('sodogetiptest')

        self.rpc_main = AuthServiceProxy("http://%s:%s@%s:%s" % (
            rpc_config['doge_rpc_username'], rpc_config['doge_rpc_password'], rpc_config['doge_rpc_host'],
            rpc_config['doge_rpc_port']))

        self.rpc_antispam = AuthServiceProxy("http://%s:%s@%s:%s" % (
            rpc_config['doge_rpc_username'], rpc_config['doge_rpc_password'], rpc_config['doge_rpc_host'],
            rpc_config['doge_rpc_port']), timeout=120)

    def main(self):
        bot_logger.logger.info('Main Bot loop !')
        while True:
            try:
                if not os.path.exists(DATA_PATH + bot_config['user_history_path']):
                    os.makedirs(DATA_PATH + bot_config['user_history_path'])

                # create file if not exist (user storage)
                utils.create_user_storage()

                # create file if not exist (tip unregistered user )
                utils.create_unregistered_tip_storage()

                for msg in self.reddit.inbox.unread(limit=None):

                    if (type(msg) is not Message) and (type(msg) is not Comment):
                        bot_logger.logger.info('Not a good message !')
                        msg.reply(lang.message_not_supported)
                        self.mark_msg_read(msg)
                    else:
                        bot_logger.logger.info("%s - %s sub : %s" % (str(msg), msg.author.name, msg.subject))
                        msg_body = msg.body.strip()
                        msg_subject = msg.subject.strip()
                        split_message = msg_body.split()

                        if msg_body == '+register' and msg_subject == '+register':
                            self.mark_msg_read(msg)
                            bot_command.register_user(self.rpc_main, msg)

                        elif msg_body == '+info' and msg_subject == '+info':
                            self.mark_msg_read(msg)
                            bot_command.info_user(self.rpc_main, msg)

                        elif msg_body == '+help' and msg_subject == '+help':
                            self.mark_msg_read(msg)
                            bot_command.help_user(self.rpc_main, msg)

                        elif msg_body == '+balance' or msg_subject == '+balance':
                            self.mark_msg_read(msg)
                            bot_command.balance_user(self.rpc_main, msg)

                        elif msg_body == '+history' or msg_subject == '+history':
                            self.mark_msg_read(msg)
                            bot_command.history_user(msg)

                        elif split_message.count('+withdraw') and msg_subject == '+withdraw':
                            self.mark_msg_read(msg)
                            bot_command.withdraw_user(self.rpc_main, msg)

                        elif split_message.count('+/u/sodogetiptest'):
                            self.mark_msg_read(msg)
                            bot_command.tip_user(self.rpc_main, self.reddit, msg)

                        else:
                            self.mark_msg_read(msg)
                            # msg.reply('Currently not supported')
                            bot_logger.logger.info('Currently not supported')

                # to not explode rate limit :)
                bot_logger.logger.info('Make an pause !')
                time.sleep(3)
            except:
                traceback.print_exc()
                bot_logger.logger.error('Main Bot loop crashed...')
                time.sleep(10)

    def mark_msg_read(self, msg):
        unread_messages = [msg]
        self.reddit.inbox.mark_read(unread_messages)

    def process_pending_tip(self):
        while True:
            bot_logger.logger.info('Make clean of unregistered tips')
            bot_command.replay_remove_pending_tip(self.rpc_main)
            time.sleep(60)

    def anti_spamming_tx(self):
        # protect against spam attacks of an address having UTXOs.
        while True:
            bot_logger.logger.info('Make clean of tx')
            # get list of account
            list_account = user_function.get_users()
            for account, address in list_account.items():
                time.sleep(5)  # don't flood daemon
                list_tx = self.rpc_antispam.listunspent(1, 99999999999, [address])
                unspent_amounts = []
                for i in range(0, len(list_tx), 1):
                    unspent_amounts.append(list_tx[i]['amount'])
                    if i > 200:
                        break

                if len(list_tx) > int(bot_config['spam_limit']):
                    bot_logger.logger.info('Consolidate %s account !' % account)
                    amount = crypto.get_user_balance(self.rpc_antispam, account)
                    crypto.send_to(self.rpc_antispam, address, address, sum(unspent_amounts), True)
            time.sleep(240)

if __name__ == "__main__":
    bot_logger.logger.info("Bot Started !!")
    while True:
        try:
            Bot = SoDogeTip()

            thread_master = Thread(target=Bot.main)
            thread_pending_tip = Thread(target=Bot.process_pending_tip)
            thread_anti_spamming_tx = Thread(target=Bot.anti_spamming_tx)

            thread_master.start()
            thread_pending_tip.start()
            thread_anti_spamming_tx.start()

            thread_master.join()
            thread_pending_tip.join()
            thread_anti_spamming_tx.join()

            bot_logger.logger.error('All bot task finished ...')
        except:
            traceback.print_exc()
            bot_logger.logger.error('Resuming in 30sec...')
            time.sleep(30)
<commit_msg>Update<commit_after>import os
import time
import traceback
from threading import Thread

import praw
from bitcoinrpc.authproxy import AuthServiceProxy
from praw.models import Message, Comment

import bot_command
import bot_logger
import crypto
import lang
import user_function
import utils
from config import rpc_config, bot_config, DATA_PATH

class SoDogeTip():
    def __init__(self):
        self.reddit = praw.Reddit('sodogetiptest')

        self.rpc_main = AuthServiceProxy("http://%s:%s@%s:%s" % (
            rpc_config['doge_rpc_username'], rpc_config['doge_rpc_password'], rpc_config['doge_rpc_host'],
            rpc_config['doge_rpc_port']), timeout=120)

        self.rpc_antispam = AuthServiceProxy("http://%s:%s@%s:%s" % (
            rpc_config['doge_rpc_username'], rpc_config['doge_rpc_password'], rpc_config['doge_rpc_host'],
            rpc_config['doge_rpc_port']), timeout=120)

    def main(self):
        bot_logger.logger.info('Main Bot loop !')
        while True:
            try:
                if not os.path.exists(DATA_PATH + bot_config['user_history_path']):
                    os.makedirs(DATA_PATH + bot_config['user_history_path'])

                # create file if not exist (user storage)
                utils.create_user_storage()

                # create file if not exist (tip unregistered user )
                utils.create_unregistered_tip_storage()

                for msg in self.reddit.inbox.unread(limit=None):

                    if (type(msg) is not Message) and (type(msg) is not Comment):
                        bot_logger.logger.info('Not a good message !')
                        msg.reply(lang.message_not_supported)
                        self.mark_msg_read(msg)
                    else:
                        bot_logger.logger.info("%s - %s sub : %s" % (str(msg), msg.author.name, msg.subject))
                        msg_body = msg.body.strip()
                        msg_subject = msg.subject.strip()
                        split_message = msg_body.split()

                        if msg_body == '+register' and msg_subject == '+register':
                            self.mark_msg_read(msg)
                            bot_command.register_user(self.rpc_main, msg)

                        elif msg_body == '+info' and msg_subject == '+info':
                            self.mark_msg_read(msg)
                            bot_command.info_user(self.rpc_main, msg)

                        elif msg_body == '+help' and msg_subject == '+help':
                            self.mark_msg_read(msg)
                            bot_command.help_user(self.rpc_main, msg)

                        elif msg_body == '+balance' or msg_subject == '+balance':
                            self.mark_msg_read(msg)
                            bot_command.balance_user(self.rpc_main, msg)

                        elif msg_body == '+history' or msg_subject == '+history':
                            self.mark_msg_read(msg)
                            bot_command.history_user(msg)

                        elif split_message.count('+withdraw') and msg_subject == '+withdraw':
                            self.mark_msg_read(msg)
                            bot_command.withdraw_user(self.rpc_main, msg)

                        elif split_message.count('+/u/sodogetiptest'):
                            self.mark_msg_read(msg)
                            bot_command.tip_user(self.rpc_main, self.reddit, msg)

                        else:
                            self.mark_msg_read(msg)
                            # msg.reply('Currently not supported')
                            bot_logger.logger.info('Currently not supported')

                # to not explode rate limit :)
                bot_logger.logger.info('Make an pause !')
                time.sleep(3)
            except:
                traceback.print_exc()
                bot_logger.logger.error('Main Bot loop crashed...')
                time.sleep(10)

    def mark_msg_read(self, msg):
        unread_messages = [msg]
        self.reddit.inbox.mark_read(unread_messages)

    def process_pending_tip(self):
        while True:
            bot_logger.logger.info('Make clean of unregistered tips')
            bot_command.replay_remove_pending_tip(self.rpc_main)
            time.sleep(60)

    def anti_spamming_tx(self):
        # protect against spam attacks of an address having UTXOs.
        while True:
            bot_logger.logger.info('Make clean of tx')
            # get list of account
            list_account = user_function.get_users()
            for account, address in list_account.items():
                time.sleep(5)  # don't flood daemon
                list_tx = self.rpc_antispam.listunspent(1, 99999999999, [address])
                unspent_amounts = []
                for i in range(0, len(list_tx), 1):
                    unspent_amounts.append(list_tx[i]['amount'])
                    if i > 200:
                        break

                if len(list_tx) > int(bot_config['spam_limit']):
                    bot_logger.logger.info('Consolidate %s account !' % account)
                    amount = crypto.get_user_balance(self.rpc_antispam, account)
                    crypto.send_to(self.rpc_antispam, address, address, sum(unspent_amounts), True)
            time.sleep(240)

if __name__ == "__main__":
    bot_logger.logger.info("Bot Started !!")
    while True:
        try:
            Bot = SoDogeTip()

            thread_master = Thread(target=Bot.main)
            thread_pending_tip = Thread(target=Bot.process_pending_tip)
            thread_anti_spamming_tx = Thread(target=Bot.anti_spamming_tx)

            thread_master.start()
            thread_pending_tip.start()
            thread_anti_spamming_tx.start()

            thread_master.join()
            thread_pending_tip.join()
            thread_anti_spamming_tx.join()

            bot_logger.logger.error('All bot task finished ...')
        except:
            traceback.print_exc()
            bot_logger.logger.error('Resuming in 30sec...')
            time.sleep(30)
<|endoftext|>


<commit_before>from django.shortcuts import render
from django.http import HttpResponse
from .models import DoorStatus, OpenData
from django.views.decorators.csrf import csrf_exempt
from django.utils import timezone
from website import settings
from datetime import datetime
import json

# Create your views here.

@csrf_exempt
def door_post(request):
    if request.method == 'POST':
        unico = request.body.decode('utf-8')
        data = json.loads(unico)
        if 'key' in data:
            if data['key'] == settings.DOOR_KEY:
                if 'status' in data:
                    status = data['status']

                    if DoorStatus.objects.filter(name='hackerspace').count():
                        door_status_object = DoorStatus.objects.get(name='hackerspace')
                    else:
                        door_status_object = DoorStatus(name='hackerspace')

                    if status == True:
                        door_status_object.status = status
                        door_status_object.save()
                        if 'timeStart' in data and 'dateStart' in data:
                            timeStart = data['timeStart']
                            dateStart = data['dateStart']
                            opened = datetime.strptime(dateStart+"."+timeStart,"%Y-%m-%d.%H:%M:%S")
                            door_status_object.datetime = opened
                            door_status_object.save()
                    elif status == False:
                        door_status_object.status = status
                        door_status_object.save()
                        if 'timeStart' in data and 'dateStart' in data and 'timeEnd' in data and 'dateEnd' in data:
                            timeStart = data['timeStart']
                            dateStart = data['dateStart']
                            timeEnd = data['timeEnd']
                            dateEnd = data['dateEnd']
                            opened = datetime.strptime(dateStart+"."+timeStart,"%Y-%m-%d.%H:%M:%S")
                            closed = datetime.strptime(dateEnd+"."+timeEnd,"%Y-%m-%d.%H:%M:%S")
                            openData = OpenData(opened=opened, closed=closed)
                            openData.save()

                            door_status_object.datetime = closed
                            door_status_object.save()
    return HttpResponse(" ")

@csrf_exempt
def get_status(request):
    if DoorStatus.objects.filter(name='hackerspace').count():
        status = DoorStatus.objects.get(name='hackerspace').status
    else:
        status = True
    return HttpResponse(status)

def get_json(request):
    if DoorStatus.objects.filter(name='hackerspace').count():
        status = DoorStatus.objects.get(name='hackerspace').status
        lastChanged = str(DoorStatus.objects.get(name='hackerspace').datetime)
    else:
        status = True
        lastChanged = 'error'
    data = {}
    data['status'] = status
    data['lastChanged'] = lastChanged
    json_data = json.dumps(data)
    return HttpResponse(json_data)

def door_data(request):
    opendata_list = OpenData.objects.all()
    for data in opendata_list:
        data.deltaTime = data.closed - data.opened
    if DoorStatus.objects.filter(name='hackerspace').count():
        status = DoorStatus.objects.get(name='hackerspace')
    else:
        status = DoorStatus(name='hackerspace')
    context = {
        'opendata_list': opendata_list,
        'status': status,
    }

    return render(request, 'door_data.html', context)
<commit_msg>Use jsonResponse in door get_json<commit_after>from django.shortcuts import render
from django.http import HttpResponse
from .models import DoorStatus, OpenData
from django.views.decorators.csrf import csrf_exempt
from django.utils import timezone
from website import settings
from datetime import datetime
from django.http import JsonResponse
import json

# Create your views here.

@csrf_exempt
def door_post(request):
    if request.method == 'POST':
        unico = request.body.decode('utf-8')
        data = json.loads(unico)
        if 'key' in data:
            if data['key'] == settings.DOOR_KEY:
                if 'status' in data:
                    status = data['status']

                    if DoorStatus.objects.filter(name='hackerspace').count():
                        door_status_object = DoorStatus.objects.get(name='hackerspace')
                    else:
                        door_status_object = DoorStatus(name='hackerspace')

                    if status == True:
                        door_status_object.status = status
                        door_status_object.save()
                        if 'timeStart' in data and 'dateStart' in data:
                            timeStart = data['timeStart']
                            dateStart = data['dateStart']
                            opened = datetime.strptime(dateStart+"."+timeStart,"%Y-%m-%d.%H:%M:%S")
                            door_status_object.datetime = opened
                            door_status_object.save()
                    elif status == False:
                        door_status_object.status = status
                        door_status_object.save()
                        if 'timeStart' in data and 'dateStart' in data and 'timeEnd' in data and 'dateEnd' in data:
                            timeStart = data['timeStart']
                            dateStart = data['dateStart']
                            timeEnd = data['timeEnd']
                            dateEnd = data['dateEnd']
                            opened = datetime.strptime(dateStart+"."+timeStart,"%Y-%m-%d.%H:%M:%S")
                            closed = datetime.strptime(dateEnd+"."+timeEnd,"%Y-%m-%d.%H:%M:%S")
                            openData = OpenData(opened=opened, closed=closed)
                            openData.save()

                            door_status_object.datetime = closed
                            door_status_object.save()
    return HttpResponse(" ")

@csrf_exempt
def get_status(request):
    if DoorStatus.objects.filter(name='hackerspace').count():
        status = DoorStatus.objects.get(name='hackerspace').status
    else:
        status = True
    return HttpResponse(status)

def get_json(request):
    if DoorStatus.objects.filter(name='hackerspace').count():
        status = DoorStatus.objects.get(name='hackerspace').status
        lastChanged = str(DoorStatus.objects.get(name='hackerspace').datetime)
    else:
        status = True
        lastChanged = 'error'
    data = {}
    data['status'] = status
    data['lastChanged'] = lastChanged
    return JsonResponse(data)

def door_data(request):
    opendata_list = OpenData.objects.all()
    for data in opendata_list:
        data.deltaTime = data.closed - data.opened
    if DoorStatus.objects.filter(name='hackerspace').count():
        status = DoorStatus.objects.get(name='hackerspace')
    else:
        status = DoorStatus(name='hackerspace')
    context = {
        'opendata_list': opendata_list,
        'status': status,
    }

    return render(request, 'door_data.html', context)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
import log
import json
import time
import datetime
import sqlite3
from os.path import isfile
from collections import UserDict

_logger = log.getChild('record')
#_logger.setLevel(log.DEBUG)

class DownloadRecord(dict):
    def __init__(self, url, local_file, description, download_time=None, raw=None, is_accompany=False):
        UserDict.__init__(self)
        if download_time is None:
            download_time = datetime.datetime.fromtimestamp(time.time())
        timestr = download_time.isoformat()

        self['url'] = url
        self['local_file'] = local_file
        self['description'] = description
        self['time'] = timestr
        self['raw'] = raw
        self['is_accompany'] = is_accompany

null_record = DownloadRecord('', '', datetime.datetime.fromtimestamp(0))

class DownloadRecordManager(dict):
    def __init__(self, name):
        dict.__init__(self)
        self.name = name
        self.clear()

    def save(self, f):
        json.dump(self, f)

    def load(self, f):
        self.clear()
        try:
            content = json.load(f)
        except:
            _logger.warning('error occurs when load json file', exc_info=1)
            return
        _logger.debug('json file loaded:\n%s', str(content))
        for r in content.values():
            try:
                exists = isfile(r['local_file'])
            except:
                _logger.debug('error occurs when detecting saved file', exc_info=1)
                exists = False
            if exists:
                self.add(r)
            else:
                _logger.debug('%s doesn\'t exist any more', r)
        _logger.debug('history %s loaded', self)

    def add(self, r):
        r = dict(r)
        if 'raw' in r:
            r.pop('raw')
        self[r['url']] = r

    def get_by_url(self, url, default_rec=null_record):
        return self.get(url, default_rec)

default_manager = DownloadRecordManager('default')

class SqlDatabaseRecordManager(DownloadRecordManager):
    def add(self, r):
        self[r['url']] = r

    def save(self, f):
        conn = sqlite3.connect(f)
        cur = conn.cursor()
        cur.execute('''
            CREATE TABLE IF NOT EXISTS [BingWallpaperRecords] (
              [Url] CHAR(1024) NOT NULL ON CONFLICT FAIL,
              [DownloadTime] DATETIME NOT NULL ON CONFLICT FAIL,
              [LocalFilePath] CHAR(1024),
              [Description] TEXT(1024),
              [Image] BLOB,
              [IsAccompany] BOOLEAN DEFAULT False,
              CONSTRAINT [sqlite_autoindex_BingWallpaperRecords_1] PRIMARY KEY ([Url]));
        ''')
        conn.commit()
        for k,v in self.items():
            cur.execute('''
                INSERT OR REPLACE INTO [BingWallpaperRecords]
                  (Url, DownloadTime, LocalFilePath, Description, Image, IsAccompany)
                  VALUES (?, ?, ?, ?, ?, ?)
            ''', (k, v['time'], v['local_file'], v['description'], v['raw'], v['is_accompany']))
        conn.commit()

    def load(self, f):
        raise NotImplementedError("sql database is write-only")
<commit_msg>use utc in download record time field, make sqlite compatible<commit_after>#!/usr/bin/env python3
import log
import json
import datetime
import sqlite3
from os.path import isfile
from collections import UserDict

_logger = log.getChild('record')
#_logger.setLevel(log.DEBUG)

class DownloadRecord(dict):
    def __init__(self, url, local_file, description, download_time=None, raw=None, is_accompany=False):
        UserDict.__init__(self)
        if download_time is None:
            download_time = datetime.datetime.utcnow()
        timestr = download_time.isoformat()

        self['url'] = url
        self['local_file'] = local_file
        self['description'] = description
        self['time'] = timestr
        self['raw'] = raw
        self['is_accompany'] = is_accompany

null_record = DownloadRecord('', '', datetime.datetime.fromtimestamp(0))

class DownloadRecordManager(dict):
    def __init__(self, name):
        dict.__init__(self)
        self.name = name
        self.clear()

    def save(self, f):
        json.dump(self, f)

    def load(self, f):
        self.clear()
        try:
            content = json.load(f)
        except:
            _logger.warning('error occurs when load json file', exc_info=1)
            return
        _logger.debug('json file loaded:\n%s', str(content))
        for r in content.values():
            try:
                exists = isfile(r['local_file'])
            except:
                _logger.debug('error occurs when detecting saved file', exc_info=1)
                exists = False
            if exists:
                self.add(r)
            else:
                _logger.debug('%s doesn\'t exist any more', r)
        _logger.debug('history %s loaded', self)

    def add(self, r):
        r = dict(r)
        if 'raw' in r:
            r.pop('raw')
        self[r['url']] = r

    def get_by_url(self, url, default_rec=null_record):
        return self.get(url, default_rec)

default_manager = DownloadRecordManager('default')

class SqlDatabaseRecordManager(DownloadRecordManager):
    def add(self, r):
        self[r['url']] = r

    def save(self, f):
        conn = sqlite3.connect(f)
        cur = conn.cursor()
        cur.execute('''
            CREATE TABLE IF NOT EXISTS [BingWallpaperRecords] (
              [Url] CHAR(1024) NOT NULL ON CONFLICT FAIL,
              [DownloadTime] DATETIME NOT NULL ON CONFLICT FAIL,
              [LocalFilePath] CHAR(1024),
              [Description] TEXT(1024),
              [Image] BLOB,
              [IsAccompany] BOOLEAN DEFAULT False,
              CONSTRAINT [sqlite_autoindex_BingWallpaperRecords_1] PRIMARY KEY ([Url]));
        ''')
        conn.commit()
        for k,v in self.items():
            cur.execute('''
                INSERT OR REPLACE INTO [BingWallpaperRecords]
                  (Url, DownloadTime, LocalFilePath, Description, Image, IsAccompany)
                  VALUES (?, ?, ?, ?, ?, ?)
            ''', (k, v['time'], v['local_file'], v['description'], v['raw'], v['is_accompany']))
        conn.commit()

    def load(self, f):
        raise NotImplementedError("sql database is write-only")
<|endoftext|>


<commit_before>import math
import sys
import pyopencl as cl
import numpy

from PIL import Image

from log import *

def reduceImage(image, clContext, clQueue, buckets):
    if not hasattr(reduceImage, "program"):
        kernelFile = open('src/reduceImage.cl', 'r')
        reduceImage.program = cl.Program(clContext, kernelFile.read()).build()
        kernelFile.close()

    mf = cl.mem_flags
    input = numpy.asarray(image).astype(numpy.uint8)
    output = numpy.zeros((image.size[1], image.size[0] / buckets)).astype(numpy.uint8)
    q = numpy.zeros((image.size[1], image.size[0] / buckets)).astype(numpy.uint8)

    inputBuffer = cl.Buffer(clContext, mf.READ_WRITE | mf.COPY_HOST_PTR, hostbuf=input)
    outputBuffer = cl.Buffer(clContext, mf.READ_WRITE | mf.COPY_HOST_PTR, hostbuf=output)
    qBuffer = cl.Buffer(clContext, mf.READ_WRITE | mf.COPY_HOST_PTR, hostbuf=q)

    reduceImage.program.reduceImage(clQueue, [(image.size[0] / buckets) * image.size[1]], None, inputBuffer, outputBuffer, qBuffer, numpy.uint32(image.size[0]), numpy.uint32(image.size[1]), numpy.uint32(buckets)).wait()

    cl.enqueue_read_buffer(clQueue, outputBuffer, output).wait()

    outputImage = Image.fromarray(output)

    return outputImage
<commit_msg>fix some noise<commit_after>import math
import sys
import pyopencl as cl
import numpy
import numpy.numarray.nd_image

from PIL import Image

from log import *

def reduceImage(image, clContext, clQueue, buckets):
    if not hasattr(reduceImage, "program"):
        kernelFile = open('src/reduceImage.cl', 'r')
        reduceImage.program = cl.Program(clContext, kernelFile.read()).build()
        kernelFile.close()

    mf = cl.mem_flags
    input = numpy.asarray(image).astype(numpy.uint8)
    output = numpy.zeros((image.size[1], image.size[0] / buckets)).astype(numpy.uint8)
    q = numpy.zeros((image.size[1], image.size[0] / buckets)).astype(numpy.uint8)

    inputBuffer = cl.Buffer(clContext, mf.READ_WRITE | mf.COPY_HOST_PTR, hostbuf=input)
    outputBuffer = cl.Buffer(clContext, mf.READ_WRITE | mf.COPY_HOST_PTR, hostbuf=output)
    qBuffer = cl.Buffer(clContext, mf.READ_WRITE | mf.COPY_HOST_PTR, hostbuf=q)

    reduceImage.program.reduceImage(clQueue, [(image.size[0] / buckets) * image.size[1]], None, inputBuffer, outputBuffer, qBuffer, numpy.uint32(image.size[0]), numpy.uint32(image.size[1]), numpy.uint32(buckets)).wait()

    cl.enqueue_read_buffer(clQueue, outputBuffer, output).wait()

    filtered = numpy.numarray.nd_image.grey_opening(output, size=(3,3))

    outputImage = Image.fromarray(filtered)

    return outputImage
<|endoftext|>


<commit_before>"""
The MIT License (MIT)

Copyright (c) 2017 Stefan Graupner

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""

import json
import time

from colorama import Fore, Style

from gi.repository import OParl

from src.utils import *
from src.cache import *

class Result(object):
    """
        Validation Result

        The validation result contains a set of messages for each object
        type as well as some general statistics and messages regarding
        validation events that do not belong into the above categories.
        That last category, among other things, keeps taps on what objects
        or files were unreachable.
    """

    total_entities = 0
    failed_entities = 0

    object_messages = {}

    """ this is a list of objects that lead to unrecoverable error in respect to the spec """
    fatal_objects = []

    network = {
        'average_ttl': 0,
        'ssl': False,
        'encodings': []
    }

    oparl_version = '1.0'

    cache = None

    def __init__(self, cache):
        self.cache = cache

    def format_severity(self, severity):
        # TODO: Rewrite this to handle ValidationResult Severities?
        if severity == Result.Severity.Debug:
            return "Debug"
        if severity == Result.Severity.Info:
            return "Info"
        if severity == Result.Severity.Warning:
            return "Warning"
        if severity == Result.Severity.Error:
            return "Error"

    def parse_validation_result(self, object, validation_result):
        """
            Parse a liboparl ValidationResult into a validator message
        """
        severity = validation_result.get_severity()
        description = validation_result.get_description()

        oparl_type = OParlType(object)

        if oparl_type.entity not in self.object_messages:
            self.object_messages[oparl_type.entity] = {}

        new_message = {
            'severity': severity,
            'message': description,
            'count': 0,
            'objects': []
        }

        message_hash = sha1_hexdigest(oparl_type.entity + description)

        # insert message dict
        if message_hash not in self.object_messages[oparl_type.entity]:
            self.object_messages[oparl_type.entity][message_hash] = new_message

        # increment message occurence counter and add object id to reference list
        message = self.object_messages[oparl_type.entity][message_hash]

        message['count'] += 1
        if object.get_id() not in message['objects']:
            message['objects'].append(object.get_id())

        self.object_messages[oparl_type.entity][message_hash] = message

    def __str__(self):
        return json.dumps({
            'counts': {
                'total': self.total_entities,
                'valid': self.total_entities - self.failed_entities,
                'failed': self.failed_entities,
                'fatal': len(self.fatal_objects)
            },
            'object_messages': [v for v in self.object_messages.values()],
            'network': self.network,
            'oparl_version': self.oparl_version,
            'timestamp': time.time()
        })
<commit_msg>timedelta != int<commit_after>"""
The MIT License (MIT)

Copyright (c) 2017 Stefan Graupner

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""

import json
import time

from colorama import Fore, Style

from gi.repository import OParl

from src.utils import *
from src.cache import *

class Result(object):
    """
        Validation Result

        The validation result contains a set of messages for each object
        type as well as some general statistics and messages regarding
        validation events that do not belong into the above categories.
        That last category, among other things, keeps taps on what objects
        or files were unreachable.
    """

    total_entities = 0
    failed_entities = 0

    object_messages = {}

    """ this is a list of objects that lead to unrecoverable error in respect to the spec """
    fatal_objects = []

    network = {
        'average_ttl': 0,
        'ssl': False,
        'encodings': []
    }

    oparl_version = '1.0'

    cache = None

    def __init__(self, cache):
        self.cache = cache

    def format_severity(self, severity):
        # TODO: Rewrite this to handle ValidationResult Severities?
        if severity == Result.Severity.Debug:
            return "Debug"
        if severity == Result.Severity.Info:
            return "Info"
        if severity == Result.Severity.Warning:
            return "Warning"
        if severity == Result.Severity.Error:
            return "Error"

    def parse_validation_result(self, object, validation_result):
        """
            Parse a liboparl ValidationResult into a validator message
        """
        severity = validation_result.get_severity()
        description = validation_result.get_description()

        oparl_type = OParlType(object)

        if oparl_type.entity not in self.object_messages:
            self.object_messages[oparl_type.entity] = {}

        new_message = {
            'severity': severity,
            'message': description,
            'count': 0,
            'objects': []
        }

        message_hash = sha1_hexdigest(oparl_type.entity + description)

        # insert message dict
        if message_hash not in self.object_messages[oparl_type.entity]:
            self.object_messages[oparl_type.entity][message_hash] = new_message

        # increment message occurence counter and add object id to reference list
        message = self.object_messages[oparl_type.entity][message_hash]

        message['count'] += 1
        if object.get_id() not in message['objects']:
            message['objects'].append(object.get_id())

        self.object_messages[oparl_type.entity][message_hash] = message

    def __str__(self):
        return json.dumps({
            'counts': {
                'total': self.total_entities,
                'valid': self.total_entities - self.failed_entities,
                'failed': self.failed_entities,
                'fatal': len(self.fatal_objects)
            },
            'object_messages': [v for v in self.object_messages.values()],
            'network': self.network,
            'oparl_version': self.oparl_version,
            'timestamp': int(time.time())
        })
<|endoftext|>


<commit_before>from flask import Flask, render_template, send_from_directory, request, jsonify
from flask_webpack import Webpack
from gevent.wsgi import WSGIServer

import os

from elasticsearch import Elasticsearch

from search import build_search_query, build_es_search_body_request, \
    build_es_aggregation_body_request, format_search_results, \
    format_aggregation_results, build_autocomplete_search_body_request, \
    format_autocomplete_results

es = Elasticsearch(os.environ['ES_URI'], timeout=5, retry_on_timeout=False)
ES_INDEX = 'searchable_items_blue'

app = Flask(__name__)

webpack = Webpack()
params = {
    'DEBUG': True,
    'WEBPACK_MANIFEST_PATH': './build/manifest.json'
}

app.config.update(params)
webpack.init_app(app)

@app.route('/api/search')
def search():
    query = request.args.get('q', '')
    limit = int(request.args.get('limit', 10))
    offset = int(request.args.get('offset', 0))
    category = request.args.get('category', '')
    sort_by = request.args.get('sort_by', '')

    category_filters = {
        "gene": ['gene_type', 'gene_biological_process', 'gene_molecular_function', 'gene_cellular_component', 'species'],
        "go": ['go_species', 'go_genes', 'go_type']
    }

    search_fields = ['name', 'gene_symbol', 'gene_synonyms', 'description', 'external_ids', 'species', 'gene_biological_process', 'gene_molecular_function', 'gene_cellular_component', 'go_type', 'go_genes']

    json_response_fields = ['name', 'gene_symbol', 'gene_synonyms', 'gene_type', 'gene_chromosome_starts', 'gene_chromosome_ends', 'description', 'external_ids', 'species', 'gene_biological_process', 'gene_molecular_function', 'gene_cellular_component', 'go_type', 'go_genes', 'category', 'href']

    es_query = build_search_query(query, search_fields, category,
                                  category_filters, request.args)

    search_body = build_es_search_body_request(query,
                                               category,
                                               es_query,
                                               json_response_fields,
                                               search_fields,
                                               sort_by)

    search_results = es.search(
        index=ES_INDEX,
        body=search_body,
        size=limit,
        from_=offset
    )

    if search_results['hits']['total'] == 0:
        return jsonify({
            'total': 0,
            'results': [],
            'aggregations': []
        })

    aggregation_body = build_es_aggregation_body_request(
        es_query,
        category,
        category_filters
    )

    aggregation_results = es.search(
        index=ES_INDEX,
        body=aggregation_body
    )

    response = {
        'total': search_results['hits']['total'],
        'results': format_search_results(search_results, json_response_fields),
        'aggregations': format_aggregation_results(
            aggregation_results,
            category,
            category_filters
        )
    }

    return jsonify(response)

@app.route('/api/search_autocomplete')
def search_autocomplete():
    query = request.args.get('q', '')
    category = request.args.get('category', '')
    field = request.args.get('field', 'name')

    if query == '':
        return jsonify({
            "results": None
        })

    autocomplete_results = es.search(
        index=ES_INDEX,
        body=build_autocomplete_search_body_request(query, category, field)
    )

    return jsonify({
        "results": format_autocomplete_results(autocomplete_results, field)
    })

# make static assets available anyway
@app.route('/assets/<path:path>')
def send_static(path):
    return send_from_directory('build', path)

# render user interfaces in client JS
@app.route('/')
@app.route('/about')
@app.route('/help')
@app.route('/search')
def react_render():
        return render_template('index.jinja2')

if __name__ == '__main__':
    if os.environ.get('PRODUCTION', ''):
        http_server = WSGIServer(('', 5000), app)
        http_server.serve_forever()
    else:
        app.run(debug=True)
<commit_msg>reordering<commit_after>from flask import Flask, render_template, send_from_directory, request, jsonify
from flask_webpack import Webpack
from gevent.wsgi import WSGIServer

import os

from elasticsearch import Elasticsearch

from search import build_search_query, build_es_search_body_request, \
    build_es_aggregation_body_request, format_search_results, \
    format_aggregation_results, build_autocomplete_search_body_request, \
    format_autocomplete_results

es = Elasticsearch(os.environ['ES_URI'], timeout=5, retry_on_timeout=False)
ES_INDEX = 'searchable_items_blue'

app = Flask(__name__)

webpack = Webpack()
params = {
    'DEBUG': True,
    'WEBPACK_MANIFEST_PATH': './build/manifest.json'
}

app.config.update(params)
webpack.init_app(app)

@app.route('/api/search')
def search():
    query = request.args.get('q', '')
    limit = int(request.args.get('limit', 10))
    offset = int(request.args.get('offset', 0))
    category = request.args.get('category', '')
    sort_by = request.args.get('sort_by', '')

    category_filters = {
        "gene": ['gene_type', 'gene_biological_process', 'gene_molecular_function', 'gene_cellular_component', 'species'],
        "go": ['go_type', 'go_species', 'go_genes']
    }

    search_fields = ['name', 'gene_symbol', 'gene_synonyms', 'description', 'external_ids', 'species', 'gene_biological_process', 'gene_molecular_function', 'gene_cellular_component', 'go_type', 'go_genes']

    json_response_fields = ['name', 'gene_symbol', 'gene_synonyms', 'gene_type', 'gene_chromosome_starts', 'gene_chromosome_ends', 'description', 'external_ids', 'species', 'gene_biological_process', 'gene_molecular_function', 'gene_cellular_component', 'go_type', 'go_genes', 'category', 'href']

    es_query = build_search_query(query, search_fields, category,
                                  category_filters, request.args)

    search_body = build_es_search_body_request(query,
                                               category,
                                               es_query,
                                               json_response_fields,
                                               search_fields,
                                               sort_by)

    search_results = es.search(
        index=ES_INDEX,
        body=search_body,
        size=limit,
        from_=offset
    )

    if search_results['hits']['total'] == 0:
        return jsonify({
            'total': 0,
            'results': [],
            'aggregations': []
        })

    aggregation_body = build_es_aggregation_body_request(
        es_query,
        category,
        category_filters
    )

    aggregation_results = es.search(
        index=ES_INDEX,
        body=aggregation_body
    )

    response = {
        'total': search_results['hits']['total'],
        'results': format_search_results(search_results, json_response_fields),
        'aggregations': format_aggregation_results(
            aggregation_results,
            category,
            category_filters
        )
    }

    return jsonify(response)

@app.route('/api/search_autocomplete')
def search_autocomplete():
    query = request.args.get('q', '')
    category = request.args.get('category', '')
    field = request.args.get('field', 'name')

    if query == '':
        return jsonify({
            "results": None
        })

    autocomplete_results = es.search(
        index=ES_INDEX,
        body=build_autocomplete_search_body_request(query, category, field)
    )

    return jsonify({
        "results": format_autocomplete_results(autocomplete_results, field)
    })

# make static assets available anyway
@app.route('/assets/<path:path>')
def send_static(path):
    return send_from_directory('build', path)

# render user interfaces in client JS
@app.route('/')
@app.route('/about')
@app.route('/help')
@app.route('/search')
def react_render():
        return render_template('index.jinja2')

if __name__ == '__main__':
    if os.environ.get('PRODUCTION', ''):
        http_server = WSGIServer(('', 5000), app)
        http_server.serve_forever()
    else:
        app.run(debug=True)
<|endoftext|>


<commit_before>from flask import Flask, request, json
from flask_socketio import SocketIO, leave_room, emit
import requests

from Controllers.EvaluatorController import EvaluatorController

app = Flask(__name__, static_folder='')
app.config['SECRET_KEY'] = '@server-secret'
socketio = SocketIO(app)

class Socket:
    """
        Class used to emit answer to specific client
    """

    def __init__(self, sid):
        self.sid = sid
        self.connected = True

    # Emits data to a socket's unique room
    def emit(self, event, data):
        print('going to emit to', self.sid)
        socketio.emit(event, data, room=self.sid, namespace='/code-recommendations')

@app.route('/')
def index():
    return 'Hello, World new version!'

@app.route('/source-codes', methods=['POST'])
def source_codes():
    print('receive data in source-codes new version')
    EvaluatorController().evaluate_search_codes(request)
    return json.dumps({'success': True})

def get_source_codes(data):
    url = 'http://0.0.0.0:1111/crawl'
    headers = {'Content-Type': 'application/json'}
    print('going to request new version')
    requests.request(url=url, method='GET', data=data, headers=headers)

@app.route('/train-network', methods=['POST'])
def train_network():
    print('/train-network')
    print(request.get_json())
    print(type(request.get_json()))
    print(request.get_json()['train_text'])
    train_base = request.get_json().get('train_text')
    EvaluatorController().train_network(train_database=train_base)
    return json.dumps({'success': True})

# TODO: maybe use on connect
@socketio.on('connect')
def connect():
    print('connectttsssss')

@socketio.on('getCodes', namespace='/code-recommendations')
def get_recommendation_codes(data):
    data = json.loads(data)
    print(data)
    EvaluatorController().get_recommendation_code(request_id=request.sid,
                                                  language=data['language'],
                                                  query=data['query'],
                                                  comments=data['comments'],
                                                  libs=data['libs'])

def emit_code_recommendations(request_id, data):
    Socket(request_id).emit('recommendationCodes', data)

if __name__ == "__main__":
    socketio.run(app, host='0.0.0.0', port=6060, debug=True)
<commit_msg>adding log<commit_after>from flask import Flask, request, json
from flask_socketio import SocketIO, leave_room, emit
import requests

from Controllers.EvaluatorController import EvaluatorController

app = Flask(__name__, static_folder='')
app.config['SECRET_KEY'] = '@server-secret'
socketio = SocketIO(app)

class Socket:
    """
        Class used to emit answer to specific client
    """

    def __init__(self, sid):
        self.sid = sid
        self.connected = True

    # Emits data to a socket's unique room
    def emit(self, event, data):
        print('going to emit to', self.sid)
        socketio.emit(event, data, room=self.sid, namespace='/code-recommendations')

@app.route('/')
def index():
    return 'Hello, World new version!'

@app.route('/source-codes', methods=['POST'])
def source_codes():
    print('receive data in source-codes new version')
    EvaluatorController().evaluate_search_codes(request)
    return json.dumps({'success': True})

def get_source_codes(data):
    url = 'http://0.0.0.0:1111/crawl'
    headers = {'Content-Type': 'application/json'}
    print('going to request new version')
    requests.request(url=url, method='GET', data=data, headers=headers)

@app.route('/train-network', methods=['POST'])
def train_network():
    print('/train-network')
    print(request.get_json())
    print(type(request.get_json()))
    print(request.get_data())
    print(type(request.get_data()))
    print(json.load(request.get_json()))
    print(json.load(request.get_data()))
    train_base = json.load(request.get_json()).get('train_text')
    EvaluatorController().train_network(train_database=train_base)
    return json.dumps({'success': True})

# TODO: maybe use on connect
@socketio.on('connect')
def connect():
    print('connectttsssss')

@socketio.on('getCodes', namespace='/code-recommendations')
def get_recommendation_codes(data):
    data = json.loads(data)
    print(data)
    EvaluatorController().get_recommendation_code(request_id=request.sid,
                                                  language=data['language'],
                                                  query=data['query'],
                                                  comments=data['comments'],
                                                  libs=data['libs'])

def emit_code_recommendations(request_id, data):
    Socket(request_id).emit('recommendationCodes', data)

if __name__ == "__main__":
    socketio.run(app, host='0.0.0.0', port=6060, debug=True)
<|endoftext|>