<commit_before># seed.py
# Various environment helper functions for random
# seeds, including saving and loading seeds.

from __future__ import division
import os.path
import datetime

from numpy import sqrt
import numpy as np

def save_seeds(seeds, description='', filename=None, dir='../data'):
    t = datetime.datetime.now()
    if filename is None:
        ts = ['seeds', t.year, t.month, t.day, t.hour, t.second]
        filename = '_'.join([str(t) for t in ts])
    path = os.path.join(dir, filename)
    np.save(path, seeds)

def load_seeds(filename, dir='../data'):
    path = os.path.join(dir, filename)
    return np.load(path)

def set_simulation_parameters(seed, N, i):
    """
    Common simulation parameters for FORCE/DFORCE

    seed: float
    N: int
    i: int

    """
    prng = np.random.RandomState(seed)

    J = prng.normal(0, sqrt(1 / N), (N, N)) # primary stochastic matrix
    Wi = prng.normal(0, sqrt(1 / i), (i, i)) # secondary stochastic matrix
    wi = prng.normal(0, sqrt(1 / i), (i, i)) # t
    x0 = prng.uniform(-0.5, 0.5, N) # initial x0

    u = prng.uniform(-1, 1, N)
    w = prng.uniform(-1 / sqrt(N), 1 / sqrt(N), N)  # Initial weights

    return J, Wi, wi, x0, u, w<commit_msg>seed documenter added; TODO; testing descriptor<commit_after># seed.py
# Various environment helper functions for random
# seeds, including saving and loading seeds.

from __future__ import division
import os.path
import datetime
import csv

from numpy import sqrt
import numpy as np

def save_seeds(seeds, description='', filename=None, dir='../data', seed_key=None):
    t = datetime.datetime.now()
    if filename is None:
        ts = ['seeds', t.year, t.month, t.day, t.hour, t.second]
        filename = '_'.join([str(t) for t in ts])
    path = os.path.join(dir, filename)
    np.save(path, seeds)
    _insert_csv(seeds, description, filename, seed_key, dir)

def _insert_csv(seeds, description, filename, seed_key, dir):
    """
    Documentations for different seeds.
    """
    rows = []
    path = os.path.join(dir,'seeds.csv')

    if os.path.isfile(path):
         with open(path, 'r') as csvfile:
            reader = csv.reader(csvfile)
            for row in reader:
                rows.append(row)

    if seed_key is None:
        if len(rows) != 0:
            seed_key = int(rows[-1][0])+1 # Using last seed key + 1
        else:
            seed_key = 0

    keys = [row[0] for row in rows]
    while seed_key in keys:
        seed_key += 1

    print "Seed Key: %d" % seed_key

    new_row = [seed_key, filename, str(seeds), description]

    with open(path, 'w') as csvfile:
        fieldnames = ['key', 'filename', 'seeds', 'description']
        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
        for row in rows:
            writer.writerow({fn:f for fn, f in zip(fieldnames, row)})
        writer.writerow({fn:f for fn, f in zip(fieldnames, new_row)})

def load_seeds(filename, dir='../data'):
    path = os.path.join(dir, filename)
    return np.load(path)

def set_simulation_parameters(seed, N, i):
    """
    Common simulation parameters for FORCE/DFORCE

    seed: float
    N: int
    i: int
    """
    prng = np.random.RandomState(seed)

    J = prng.normal(0, sqrt(1 / N), (N, N)) # primary stochastic matrix
    Wi = prng.normal(0, sqrt(1 / i), (i, i)) # secondary stochastic matrix
    wi = prng.normal(0, sqrt(1 / i), (i, i)) # ternary stochastic matrix
    x0 = prng.uniform(-0.5, 0.5, N) # initial state (x0)

    u = prng.uniform(-1, 1, N) #
    w = prng.uniform(-1 / sqrt(N), 1 / sqrt(N), N)  # Initial weights

    return J, Wi, wi, x0, u, w<|endoftext|>


<commit_before># coding=utf8

# Copyright (C) 2012 Saúl Ibarra Corretgé <saghul@gmail.com>
#

__all__ = ['install']
__version__ = '0.1.0'

# Patchers

def patch_sleep():
    import gevent
    def sleep(seconds=0, ref=True):
        hub = gevent.hub.get_hub()
        loop = hub.loop
        watcher = loop.timer(max(seconds, 0), ref=ref)
        hub.wait(watcher)
    gevent.sleep = sleep
    gevent.hub.sleep = sleep

def patch_loop():
    from .loop import UVLoop
    from gevent.hub import Hub
    from gevent.resolver_thread import Resolver
    Hub.loop_class = UVLoop
    # The c-ares based resolver cannot be used for the moment
    Hub.resolver_class = Resolver

def install():
    patch_sleep()
    patch_loop()

<commit_msg>Raised version<commit_after># coding=utf8

# Copyright (C) 2012 Saúl Ibarra Corretgé <saghul@gmail.com>
#

__all__ = ['install']
__version__ = '0.2.0'

# Patchers

def patch_sleep():
    import gevent
    def sleep(seconds=0, ref=True):
        hub = gevent.hub.get_hub()
        loop = hub.loop
        watcher = loop.timer(max(seconds, 0), ref=ref)
        hub.wait(watcher)
    gevent.sleep = sleep
    gevent.hub.sleep = sleep

def patch_loop():
    from .loop import UVLoop
    from gevent.hub import Hub
    from gevent.resolver_thread import Resolver
    Hub.loop_class = UVLoop
    # The c-ares based resolver cannot be used for the moment
    Hub.resolver_class = Resolver

def install():
    patch_sleep()
    patch_loop()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Form implementation generated from reading ui file 'ui/graph.ui'
#
# Created: Wed May 16 11:38:13 2012
#      by: PyQt4 UI code generator 4.8.5
#
# WARNING! All changes made in this file will be lost!

from PyQt4 import QtCore, QtGui

try:
    _fromUtf8 = QtCore.QString.fromUtf8
except AttributeError:
    _fromUtf8 = lambda s: s

class Ui_Graph(object):
    def setupUi(self, Graph):
        Graph.setObjectName(_fromUtf8("Graph"))
        Graph.resize(645, 552)
        Graph.setWindowTitle(QtGui.QApplication.translate("Graph", "Graphique", None, QtGui.QApplication.UnicodeUTF8))
        self.dockWidgetContents = QtGui.QWidget()
        self.dockWidgetContents.setObjectName(_fromUtf8("dockWidgetContents"))
        self.verticalLayout = QtGui.QVBoxLayout(self.dockWidgetContents)
        self.verticalLayout.setObjectName(_fromUtf8("verticalLayout"))
        self.horizontalLayout = QtGui.QHBoxLayout()
        self.horizontalLayout.setObjectName(_fromUtf8("horizontalLayout"))
        self.taux_btn = QtGui.QCheckBox(self.dockWidgetContents)
        self.taux_btn.setToolTip(QtGui.QApplication.translate("Graph", "Voir les taux moyens et marginaux d\'imposition", None, QtGui.QApplication.UnicodeUTF8))
        self.taux_btn.setText(QtGui.QApplication.translate("Graph", "Taux moyens\n"
"Taux marginaux", None, QtGui.QApplication.UnicodeUTF8))
        self.taux_btn.setObjectName(_fromUtf8("taux_btn"))
        self.horizontalLayout.addWidget(self.taux_btn)
        self.absBox = QtGui.QComboBox(self.dockWidgetContents)
        self.absBox.setObjectName(_fromUtf8("absBox"))
        self.horizontalLayout.addWidget(self.absBox)
        spacerItem = QtGui.QSpacerItem(40, 20, QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Minimum)
        self.horizontalLayout.addItem(spacerItem)
        self.hidelegend_btn = QtGui.QPushButton(self.dockWidgetContents)
        self.hidelegend_btn.setToolTip(QtGui.QApplication.translate("Graph", "Masquer la légende", None, QtGui.QApplication.UnicodeUTF8))
        self.hidelegend_btn.setText(_fromUtf8(""))
        icon = QtGui.QIcon()
        icon.addPixmap(QtGui.QPixmap(_fromUtf8(":/images/legende.png")), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.hidelegend_btn.setIcon(icon)
        self.hidelegend_btn.setIconSize(QtCore.QSize(22, 22))
        self.hidelegend_btn.setCheckable(True)
        self.hidelegend_btn.setObjectName(_fromUtf8("hidelegend_btn"))
        self.horizontalLayout.addWidget(self.hidelegend_btn)
        self.option_btn = QtGui.QPushButton(self.dockWidgetContents)
        self.option_btn.setToolTip(QtGui.QApplication.translate("Graph", "Affichage et couleurs du graphique", None, QtGui.QApplication.UnicodeUTF8))
        self.option_btn.setText(QtGui.QApplication.translate("Graph", "Options", None, QtGui.QApplication.UnicodeUTF8))
        self.option_btn.setObjectName(_fromUtf8("option_btn"))
        self.horizontalLayout.addWidget(self.option_btn)
        self.verticalLayout.addLayout(self.horizontalLayout)
        self.mplwidget = MatplotlibWidget(self.dockWidgetContents)
        self.mplwidget.setObjectName(_fromUtf8("mplwidget"))
        self.verticalLayout.addWidget(self.mplwidget)
        Graph.setWidget(self.dockWidgetContents)

        self.retranslateUi(Graph)
        QtCore.QMetaObject.connectSlotsByName(Graph)

    def retranslateUi(self, Graph):
        pass

from matplotlibwidget import MatplotlibWidget
import resources_rc
<commit_msg>problem with matplotlibwidget to fix<commit_after># -*- coding: utf-8 -*-

# Form implementation generated from reading ui file 'ui/graph.ui'
#
# Created: Wed May 16 11:38:13 2012
#      by: PyQt4 UI code generator 4.8.5
#
# WARNING! All changes made in this file will be lost!

from PyQt4 import QtCore, QtGui

try:
    _fromUtf8 = QtCore.QString.fromUtf8
except AttributeError:
    _fromUtf8 = lambda s: s

class Ui_Graph(object):
    def setupUi(self, Graph):
        Graph.setObjectName(_fromUtf8("Graph"))
        Graph.resize(645, 552)
        Graph.setWindowTitle(QtGui.QApplication.translate("Graph", "Graphique", None, QtGui.QApplication.UnicodeUTF8))
        self.dockWidgetContents = QtGui.QWidget()
        self.dockWidgetContents.setObjectName(_fromUtf8("dockWidgetContents"))
        self.verticalLayout = QtGui.QVBoxLayout(self.dockWidgetContents)
        self.verticalLayout.setObjectName(_fromUtf8("verticalLayout"))
        self.horizontalLayout = QtGui.QHBoxLayout()
        self.horizontalLayout.setObjectName(_fromUtf8("horizontalLayout"))
        self.taux_btn = QtGui.QCheckBox(self.dockWidgetContents)
        self.taux_btn.setToolTip(QtGui.QApplication.translate("Graph", "Voir les taux moyens et marginaux d\'imposition", None, QtGui.QApplication.UnicodeUTF8))
        self.taux_btn.setText(QtGui.QApplication.translate("Graph", "Taux moyens\n"
"Taux marginaux", None, QtGui.QApplication.UnicodeUTF8))
        self.taux_btn.setObjectName(_fromUtf8("taux_btn"))
        self.horizontalLayout.addWidget(self.taux_btn)
        self.absBox = QtGui.QComboBox(self.dockWidgetContents)
        self.absBox.setObjectName(_fromUtf8("absBox"))
        self.horizontalLayout.addWidget(self.absBox)
        spacerItem = QtGui.QSpacerItem(40, 20, QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Minimum)
        self.horizontalLayout.addItem(spacerItem)
        self.hidelegend_btn = QtGui.QPushButton(self.dockWidgetContents)
        self.hidelegend_btn.setToolTip(QtGui.QApplication.translate("Graph", "Masquer la légende", None, QtGui.QApplication.UnicodeUTF8))
        self.hidelegend_btn.setText(_fromUtf8(""))
        icon = QtGui.QIcon()
        icon.addPixmap(QtGui.QPixmap(_fromUtf8(":/images/legende.png")), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.hidelegend_btn.setIcon(icon)
        self.hidelegend_btn.setIconSize(QtCore.QSize(22, 22))
        self.hidelegend_btn.setCheckable(True)
        self.hidelegend_btn.setObjectName(_fromUtf8("hidelegend_btn"))
        self.horizontalLayout.addWidget(self.hidelegend_btn)
        self.option_btn = QtGui.QPushButton(self.dockWidgetContents)
        self.option_btn.setToolTip(QtGui.QApplication.translate("Graph", "Affichage et couleurs du graphique", None, QtGui.QApplication.UnicodeUTF8))
        self.option_btn.setText(QtGui.QApplication.translate("Graph", "Options", None, QtGui.QApplication.UnicodeUTF8))
        self.option_btn.setObjectName(_fromUtf8("option_btn"))
        self.horizontalLayout.addWidget(self.option_btn)
        self.verticalLayout.addLayout(self.horizontalLayout)
        self.mplwidget = MatplotlibWidget(self.dockWidgetContents)
        self.mplwidget.setObjectName(_fromUtf8("mplwidget"))
        self.verticalLayout.addWidget(self.mplwidget)
        Graph.setWidget(self.dockWidgetContents)

        self.retranslateUi(Graph)
        QtCore.QMetaObject.connectSlotsByName(Graph)

    def retranslateUi(self, Graph):
        pass

from widgets.matplotlibwidget import MatplotlibWidget
import resources_rc
<|endoftext|>


<commit_before>"""A few things that didn't seem to fit anywhere else."""

import cPickle
import errno
import os
import pwd
import tempfile
import threading

import logger

PID_FILE = '/var/run/node_mgr.pid'

def as_daemon_thread(run):
    """Call function <run> with no arguments in its own thread."""
    thr = threading.Thread(target=run)
    thr.setDaemon(True)
    thr.start()

def close_nonstandard_fds():
    """Close all open file descriptors other than 0, 1, and 2."""
    _SC_OPEN_MAX = 4
    for fd in range(3, os.sysconf(_SC_OPEN_MAX)):
        try: os.close(fd)
        except OSError: pass  # most likely an fd that isn't open

# after http://www.erlenstar.demon.co.uk/unix/faq_2.html
def daemon():
    """Daemonize the current process."""
    if os.fork() != 0: os._exit(0)
    os.setsid()
    if os.fork() != 0: os._exit(0)
    os.chdir('/')
    os.umask(0)
    devnull = os.open(os.devnull, os.O_RDWR)
    for fd in range(3): os.dup2(devnull, fd)

def fork_as(su, function, *args):
    """fork(), cd / to avoid keeping unused directories open, close all nonstandard file descriptors (to avoid capturing open sockets), fork() again (to avoid zombies) and call <function> with arguments <args> in the grandchild process.  If <su> is not None, set our group and user ids appropriately in the child process."""
    child_pid = os.fork()
    if child_pid == 0:
        try:
            os.chdir('/')
            close_nonstandard_fds()
            pw_ent = pwd.getpwnam(su)
            os.setegid(pw_ent[3])
            os.seteuid(pw_ent[2])
            child_pid = os.fork()
            if child_pid == 0: function(*args)
        except:
            os.seteuid(os.getuid())  # undo su so we can write the log file
            os.setegid(os.getgid())
            logger.log_exc()
        os._exit(0)
    else: os.waitpid(child_pid, 0)

def pid_file():
    """We use a pid file to ensure that only one copy of NM is running at a given time.  If successful, this function will write a pid file containing the pid of the current process.  The return value is the pid of the other running process, or None otherwise."""
    other_pid = None
    if os.access(PID_FILE, os.F_OK):  # check for a pid file
        handle = open(PID_FILE)  # pid file exists, read it
        other_pid = int(handle.read())
        handle.close()
        # check for a process with that pid by sending signal 0
        try: os.kill(other_pid, 0)
        except OSError, e:
            if e.errno == errno.ESRCH: other_pid = None  # doesn't exist
            else: raise  # who knows
    if other_pid == None:
        # write a new pid file
        write_file(PID_FILE, lambda f: f.write(str(os.getpid())))
    return other_pid

def write_file(filename, do_write, **kw_args):
    """Write file <filename> atomically by opening a temporary file, using <do_write> to write that file, and then renaming the temporary file."""
    os.rename(write_temp_file(do_write, **kw_args), filename)

def write_temp_file(do_write, mode=None, uidgid=None):
    fd, temporary_filename = tempfile.mkstemp()
    if mode: os.chmod(temporary_filename, mode)
    if uidgid: os.chown(temporary_filename, *uidgid)
    f = os.fdopen(fd, 'w')
    try: do_write(f)
    finally: f.close()
    return temporary_filename
<commit_msg>- write pid to /var/run/nm.pid<commit_after>"""A few things that didn't seem to fit anywhere else."""

import cPickle
import errno
import os
import pwd
import tempfile
import threading

import logger

PID_FILE = '/var/run/nm.pid'

def as_daemon_thread(run):
    """Call function <run> with no arguments in its own thread."""
    thr = threading.Thread(target=run)
    thr.setDaemon(True)
    thr.start()

def close_nonstandard_fds():
    """Close all open file descriptors other than 0, 1, and 2."""
    _SC_OPEN_MAX = 4
    for fd in range(3, os.sysconf(_SC_OPEN_MAX)):
        try: os.close(fd)
        except OSError: pass  # most likely an fd that isn't open

# after http://www.erlenstar.demon.co.uk/unix/faq_2.html
def daemon():
    """Daemonize the current process."""
    if os.fork() != 0: os._exit(0)
    os.setsid()
    if os.fork() != 0: os._exit(0)
    os.chdir('/')
    os.umask(0)
    devnull = os.open(os.devnull, os.O_RDWR)
    for fd in range(3): os.dup2(devnull, fd)

def fork_as(su, function, *args):
    """fork(), cd / to avoid keeping unused directories open, close all nonstandard file descriptors (to avoid capturing open sockets), fork() again (to avoid zombies) and call <function> with arguments <args> in the grandchild process.  If <su> is not None, set our group and user ids appropriately in the child process."""
    child_pid = os.fork()
    if child_pid == 0:
        try:
            os.chdir('/')
            close_nonstandard_fds()
            pw_ent = pwd.getpwnam(su)
            os.setegid(pw_ent[3])
            os.seteuid(pw_ent[2])
            child_pid = os.fork()
            if child_pid == 0: function(*args)
        except:
            os.seteuid(os.getuid())  # undo su so we can write the log file
            os.setegid(os.getgid())
            logger.log_exc()
        os._exit(0)
    else: os.waitpid(child_pid, 0)

def pid_file():
    """We use a pid file to ensure that only one copy of NM is running at a given time.  If successful, this function will write a pid file containing the pid of the current process.  The return value is the pid of the other running process, or None otherwise."""
    other_pid = None
    if os.access(PID_FILE, os.F_OK):  # check for a pid file
        handle = open(PID_FILE)  # pid file exists, read it
        other_pid = int(handle.read())
        handle.close()
        # check for a process with that pid by sending signal 0
        try: os.kill(other_pid, 0)
        except OSError, e:
            if e.errno == errno.ESRCH: other_pid = None  # doesn't exist
            else: raise  # who knows
    if other_pid == None:
        # write a new pid file
        write_file(PID_FILE, lambda f: f.write(str(os.getpid())))
    return other_pid

def write_file(filename, do_write, **kw_args):
    """Write file <filename> atomically by opening a temporary file, using <do_write> to write that file, and then renaming the temporary file."""
    os.rename(write_temp_file(do_write, **kw_args), filename)

def write_temp_file(do_write, mode=None, uidgid=None):
    fd, temporary_filename = tempfile.mkstemp()
    if mode: os.chmod(temporary_filename, mode)
    if uidgid: os.chown(temporary_filename, *uidgid)
    f = os.fdopen(fd, 'w')
    try: do_write(f)
    finally: f.close()
    return temporary_filename
<|endoftext|>


<commit_before>import dataset
import sys
import numpy as np
import keras
import os
import logging
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Reshape, BatchNormalization, Lambda
from keras.layers import Conv2D, MaxPooling2D
from keras import backend as K
from keras.regularizers import l2
from keras.callbacks import ModelCheckpoint, TensorBoard, Callback
from haikunator import Haikunator
from PIL import Image, ImageDraw
from logging import FileHandler
import subprocess

haikunator = Haikunator()

class LandmarkPreview(Callback):
    def __init__(self, out_dir,
                 batch_size=10, validation_gen=None):
        super().__init__()
        self.out_dir = out_dir
        self.batch_size = batch_size
        self.validation_gen = validation_gen
        os.makedirs(out_dir, exist_ok=True)

    def draw_landmarks(self, image, landmarks, r=1, fill_color=(255,0,0,100)):
        draw = ImageDraw.Draw(image)
        for row in landmarks:
            x, y = row
            draw.ellipse((x-r, y-r, x+r, y+r), fill=fill_color)

    def on_epoch_end(self, epoch, logs=None):
        logs = logs or {}
        if self.validation_data:
            data = self.validation_data
        else:
            x, y = next(self.validation_gen)
            data = [x, y, None]

        val_size = data[0].shape[0]
        samples = np.random.choice(np.arange(val_size), self.batch_size)

        for i, x, y_true, y_pred in zip(
                np.arange(val_size),
                data[0][samples],
                data[1][samples],
                self.model.predict(data[0][samples]),
            ):
            img = Image.fromarray(np.squeeze(x, axis=2) * 255.0).convert("RGBA")
            # img.save(os.path.join(self.out_dir, "epoh_%02d_%02d_marker.png" % (epoch, i)))
            self.draw_landmarks(img, y_true, r=1, fill_color=(255,0,0,100))
            self.draw_landmarks(img, y_pred, r=1, fill_color=(0,255,0,100))
            img.save(os.path.join(self.out_dir, "epoh_%02d_%02d_marker.png" % (epoch, i)))

def complex_model(input_shape, l2_reg):
    model = Sequential()
    model.add(Conv2D(32, kernel_size=(5, 5),
                     activation='relu',
                     padding='SAME',
                     kernel_regularizer=l2(l2_reg),
                     input_shape=input_shape))
    model.add(BatchNormalization())
    model.add(Conv2D(
        32,
        kernel_size=(3, 3),
        activation='relu',
        kernel_regularizer=l2(l2_reg),
        padding='SAME',
    ))
    model.add(BatchNormalization())
    model.add(MaxPooling2D())
    model.add(Conv2D(
        64,
        kernel_size=(3, 3),
        activation='relu',
        kernel_regularizer=l2(l2_reg),
        padding='SAME',
    ))
    model.add(BatchNormalization())
    model.add(MaxPooling2D())
    model.add(Conv2D(
        128,
        kernel_size=(3, 3),
        activation='relu',
        kernel_regularizer=l2(l2_reg),
        padding='SAME',
    ))
    model.add(BatchNormalization())
    model.add(MaxPooling2D())
    model.add(Flatten())
    model.add(Dropout(0.5))
    model.add(Dense(
        32,
        kernel_regularizer=l2(l2_reg),
    ))
    model.add(Dense(
        128,
        activation='relu',
        kernel_regularizer=l2(l2_reg),
    ))
    model.add(BatchNormalization())
    model.add(Dense(
        66 * 2,
        activation='relu'
    ))
    model.add(Lambda(lambda x: x * 0.5 * input_shape[0]))
    model.add(Reshape((66, 2)))
    model.compile(loss=keras.losses.mean_squared_error,
                  optimizer='adam',
    )
    return model

def simple_model(input_shape):
    model = Sequential()
    model.add(Conv2D(32, kernel_size=(3, 3),
                     activation='relu',
                     padding='SAME',
                     input_shape=input_shape))
    model.add(BatchNormalization())
    model.add(MaxPooling2D())
    model.add(Conv2D(64, kernel_size=(3, 3),
                     activation='relu',
                     padding='SAME',
    ))
    model.add(MaxPooling2D())
    model.add(MaxPooling2D())
    model.add(Flatten())
    model.add(Dropout(0.5))
    model.add(Dense(66 * 2, activation='relu'))
    model.add(Lambda(lambda x: x * input_shape[0]))
    model.add(Reshape((66, 2)))
    model.compile(loss=keras.losses.mean_squared_error,
                  optimizer='adam',
    )
    return model

if __name__ == "__main__":
    log_fmt = '[%(levelname)s] %(name)s: %(message)s'
    logging.basicConfig(level=logging.INFO, format=log_fmt)

    name = haikunator.haikunate()
    logger = logging.getLogger(name)
    os.makedirs(os.path.join('.', 'logs', name), exist_ok=True)
    fh = FileHandler(os.path.join('.', 'logs', name, 'log.txt'))
    fh.setLevel(logging.INFO)
    logger.addHandler(fh)

    git_hash = subprocess.check_output(["git", "rev-parse", "HEAD"])
    logger.info("Git commit status %s", git_hash)
    logger.info("Started data loading.")
    dataset = dataset.Pain(sys.argv[1], picture_size=128)
    # model = complex_model((128, 128, 1), 1e-3)
    model = simple_model((128, 128, 1))
    logger.info("Model summary\n%s", model.to_json(indent=4))
    model.summary(print_fn=lambda x: logger.info(str(x)))

    checkpointer = ModelCheckpoint(
             os.path.join('.', 'logs', name, 'checkpoint'),
             verbose=0,
             save_best_only=True
    )
    tensorboard = TensorBoard(
        os.path.join('.', 'logs', name),
        # write_grads=True,
        # histogram_freq=3
    )
    landmarks = LandmarkPreview(
        os.path.join('.', 'logs', name, 'pics'),
        validation_gen = dataset.test_generator(10)
    )

    model.fit_generator(
        generator=dataset.train_generator(32),
        steps_per_epoch=3000,
        epochs=30,
        verbose=1,
        validation_data=dataset.test_generator(10),
        validation_steps=1,
        callbacks = [checkpointer, tensorboard, landmarks],
        # max_queue_size = 100,
        use_multiprocessing=True
    )
    logger.info("Model training finished!")
    model.save(os.path.join('.', 'models', name + "_model.h5"))
    logger.info("Model saved")
<commit_msg>Enlarged debug imgs<commit_after>import dataset
import sys
import numpy as np
import keras
import os
import logging
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Reshape, BatchNormalization, Lambda
from keras.layers import Conv2D, MaxPooling2D
from keras import backend as K
from keras.regularizers import l2
from keras.callbacks import ModelCheckpoint, TensorBoard, Callback
from haikunator import Haikunator
from PIL import Image, ImageDraw
from logging import FileHandler
import subprocess

haikunator = Haikunator()

class LandmarkPreview(Callback):
    def __init__(self, out_dir,
                 batch_size=10, validation_gen=None):
        super().__init__()
        self.out_dir = out_dir
        self.batch_size = batch_size
        self.validation_gen = validation_gen
        os.makedirs(out_dir, exist_ok=True)

    def draw_landmarks(self, image, landmarks, r=1, fill_color=(255,0,0,100)):
        draw = ImageDraw.Draw(image)
        for row in landmarks:
            x, y = row
            draw.ellipse((x-r, y-r, x+r, y+r), fill=fill_color)

    def on_epoch_end(self, epoch, logs=None):
        logs = logs or {}
        if self.validation_data:
            data = self.validation_data
        else:
            x, y = next(self.validation_gen)
            data = [x, y, None]

        val_size = data[0].shape[0]
        samples = np.random.choice(np.arange(val_size), self.batch_size)

        for i, x, y_true, y_pred in zip(
                np.arange(val_size),
                data[0][samples],
                data[1][samples],
                self.model.predict(data[0][samples]),
            ):
            img = Image.fromarray(np.squeeze(x, axis=2) * 255.0).convert("RGBA")
            w, h = img.size
            img = img.resize((2*w, 2*h), resample=Image.BICUBIC)
            # img.save(os.path.join(self.out_dir, "epoh_%02d_%02d_marker.png" % (epoch, i)))
            self.draw_landmarks(img, 2 * y_true, r=1, fill_color=(255,0,0,100))
            self.draw_landmarks(img, 2 * y_pred, r=1, fill_color=(0,255,0,100))
            img.save(os.path.join(self.out_dir, "epoh_%02d_%02d_marker.png" % (epoch, i)))

def complex_model(input_shape, l2_reg):
    model = Sequential()
    model.add(Conv2D(32, kernel_size=(5, 5),
                     activation='relu',
                     padding='SAME',
                     kernel_regularizer=l2(l2_reg),
                     input_shape=input_shape))
    model.add(BatchNormalization())
    model.add(Conv2D(
        32,
        kernel_size=(3, 3),
        activation='relu',
        kernel_regularizer=l2(l2_reg),
        padding='SAME',
    ))
    model.add(BatchNormalization())
    model.add(MaxPooling2D())
    model.add(Conv2D(
        64,
        kernel_size=(3, 3),
        activation='relu',
        kernel_regularizer=l2(l2_reg),
        padding='SAME',
    ))
    model.add(BatchNormalization())
    model.add(MaxPooling2D())
    model.add(Conv2D(
        128,
        kernel_size=(3, 3),
        activation='relu',
        kernel_regularizer=l2(l2_reg),
        padding='SAME',
    ))
    model.add(BatchNormalization())
    model.add(MaxPooling2D())
    model.add(Flatten())
    model.add(Dropout(0.5))
    model.add(Dense(
        32,
        kernel_regularizer=l2(l2_reg),
    ))
    model.add(Dense(
        128,
        activation='relu',
        kernel_regularizer=l2(l2_reg),
    ))
    model.add(BatchNormalization())
    model.add(Dense(
        66 * 2,
        activation='relu'
    ))
    model.add(Lambda(lambda x: x * 0.5 * input_shape[0]))
    model.add(Reshape((66, 2)))
    model.compile(loss=keras.losses.mean_squared_error,
                  optimizer='adam',
    )
    return model

def simple_model(input_shape):
    model = Sequential()
    model.add(Conv2D(32, kernel_size=(3, 3),
                     activation='relu',
                     padding='SAME',
                     input_shape=input_shape))
    model.add(BatchNormalization())
    model.add(MaxPooling2D())
    model.add(Conv2D(64, kernel_size=(3, 3),
                     activation='relu',
                     padding='SAME',
    ))
    model.add(MaxPooling2D())
    model.add(MaxPooling2D())
    model.add(Flatten())
    model.add(Dropout(0.5))
    model.add(Dense(66 * 2, activation='relu'))
    model.add(Lambda(lambda x: x * input_shape[0]))
    model.add(Reshape((66, 2)))
    model.compile(loss=keras.losses.mean_squared_error,
                  optimizer='adam',
    )
    return model

if __name__ == "__main__":
    log_fmt = '[%(levelname)s] %(name)s: %(message)s'
    logging.basicConfig(level=logging.INFO, format=log_fmt)

    name = haikunator.haikunate()
    logger = logging.getLogger(name)
    os.makedirs(os.path.join('.', 'logs', name), exist_ok=True)
    fh = FileHandler(os.path.join('.', 'logs', name, 'log.txt'))
    fh.setLevel(logging.INFO)
    logger.addHandler(fh)

    git_hash = subprocess.check_output(["git", "rev-parse", "HEAD"])
    logger.info("Git commit status %s", git_hash)
    logger.info("Started data loading.")
    dataset = dataset.Pain(sys.argv[1], picture_size=128)
    # model = complex_model((128, 128, 1), 1e-3)
    model = simple_model((128, 128, 1))
    logger.info("Model summary\n%s", model.to_json(indent=4))
    model.summary(print_fn=lambda x: logger.info(str(x)))

    checkpointer = ModelCheckpoint(
             os.path.join('.', 'logs', name, 'checkpoint'),
             verbose=0,
             save_best_only=True
    )
    tensorboard = TensorBoard(
        os.path.join('.', 'logs', name),
        # write_grads=True,
        # histogram_freq=3
    )
    landmarks = LandmarkPreview(
        os.path.join('.', 'logs', name, 'pics'),
        validation_gen = dataset.test_generator(10)
    )

    model.fit_generator(
        generator=dataset.train_generator(32),
        steps_per_epoch=3000,
        epochs=30,
        verbose=1,
        validation_data=dataset.test_generator(10),
        validation_steps=1,
        callbacks = [checkpointer, tensorboard, landmarks],
        # max_queue_size = 100,
        use_multiprocessing=True
    )
    logger.info("Model training finished!")
    model.save(os.path.join('.', 'models', name + "_model.h5"))
    logger.info("Model saved")
<|endoftext|>


<commit_before>#!/usr/bin/env python

from __future__ import print_function

import pickle
import json
import numpy as np
import glob
import os
import neukrill_net.utils as utils
import neukrill_net.image_processing as image_processing
import neukrill_net.augment as augment

import sklearn.preprocessing
import sklearn.ensemble
import sklearn.linear_model
import sklearn.cross_validation
import sklearn.dummy
from sklearn.externals import joblib
import sklearn.metrics
import sklearn.pipeline
import argparse

def main(run_settings_path):
    settings = utils.Settings('settings.json')
    with open(run_settings_path) as rf:
        run_settings = json.load(rf)
    # shoehorn run_settings filename into its own dictionary (for later)
    run_settings['filename'] = os.path.split(
                                        run_settings_path)[-1].split(".")[0]
    # and the full run settings path
    run_settings['run_settings_path'] = os.path.abspath(run_settings_path)
    # also put the settings in there
    run_settings['settings'] = settings
    # and the settings path, while we're at it
    run_settings['settings_path'] = os.path.abspath('settings.json')
    if run_settings['model type'] == 'sklearn':
        train_sklearn(run_settings)
    elif run_settings['model type'] == 'pylearn2':
        train_pylearn2(run_settings)
    else:
        raise NotImplementedError("Unsupported model type.")

def train_sklearn(run_settings):
    # unpack settings
    settings = run_settings['settings']

    # get all training file paths and class names
    image_fname_dict = settings.image_fnames

    # this should be parsed from json, but hardcoded for now
    augment_settings = run_settings["preprocessing"]

    # build processing function
    processing = augment.augmentation_wrapper(augment_settings)

    # load data as design matrix, applying processing function
    X, y = utils.load_data(image_fname_dict, classes=settings.classes,
                           processing=processing)

    # make a label encoder and encode the labels
    label_encoder = sklearn.preprocessing.LabelEncoder()
    y = label_encoder.fit_transform(y)

    if run_settings['classifier'] == 'dummy':
        # just a dummy uniform probability classifier for working purposes
        clf = sklearn.dummy.DummyClassifier(strategy='uniform')
    elif run_settings['classifier'] == 'logistic regression':
        clf = sklearn.linear_model.SGDClassifier(n_jobs=-1,
                                                 loss='log')
    elif run_settings['classifier'] == 'random forest':
        forest = sklearn.ensemble.RandomForestClassifier(n_jobs=-1,
                                                  n_estimators=100,
        #                                          verbose=1,
                                                  max_depth=5)
        scaler = sklearn.preprocessing.StandardScaler()
        clf = sklearn.pipeline.Pipeline((("scl",scaler),("clf",forest)))

    # only supporting stratified shuffle split for now
    cv = sklearn.cross_validation.StratifiedShuffleSplit(y,
                                    **run_settings['cross validation'])

    results = []
    for train, test in cv:
        clf.fit(X[train], y[train])
        p = clf.predict_proba(X[test])
        results.append(sklearn.metrics.log_loss(y[test], p))

    print("Average CV: {0} +/- {1}".format(np.mean(results),
                                    np.sqrt(np.var(results))))

    # before saving model check there is somewhere for it to save to
    modeldir = os.path.join(settings.data_dir,"models")
    if not os.path.exists(modeldir):
        os.mkdir(modeldir)
    # save the model in the data directory, in a "models" subdirectory
    # with the name of the run_settings as the name of the pkl
    picklefname = os.path.join(modeldir,run_settings['filename']+".pkl")
    joblib.dump(clf, picklefname, compress=3)

def train_pylearn2(run_settings):
    """
    Function to call operations for running a pylearn2 model using
    the settings found in run_settings.
    """
    import pylearn2.config
    # unpack settings
    settings = run_settings['settings']
    #read the YAML settings template
    with open(os.path.join("yaml_templates",run_settings['yaml file'])) as y:
        yaml_string = y.read()
    # sub in the following things for default: settings_path, run_settings_path,
    # final_shape, n_classes, save_path
    run_settings["n_classes"] = len(settings.classes)
    modeldir = os.path.join(settings.data_dir,"models")
    if not os.path.exists(modeldir):
        os.mkdir(modeldir)
    run_settings["save_path"] = os.path.join(modeldir,
            run_settings['filename'] + ".pkl")
    # time for some crude string parsing
    yaml_string = yaml_string%(run_settings)
    # write the new yaml to the data directory, in a yaml_settings subdir
    yamldir = os.path.join(settings.data_dir,"yaml_settings")
    if not os.path.exists(yamldir):
        os.mkdir(yamldir)
    yaml_path = os.path.join(yamldir,run_settings["filename"]+
            run_settings['yaml file'].split(".")[0]+".yaml")
    with open(yaml_path, "w") as f:
        f.write(yaml_string)
    # then we load the yaml file using pylearn2
    train = pylearn2.config.yaml_parse(yaml_path)
    # and run the model!
    train.main_loop()

if __name__=='__main__':
    # need to argparse for run settings path
    parser = argparse.ArgumentParser(description='Train a model and store a'
                                                 'pickled model file.')
    # nargs='?' will look for a single argument but failover to default
    parser.add_argument('run_settings', metavar='run_settings', type=str,
            nargs='?', default=os.path.join("run_settings","default.json"),
            help="Path to run settings json file.")
    args = parser.parse_args()
    main(args.run_settings)
<commit_msg>called module instead of method<commit_after>#!/usr/bin/env python

from __future__ import print_function

import pickle
import json
import numpy as np
import glob
import os
import neukrill_net.utils as utils
import neukrill_net.image_processing as image_processing
import neukrill_net.augment as augment

import sklearn.preprocessing
import sklearn.ensemble
import sklearn.linear_model
import sklearn.cross_validation
import sklearn.dummy
from sklearn.externals import joblib
import sklearn.metrics
import sklearn.pipeline
import argparse

def main(run_settings_path):
    settings = utils.Settings('settings.json')
    with open(run_settings_path) as rf:
        run_settings = json.load(rf)
    # shoehorn run_settings filename into its own dictionary (for later)
    run_settings['filename'] = os.path.split(
                                        run_settings_path)[-1].split(".")[0]
    # and the full run settings path
    run_settings['run_settings_path'] = os.path.abspath(run_settings_path)
    # also put the settings in there
    run_settings['settings'] = settings
    # and the settings path, while we're at it
    run_settings['settings_path'] = os.path.abspath('settings.json')
    if run_settings['model type'] == 'sklearn':
        train_sklearn(run_settings)
    elif run_settings['model type'] == 'pylearn2':
        train_pylearn2(run_settings)
    else:
        raise NotImplementedError("Unsupported model type.")

def train_sklearn(run_settings):
    # unpack settings
    settings = run_settings['settings']

    # get all training file paths and class names
    image_fname_dict = settings.image_fnames

    # this should be parsed from json, but hardcoded for now
    augment_settings = run_settings["preprocessing"]

    # build processing function
    processing = augment.augmentation_wrapper(augment_settings)

    # load data as design matrix, applying processing function
    X, y = utils.load_data(image_fname_dict, classes=settings.classes,
                           processing=processing)

    # make a label encoder and encode the labels
    label_encoder = sklearn.preprocessing.LabelEncoder()
    y = label_encoder.fit_transform(y)

    if run_settings['classifier'] == 'dummy':
        # just a dummy uniform probability classifier for working purposes
        clf = sklearn.dummy.DummyClassifier(strategy='uniform')
    elif run_settings['classifier'] == 'logistic regression':
        clf = sklearn.linear_model.SGDClassifier(n_jobs=-1,
                                                 loss='log')
    elif run_settings['classifier'] == 'random forest':
        forest = sklearn.ensemble.RandomForestClassifier(n_jobs=-1,
                                                  n_estimators=100,
        #                                          verbose=1,
                                                  max_depth=5)
        scaler = sklearn.preprocessing.StandardScaler()
        clf = sklearn.pipeline.Pipeline((("scl",scaler),("clf",forest)))

    # only supporting stratified shuffle split for now
    cv = sklearn.cross_validation.StratifiedShuffleSplit(y,
                                    **run_settings['cross validation'])

    results = []
    for train, test in cv:
        clf.fit(X[train], y[train])
        p = clf.predict_proba(X[test])
        results.append(sklearn.metrics.log_loss(y[test], p))

    print("Average CV: {0} +/- {1}".format(np.mean(results),
                                    np.sqrt(np.var(results))))

    # before saving model check there is somewhere for it to save to
    modeldir = os.path.join(settings.data_dir,"models")
    if not os.path.exists(modeldir):
        os.mkdir(modeldir)
    # save the model in the data directory, in a "models" subdirectory
    # with the name of the run_settings as the name of the pkl
    picklefname = os.path.join(modeldir,run_settings['filename']+".pkl")
    joblib.dump(clf, picklefname, compress=3)

def train_pylearn2(run_settings):
    """
    Function to call operations for running a pylearn2 model using
    the settings found in run_settings.
    """
    import pylearn2.config
    # unpack settings
    settings = run_settings['settings']
    #read the YAML settings template
    with open(os.path.join("yaml_templates",run_settings['yaml file'])) as y:
        yaml_string = y.read()
    # sub in the following things for default: settings_path, run_settings_path,
    # final_shape, n_classes, save_path
    run_settings["n_classes"] = len(settings.classes)
    modeldir = os.path.join(settings.data_dir,"models")
    if not os.path.exists(modeldir):
        os.mkdir(modeldir)
    run_settings["save_path"] = os.path.join(modeldir,
            run_settings['filename'] + ".pkl")
    # time for some crude string parsing
    yaml_string = yaml_string%(run_settings)
    # write the new yaml to the data directory, in a yaml_settings subdir
    yamldir = os.path.join(settings.data_dir,"yaml_settings")
    if not os.path.exists(yamldir):
        os.mkdir(yamldir)
    yaml_path = os.path.join(yamldir,run_settings["filename"]+
            run_settings['yaml file'].split(".")[0]+".yaml")
    with open(yaml_path, "w") as f:
        f.write(yaml_string)
    # then we load the yaml file using pylearn2
    train = pylearn2.config.yaml_parse.load(yaml_path)
    # and run the model!
    train.main_loop()

if __name__=='__main__':
    # need to argparse for run settings path
    parser = argparse.ArgumentParser(description='Train a model and store a'
                                                 'pickled model file.')
    # nargs='?' will look for a single argument but failover to default
    parser.add_argument('run_settings', metavar='run_settings', type=str,
            nargs='?', default=os.path.join("run_settings","default.json"),
            help="Path to run settings json file.")
    args = parser.parse_args()
    main(args.run_settings)
<|endoftext|>


<commit_before>import matplotlib as mpl
mpl.use('Agg')
import argparse
import sys
import cProfile

from src.experiment import train
from src.experiment import reset_unused_args
from src.misc.constants import *

''' ARGUMENTS '''
'''----------------------------------------------------------------------- '''
parser = argparse.ArgumentParser(description='Jets')
'''
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
'''
# Debugging
admin = parser.add_argument_group('admin')
admin.add_argument("-d", "--debug", help="sets everything small for fast model debugging. use in combination with ipdb", action='store_true', default=False)
admin.add_argument("--profile", action='store_true', default=False)

# Admin args
admin.add_argument("-s", "--silent", action='store_true', default=False)
admin.add_argument("-v", "--verbose", action='store_true', default=False)
admin.add_argument("--visualizing", action='store_true', default=False)
admin.add_argument("--no_email", action='store_true', default=False)

# Directory args
admin.add_argument("--data_dir", type=str, default=DATA_DIR)
admin.add_argument("--models_dir", type=str, default=MODELS_DIR)
admin.add_argument("--root_dir", default=MODELS_DIR)

# Slurm args
admin.add_argument("--slurm", action='store_true', default=False)
admin.add_argument("--slurm_array_job_id", default=0)
admin.add_argument("--slurm_array_task_id", default=0)
'''
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
'''
# Loading previous models args
loading = parser.add_argument_group('loading')
loading.add_argument("-l", "--load", help="model directory from which we load a state_dict", type=str, default=None)
loading.add_argument("-r", "--restart", help="restart a loaded model from where it left off", action='store_true', default=False)

'''
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
'''
# Training args
training = parser.add_argument_group('training')
training.add_argument("-e", "--epochs", type=int, default=64)
training.add_argument("-b", "--batch_size", type=int, default=128)
training.add_argument("--experiment_time", type=int, default=1000000)

'''
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
'''
# Optimization args
optim = parser.add_argument_group('optim')
optim.add_argument("--lr", type=float, default=.001)
optim.add_argument("--lr_min", type=float, default=.0000005)
optim.add_argument("--decay", type=float, default=.7)
optim.add_argument("--clip", type=float, default=1.0)
optim.add_argument("--reg", type=float, default=.00001)
optim.add_argument("--sched", type=str, default='m1')
optim.add_argument("--period", type=int, default=8)
optim.add_argument("--momentum", type=float, default=0.0)
optim.add_argument("--optim", type=str, default='adam')

'''
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
'''
# computing args

computing = parser.add_argument_group('computing')
computing.add_argument("--seed", help="Random seed used in torch and numpy", type=int, default=None)
computing.add_argument("-g", "--gpu", type=str, default="")

'''
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
'''
# Data args

data = parser.add_argument_group('data')
data.add_argument("-n", "--n_train", type=int, default=-1)
data.add_argument("--n_valid", type=int, default=10000)
data.add_argument("--dataset", type=str, default='w')
data.add_argument("--data_dropout", type=float, default=.99)
data.add_argument("--pp", action='store_true', default=False)
data.add_argument("--permute_particles", action='store_true')

'''
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
'''
# Dimension and activation args

model = parser.add_argument_group('model')
model.add_argument("--features", type=int, default=7)
model.add_argument("--hidden", type=int, default=64)
model.add_argument("--act", type=str, default='leakyrelu')
model.add_argument("--model_dropout", type=float, default=1.)

# Classifier
model.add_argument("--predict", type=str, default='simple', help='type of prediction layer')

# Transform
model.add_argument("-m", "--model", type=str, default="nmp", help="name of the model you want to train")

# NMP
model.add_argument("-i", "--iters", type=int, default=10)
model.add_argument("--mp", type=str, default='simple', help='type of message passing layer')
model.add_argument("-u", "--update", type=str, default='gru', help='type of vertex update')
model.add_argument("--message", type=str, default='2', help='type of message')
model.add_argument("--emb_init", type=str, default='1', help='type of message')
model.add_argument("-a","--adj", type=str, nargs='+', default='dm', help='type of matrix layer')
model.add_argument("--asym", action='store_true', default=False)
model.add_argument("--readout", type=str, default='dtnn', help='type of readout layer')
model.add_argument("--m_act", type=str, default='soft', help='type of nonlinearity for matrices' )
model.add_argument("--lf", type=int, default=20)
model.add_argument("--wn", action='store_true')

# Stack NMP
model.add_argument("--pool_first", action='store_true', default=False)
model.add_argument("--scales", nargs='+', type=int, default=None)
model.add_argument("--pool", type=str, default='attn', help='type of pooling layer')

# Physics NMP
model.add_argument("-t", "--trainable_physics", action='store_true', default=False)
model.add_argument("--alpha", type=float, default=1)
model.add_argument("-R", type=float, default=1)

# Physics plus learned NMP
model.add_argument("--equal_weight", action='store_true', default=False)

# Transformer
model.add_argument("--n_layers", type=int, default=3)
model.add_argument("--n_heads", type=int, default=8)
model.add_argument("--dq", type=int, default=32)
model.add_argument("--dv", type=int, default=32)

args = parser.parse_args()

arg_groups={}

for group in parser._action_groups:
    group_dict={a.dest:getattr(args,a.dest,None) for a in group._group_actions}
    arg_groups[group.title + '_args']=argparse.Namespace(**group_dict)

os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
arg_groups['admin_args'].cmd_line_args = (' '.join(sys.argv))

if __name__ == "__main__":
    if args.profile:
        cProfile.run('train(args)')
    else:
        train(**arg_groups)
<commit_msg>changes to default argparse<commit_after>import matplotlib as mpl
mpl.use('Agg')
import argparse
import sys
import cProfile

from src.experiment import train
from src.experiment import reset_unused_args
from src.misc.constants import *

''' ARGUMENTS '''
'''----------------------------------------------------------------------- '''
parser = argparse.ArgumentParser(description='Jets')
'''
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
'''
# Debugging
admin = parser.add_argument_group('admin')
admin.add_argument("-d", "--debug", help="sets everything small for fast model debugging. use in combination with ipdb", action='store_true', default=False)
admin.add_argument("--profile", action='store_true', default=False)

# Admin args
admin.add_argument("-s", "--silent", action='store_true', default=False)
admin.add_argument("-v", "--verbose", action='store_true', default=False)
admin.add_argument("--visualizing", action='store_true', default=False)
admin.add_argument("--no_email", action='store_true', default=False)

# Directory args
admin.add_argument("--data_dir", type=str, default=DATA_DIR)
admin.add_argument("--models_dir", type=str, default=MODELS_DIR)
admin.add_argument("--root_dir", default=MODELS_DIR)

# Slurm args
admin.add_argument("--slurm", action='store_true', default=False)
admin.add_argument("--slurm_array_job_id", default=0)
admin.add_argument("--slurm_array_task_id", default=0)
'''
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
'''
# Loading previous models args
loading = parser.add_argument_group('loading')
loading.add_argument("-l", "--load", help="model directory from which we load a state_dict", type=str, default=None)
loading.add_argument("-r", "--restart", help="restart a loaded model from where it left off", action='store_true', default=False)

'''
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
'''
# Training args
training = parser.add_argument_group('training')
training.add_argument("-e", "--epochs", type=int, default=64)
training.add_argument("-b", "--batch_size", type=int, default=128)
training.add_argument("--experiment_time", type=int, default=1000000)

'''
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
'''
# Optimization args
optim = parser.add_argument_group('optim')
optim.add_argument("--lr", type=float, default=.001)
optim.add_argument("--lr_min", type=float, default=.0000005)
optim.add_argument("--decay", type=float, default=.7)
optim.add_argument("--clip", type=float, default=1.0)
optim.add_argument("--reg", type=float, default=.00001)
optim.add_argument("--sched", type=str, default='m1')
optim.add_argument("--period", type=int, default=16)
optim.add_argument("--momentum", type=float, default=0.0)
optim.add_argument("--optim", type=str, default='adam')

'''
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
'''
# computing args

computing = parser.add_argument_group('computing')
computing.add_argument("--seed", help="Random seed used in torch and numpy", type=int, default=None)
computing.add_argument("-g", "--gpu", type=str, default="")

'''
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
'''
# Data args

data = parser.add_argument_group('data')
data.add_argument("-n", "--n_train", type=int, default=-1)
data.add_argument("--n_valid", type=int, default=10000)
data.add_argument("--dataset", type=str, default='w')
data.add_argument("--data_dropout", type=float, default=.99)
data.add_argument("--pp", action='store_true', default=False)
data.add_argument("--permute_particles", action='store_true')

'''
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
'''
# Dimension and activation args

model = parser.add_argument_group('model')
model.add_argument("--features", type=int, default=7)
model.add_argument("--hidden", type=int, default=64)
model.add_argument("--act", type=str, default='leakyrelu')
model.add_argument("--model_dropout", type=float, default=1.)

# Classifier
model.add_argument("--predict", type=str, default='simple', help='type of prediction layer')

# Transform
model.add_argument("-m", "--model", type=str, default="nmp", help="name of the model you want to train")

# NMP
model.add_argument("-i", "--iters", type=int, default=10)
model.add_argument("--mp", type=str, default='simple', help='type of message passing layer')
model.add_argument("-u", "--update", type=str, default='gru', help='type of vertex update')
model.add_argument("--message", type=str, default='2', help='type of message')
model.add_argument("--emb_init", type=str, default='1', help='type of message')
model.add_argument("-a","--adj", type=str, nargs='+', default='phy', help='type of matrix layer')
model.add_argument("--asym", action='store_true', default=False)
model.add_argument("--readout", type=str, default='dtnn', help='type of readout layer')
model.add_argument("--m_act", type=str, default='soft', help='type of nonlinearity for matrices' )
model.add_argument("--lf", type=int, default=20)
model.add_argument("--wn", action='store_true')

# Stack NMP
model.add_argument("--pool_first", action='store_true', default=False)
model.add_argument("--scales", nargs='+', type=int, default=None)
model.add_argument("--pool", type=str, default='attn', help='type of pooling layer')

# Physics NMP
model.add_argument("-t", "--trainable_physics", action='store_true', default=False)
model.add_argument("--alpha", type=float, default=1)
model.add_argument("-R", type=float, default=1)

# Physics plus learned NMP
model.add_argument("--equal_weight", action='store_true', default=False)

# Transformer
model.add_argument("--n_layers", type=int, default=3)
model.add_argument("--n_heads", type=int, default=8)
model.add_argument("--dq", type=int, default=32)
model.add_argument("--dv", type=int, default=32)

args = parser.parse_args()

arg_groups={}

for group in parser._action_groups:
    group_dict={a.dest:getattr(args,a.dest,None) for a in group._group_actions}
    arg_groups[group.title + '_args']=argparse.Namespace(**group_dict)

os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
arg_groups['admin_args'].cmd_line_args = (' '.join(sys.argv))

if __name__ == "__main__":
    if args.profile:
        cProfile.run('train(args)')
    else:
        train(**arg_groups)
<|endoftext|>


<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Lint as: python3
"""Trains a ldif network locally."""

# Commands required in-between imports to silence tensorflow
# pylint: disable=g-import-not-at-top
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

import sys
import time

from absl import app
from absl import flags

import numpy as np

import tensorflow as tf
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)

# LDIF is an internal package, should be imported last.
# pylint: disable=g-bad-import-order
from ldif.datasets import local_inputs
from ldif.util import gaps_util
from ldif.inference import example
from ldif.model import hparams
from ldif.inference import experiment as experiments
from ldif.training import shared_launcher
from ldif.util import file_util
from ldif.util import gpu_util
from ldif.util import path_util
from ldif.util.file_util import log
# pylint: enable=g-bad-import-order
# pylint: enable=g-import-not-at-top

FLAGS = flags.FLAGS

flags.DEFINE_integer('batch_size', 1, 'The batch size to use when training.')

flags.DEFINE_integer(
    'summary_step_interval', 10,
    'Summaries are written on step indices divisible by this number.')

flags.DEFINE_integer(
    'checkpoint_interval', 250,
    'Model checkpoints are written at the end of training, and at step'
    ' indices divisible by this number. Here written does not mean permanently'
    ' saved, due to how tf.train.Saver works. There is a maximum number of'
    ' checkpoints written before old checkpoints are erased, and a permanent'
    ' save frequency.')

flags.DEFINE_string(
    'experiment_name', 'reproduce-ldif',
    'The name of the experiment. It will be saved to disk under this name,'
    ' and can be reloaded for inference, eval, interactive notebooks, or'
    ' further training by providing this experiment_name. To resume training'
    ' from an already partially trained model, simply provide an experiment'
    ' name that is already written. Note that global step will be reloaded.')

flags.DEFINE_integer(
    'train_step_count', 1000000,
    'The number of training steps to take before training is complete.')

flags.DEFINE_string(
    'model_directory', 'trained_models/',
    'The path to the trained model root directory. Can be'
    ' absolute or relative to the LDIF repository root.')

flags.DEFINE_string(
    'dataset_directory', '',
    'The path to the dataset generated by meshes2dataset.py'
    ' Must be provided. The train split will be used.')

flags.DEFINE_string('split', 'train', 'The split to train on.')

flags.DEFINE_boolean(
    'visualize', False, 'If true, visualizes the training'
    ' data before training. Useful for debugging data errors.')

flags.DEFINE_string('model_type', 'ldif', 'The type of model to train. Can be'
                    'ldif (the LDIF paper model), sif (the SIF paper)'
                    ', or sif++ (an improved version of SIF using the LDIF'
                    ' architecture changes)')

flags.DEFINE_string('log_level', 'INFO',
                    'One of VERBOSE, INFO, WARNING, ERROR. Sets logs to print '
                    'only at or above the specified level.')

flags.DEFINE_boolean('reserve_memory_for_inference_kernel', True,
                     'Normally TensorFlow preallocates the entire GPU\'s memory'
                     ' when the session is created. The inference CUDA'
                     ' executable requires some memory to run. This flag stops'
                     ' TensorFlow from allocating everything, leaving ~1GB'
                     ' for the kernel (enough for 512^3 reconstruction).'
                     ' Automatically disabled on MacOS.')

def build_model_config(dataset):
  """Creates the ModelConfig object, which contains model hyperparameters."""
  # TODO(kgenova) This needs to somehow at least support LDIF/SIF/SingleView.
  # TODO(kgenova) Add support for eval/inference.
  builder_fun_dict = {
      'ldif': hparams.build_ldif_hparams,
      'sif': hparams.build_sif_hparams,
      'sif++': hparams.build_improved_sif_hparams
  }
  model_config = experiments.ModelConfig(builder_fun_dict[FLAGS.model_type]())
  model_config.hparams.bs = FLAGS.batch_size
  model_config.train = True
  model_config.eval = False
  model_config.inference = False
  model_config.inputs['dataset'] = dataset
  model_config.inputs['split'] = FLAGS.split
  model_config.inputs['proto'] = 'ShapeNetNSSDodecaSparseLRGMediumSlimPC'
  # This function is defined by the library; we don't need it, but
  # the launcher may call it.
  model_config.wrap_optimizer = lambda x: x
  return model_config

def visualize_data(session, dataset):
  """Visualizes the dataset with two interactive visualizer windows."""
  (bounding_box_samples, depth_renders, mesh_name, near_surface_samples, grid,
   world2grid, surface_point_samples) = session.run([
       dataset.bounding_box_samples, dataset.depth_renders, dataset.mesh_name,
       dataset.near_surface_samples, dataset.grid, dataset.world2grid,
       dataset.surface_point_samples
   ])
  gaps_util.ptsview(
      [bounding_box_samples, near_surface_samples, surface_point_samples])
  mesh_name = mesh_name.decode(sys.getdefaultencoding())
  log.info(f'depth max: {np.max(depth_renders)}')
  log.info(f'Mesh name: {mesh_name}')
  assert '|' in mesh_name
  mesh_hash = mesh_name[mesh_name.find('|') + 1:]
  log.info(f'Mesh hash: {mesh_hash}')
  dyn_obj = example.InferenceExample(FLAGS.split, 'airplane', mesh_hash)

  gaps_util.gapsview(
      msh=dyn_obj.normalized_gt_mesh,
      pts=near_surface_samples[:, :3],
      grd=grid,
      world2grid=world2grid,
      grid_threshold=-0.07)

def get_model_root():
  """Finds the path to the trained model's root directory based on flags."""
  ldif_abspath = path_util.get_path_to_ldif_root()
  model_dir_is_relative = FLAGS.model_directory[0] != '/'
  if model_dir_is_relative:
    model_dir_path = os.path.join(ldif_abspath, FLAGS.model_directory)
  else:
    model_dir_path = FLAGS.model_directory
  if not os.path.isdir(model_dir_path):
    os.makedirs(model_dir_path)
  return model_dir_path

def main(argv):
  if len(argv) > 1:
    raise app.UsageError('Too many command-line arguments.')
  tf.disable_v2_behavior()
  log.set_level(FLAGS.log_level)

  log.info('Making dataset...')
  if not FLAGS.dataset_directory:
    raise ValueError('A dataset directory must be provided.')
  # TODO(kgenova) This batch size should match.
  dataset = local_inputs.make_dataset(
      FLAGS.dataset_directory,
      mode='train',
      batch_size=FLAGS.batch_size,
      split=FLAGS.split)

  # Sets up the hyperparameters and tf.Dataset
  model_config = build_model_config(dataset)

  # Generates the graph for a single train step, including summaries
  shared_launcher.sif_transcoder(model_config)
  summary_op = tf.summary.merge_all()
  global_step_op = tf.compat.v1.train.get_global_step()

  saver = tf.train.Saver(
      max_to_keep=5, pad_step_number=False, save_relative_paths=True)

  init_op = tf.initialize_all_variables()

  model_root = get_model_root()

  experiment_dir = f'{model_root}/sif-transcoder-{FLAGS.experiment_name}'
  checkpoint_dir = f'{experiment_dir}/1-hparams/train/'

  if FLAGS.reserve_memory_for_inference_kernel and sys.platform != "darwin":
    current_free = gpu_util.get_free_gpu_memory(0)
    allowable = current_free - (1024 + 512)  # ~1GB
    allowable_fraction = allowable / current_free
    if allowable_fraction <= 0.0:
      raise ValueError(f"Can't leave 1GB over for the inference kernel, because"
                       f" there is only {allowable} total free GPU memory.")
    log.info(f'TensorFlow can use up to {allowable_fraction*100}% of the total'
             ' GPU memory.')
  else:
    allowable_fraction = 1.0
  gpu_options = tf.GPUOptions(
      per_process_gpu_memory_fraction=allowable_fraction)

  with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as session:
    writer = tf.summary.FileWriter(f'{experiment_dir}/log', session.graph)
    log.info('Initializing variables...')
    session.run([init_op])

    if FLAGS.visualize:
      visualize_data(session, model_config.inputs['dataset'])

    # Check whether the checkpoint directory already exists (resuming) or
    # needs to be created (new model).
    if not os.path.isdir(checkpoint_dir):
      log.info('No previous checkpoint detected, training from scratch.')
      os.makedirs(checkpoint_dir)
      # Serialize hparams so eval can load them:
      hparam_path = f'{checkpoint_dir}/hparam_pickle.txt'
      if not file_util.exists(hparam_path):
        hparams.write_hparams(model_config.hparams, hparam_path)
      initial_index = 0
    else:
      log.info(
          f'Checkpoint root {checkpoint_dir} exists, attempting to resume.')
      latest_checkpoint = tf.train.latest_checkpoint(checkpoint_dir)
      log.info(f'Latest checkpoint: {latest_checkpoint}')
      saver.restore(session, latest_checkpoint)
      initial_index = session.run(global_step_op)
      log.info(f'The global step is {initial_index}')
      initial_index = int(initial_index)
      log.info(f'Parsed to {initial_index}')
    start_time = time.time()
    log_every = 10
    for i in range(initial_index, FLAGS.train_step_count):
      log.verbose(f'Starting step {i}...')
      is_summary_step = i % FLAGS.summary_step_interval == 0
      if is_summary_step:
        _, summaries, loss = session.run(
            [model_config.train_op, summary_op, model_config.loss])
        writer.add_summary(summaries, i)
      else:
        _, loss = session.run([model_config.train_op, model_config.loss])
      if not (i % log_every):
        end_time = time.time()
        steps_per_second = float(log_every) / (end_time - start_time)
        start_time = end_time
        log.info(f'Step: {i}\tLoss: {loss}\tSteps/second: {steps_per_second}')

      is_checkpoint_step = i % FLAGS.checkpoint_interval == 0
      if is_checkpoint_step or i == FLAGS.train_step_count - 1:
        ckpt_path = os.path.join(checkpoint_dir, 'model.ckpt')
        log.info(f'Writing checkpoint to {ckpt_path}...')
        saver.save(session, ckpt_path, global_step=i)
    log.info('Done training!')

if __name__ == '__main__':
  app.run(main)
<commit_msg>Added early check for dataset existence.<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Lint as: python3
"""Trains a ldif network locally."""

# Commands required in-between imports to silence tensorflow
# pylint: disable=g-import-not-at-top
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

import sys
import time

from absl import app
from absl import flags

import numpy as np

import tensorflow as tf
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)

# LDIF is an internal package, should be imported last.
# pylint: disable=g-bad-import-order
from ldif.datasets import local_inputs
from ldif.util import gaps_util
from ldif.inference import example
from ldif.model import hparams
from ldif.inference import experiment as experiments
from ldif.training import shared_launcher
from ldif.util import file_util
from ldif.util import gpu_util
from ldif.util import path_util
from ldif.util.file_util import log
# pylint: enable=g-bad-import-order
# pylint: enable=g-import-not-at-top

FLAGS = flags.FLAGS

flags.DEFINE_integer('batch_size', 1, 'The batch size to use when training.')

flags.DEFINE_integer(
    'summary_step_interval', 10,
    'Summaries are written on step indices divisible by this number.')

flags.DEFINE_integer(
    'checkpoint_interval', 250,
    'Model checkpoints are written at the end of training, and at step'
    ' indices divisible by this number. Here written does not mean permanently'
    ' saved, due to how tf.train.Saver works. There is a maximum number of'
    ' checkpoints written before old checkpoints are erased, and a permanent'
    ' save frequency.')

flags.DEFINE_string(
    'experiment_name', 'reproduce-ldif',
    'The name of the experiment. It will be saved to disk under this name,'
    ' and can be reloaded for inference, eval, interactive notebooks, or'
    ' further training by providing this experiment_name. To resume training'
    ' from an already partially trained model, simply provide an experiment'
    ' name that is already written. Note that global step will be reloaded.')

flags.DEFINE_integer(
    'train_step_count', 1000000,
    'The number of training steps to take before training is complete.')

flags.DEFINE_string(
    'model_directory', 'trained_models/',
    'The path to the trained model root directory. Can be'
    ' absolute or relative to the LDIF repository root.')

flags.DEFINE_string(
    'dataset_directory', '',
    'The path to the dataset generated by meshes2dataset.py'
    ' Must be provided. The train split will be used.')

flags.DEFINE_string('split', 'train', 'The split to train on.')

flags.DEFINE_boolean(
    'visualize', False, 'If true, visualizes the training'
    ' data before training. Useful for debugging data errors.')

flags.DEFINE_string('model_type', 'ldif', 'The type of model to train. Can be'
                    'ldif (the LDIF paper model), sif (the SIF paper)'
                    ', or sif++ (an improved version of SIF using the LDIF'
                    ' architecture changes)')

flags.DEFINE_string('log_level', 'INFO',
                    'One of VERBOSE, INFO, WARNING, ERROR. Sets logs to print '
                    'only at or above the specified level.')

flags.DEFINE_boolean('reserve_memory_for_inference_kernel', True,
                     'Normally TensorFlow preallocates the entire GPU\'s memory'
                     ' when the session is created. The inference CUDA'
                     ' executable requires some memory to run. This flag stops'
                     ' TensorFlow from allocating everything, leaving ~1GB'
                     ' for the kernel (enough for 512^3 reconstruction).'
                     ' Automatically disabled on MacOS.')

def build_model_config(dataset):
  """Creates the ModelConfig object, which contains model hyperparameters."""
  # TODO(kgenova) This needs to somehow at least support LDIF/SIF/SingleView.
  # TODO(kgenova) Add support for eval/inference.
  builder_fun_dict = {
      'ldif': hparams.build_ldif_hparams,
      'sif': hparams.build_sif_hparams,
      'sif++': hparams.build_improved_sif_hparams
  }
  model_config = experiments.ModelConfig(builder_fun_dict[FLAGS.model_type]())
  model_config.hparams.bs = FLAGS.batch_size
  model_config.train = True
  model_config.eval = False
  model_config.inference = False
  model_config.inputs['dataset'] = dataset
  model_config.inputs['split'] = FLAGS.split
  model_config.inputs['proto'] = 'ShapeNetNSSDodecaSparseLRGMediumSlimPC'
  # This function is defined by the library; we don't need it, but
  # the launcher may call it.
  model_config.wrap_optimizer = lambda x: x
  return model_config

def visualize_data(session, dataset):
  """Visualizes the dataset with two interactive visualizer windows."""
  (bounding_box_samples, depth_renders, mesh_name, near_surface_samples, grid,
   world2grid, surface_point_samples) = session.run([
       dataset.bounding_box_samples, dataset.depth_renders, dataset.mesh_name,
       dataset.near_surface_samples, dataset.grid, dataset.world2grid,
       dataset.surface_point_samples
   ])
  gaps_util.ptsview(
      [bounding_box_samples, near_surface_samples, surface_point_samples])
  mesh_name = mesh_name.decode(sys.getdefaultencoding())
  log.info(f'depth max: {np.max(depth_renders)}')
  log.info(f'Mesh name: {mesh_name}')
  assert '|' in mesh_name
  mesh_hash = mesh_name[mesh_name.find('|') + 1:]
  log.info(f'Mesh hash: {mesh_hash}')
  dyn_obj = example.InferenceExample(FLAGS.split, 'airplane', mesh_hash)

  gaps_util.gapsview(
      msh=dyn_obj.normalized_gt_mesh,
      pts=near_surface_samples[:, :3],
      grd=grid,
      world2grid=world2grid,
      grid_threshold=-0.07)

def get_model_root():
  """Finds the path to the trained model's root directory based on flags."""
  ldif_abspath = path_util.get_path_to_ldif_root()
  model_dir_is_relative = FLAGS.model_directory[0] != '/'
  if model_dir_is_relative:
    model_dir_path = os.path.join(ldif_abspath, FLAGS.model_directory)
  else:
    model_dir_path = FLAGS.model_directory
  if not os.path.isdir(model_dir_path):
    os.makedirs(model_dir_path)
  return model_dir_path

def main(argv):
  if len(argv) > 1:
    raise app.UsageError('Too many command-line arguments.')
  tf.disable_v2_behavior()
  log.set_level(FLAGS.log_level)

  log.info('Making dataset...')
  if not FLAGS.dataset_directory:
    raise ValueError('A dataset directory must be provided.')
  if not os.path.isdir(FLAGS.dataset_directory):
    raise ValueError(f'No dataset directory found at {FLAGS.dataset_directory}')
  # TODO(kgenova) This batch size should match.
  dataset = local_inputs.make_dataset(
      FLAGS.dataset_directory,
      mode='train',
      batch_size=FLAGS.batch_size,
      split=FLAGS.split)

  # Sets up the hyperparameters and tf.Dataset
  model_config = build_model_config(dataset)

  # Generates the graph for a single train step, including summaries
  shared_launcher.sif_transcoder(model_config)
  summary_op = tf.summary.merge_all()
  global_step_op = tf.compat.v1.train.get_global_step()

  saver = tf.train.Saver(
      max_to_keep=5, pad_step_number=False, save_relative_paths=True)

  init_op = tf.initialize_all_variables()

  model_root = get_model_root()

  experiment_dir = f'{model_root}/sif-transcoder-{FLAGS.experiment_name}'
  checkpoint_dir = f'{experiment_dir}/1-hparams/train/'

  if FLAGS.reserve_memory_for_inference_kernel and sys.platform != "darwin":
    current_free = gpu_util.get_free_gpu_memory(0)
    allowable = current_free - (1024 + 512)  # ~1GB
    allowable_fraction = allowable / current_free
    if allowable_fraction <= 0.0:
      raise ValueError(f"Can't leave 1GB over for the inference kernel, because"
                       f" there is only {allowable} total free GPU memory.")
    log.info(f'TensorFlow can use up to {allowable_fraction*100}% of the total'
             ' GPU memory.')
  else:
    allowable_fraction = 1.0
  gpu_options = tf.GPUOptions(
      per_process_gpu_memory_fraction=allowable_fraction)

  with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as session:
    writer = tf.summary.FileWriter(f'{experiment_dir}/log', session.graph)
    log.info('Initializing variables...')
    session.run([init_op])

    if FLAGS.visualize:
      visualize_data(session, model_config.inputs['dataset'])

    # Check whether the checkpoint directory already exists (resuming) or
    # needs to be created (new model).
    if not os.path.isdir(checkpoint_dir):
      log.info('No previous checkpoint detected, training from scratch.')
      os.makedirs(checkpoint_dir)
      # Serialize hparams so eval can load them:
      hparam_path = f'{checkpoint_dir}/hparam_pickle.txt'
      if not file_util.exists(hparam_path):
        hparams.write_hparams(model_config.hparams, hparam_path)
      initial_index = 0
    else:
      log.info(
          f'Checkpoint root {checkpoint_dir} exists, attempting to resume.')
      latest_checkpoint = tf.train.latest_checkpoint(checkpoint_dir)
      log.info(f'Latest checkpoint: {latest_checkpoint}')
      saver.restore(session, latest_checkpoint)
      initial_index = session.run(global_step_op)
      log.info(f'The global step is {initial_index}')
      initial_index = int(initial_index)
      log.info(f'Parsed to {initial_index}')
    start_time = time.time()
    log_every = 10
    for i in range(initial_index, FLAGS.train_step_count):
      log.verbose(f'Starting step {i}...')
      is_summary_step = i % FLAGS.summary_step_interval == 0
      if is_summary_step:
        _, summaries, loss = session.run(
            [model_config.train_op, summary_op, model_config.loss])
        writer.add_summary(summaries, i)
      else:
        _, loss = session.run([model_config.train_op, model_config.loss])
      if not (i % log_every):
        end_time = time.time()
        steps_per_second = float(log_every) / (end_time - start_time)
        start_time = end_time
        log.info(f'Step: {i}\tLoss: {loss}\tSteps/second: {steps_per_second}')

      is_checkpoint_step = i % FLAGS.checkpoint_interval == 0
      if is_checkpoint_step or i == FLAGS.train_step_count - 1:
        ckpt_path = os.path.join(checkpoint_dir, 'model.ckpt')
        log.info(f'Writing checkpoint to {ckpt_path}...')
        saver.save(session, ckpt_path, global_step=i)
    log.info('Done training!')

if __name__ == '__main__':
  app.run(main)
<|endoftext|>


<commit_before>import autograd as ag
import click
import copy
import numpy as np
import logging
import pickle

from sklearn.cross_validation import train_test_split
from sklearn.metrics import roc_auc_score
from sklearn.preprocessing import RobustScaler
from sklearn.utils import check_random_state

from recnn.preprocessing import rewrite_content
from recnn.preprocessing import permute_by_pt
from recnn.preprocessing import extract
from recnn.recnn import log_loss
from recnn.recnn import adam
from recnn.recnn import grnn_init_simple
from recnn.recnn import grnn_predict_simple
from recnn.recnn import grnn_init_gated
from recnn.recnn import grnn_predict_gated

logging.basicConfig(level=logging.INFO,
                    format="[%(asctime)s %(levelname)s] %(message)s")

@click.command()
@click.argument("filename_train")
@click.argument("filename_model")
@click.option("--simple", is_flag=True, default=False)
@click.option("--n_features", default=7)
@click.option("--n_hidden", default=40)
@click.option("--n_epochs", default=20)
@click.option("--batch_size", default=64)
@click.option("--step_size", default=0.0005)
@click.option("--decay", default=0.9)
@click.option("--random_state", default=1)
def train(filename_train,
          filename_model,
          simple=False,
          n_features=7,
          n_hidden=30,
          n_epochs=5,
          batch_size=64,
          step_size=0.01,
          decay=0.7,
          random_state=1):
    # Initialization
    gated = not simple
    logging.info("Calling with...")
    logging.info("\tfilename_train = %s" % filename_train)
    logging.info("\tfilename_model = %s" % filename_model)
    logging.info("\tgated = %s" % gated)
    logging.info("\tn_features = %d" % n_features)
    logging.info("\tn_hidden = %d" % n_hidden)
    logging.info("\tn_epochs = %d" % n_epochs)
    logging.info("\tbatch_size = %d" % batch_size)
    logging.info("\tstep_size = %f" % step_size)
    logging.info("\tdecay = %f" % decay)
    logging.info("\trandom_state = %d" % random_state)
    rng = check_random_state(random_state)

    # Make data
    logging.info("Loading data...")

    fd = open(filename_train, "rb")
    X, y = pickle.load(fd)
    fd.close()
    y = np.array(y)

    logging.info("\tfilename = %s" % filename_train)
    logging.info("\tX size = %d" % len(X))
    logging.info("\ty size = %d" % len(y))

    # Preprocessing
    logging.info("Preprocessing...")
    X = [extract(permute_by_pt(rewrite_content(jet))) for jet in X]
    tf = RobustScaler().fit(np.vstack([jet["content"] for jet in X]))

    for jet in X:
        jet["content"] = tf.transform(jet["content"])

    # Split into train+test
    logging.info("Splitting into train and validation...")

    X_train, X_valid, y_train, y_valid = train_test_split(X, y,
                                                          test_size=5000,
                                                          random_state=rng)

    # Training
    logging.info("Training...")

    if gated:
        predict = grnn_predict_gated
        init = grnn_init_gated
    else:
        predict = grnn_predict_simple
        init = grnn_init_simple

    trained_params = init(n_features, n_hidden, random_state=rng)
    n_batches = int(np.ceil(len(X_train) / batch_size))
    best_score = [-np.inf]  # yuck, but works
    best_params = [trained_params]

    def loss(X, y, params):
        y_pred = predict(params, X)
        l = log_loss(y, y_pred).mean()
        return l

    def objective(params, iteration):
        rng = check_random_state(iteration % n_batches)
        start = rng.randint(len(X_train) - batch_size)
        idx = slice(start, start+batch_size)
        return loss(X_train[idx], y_train[idx], params)

    def callback(params, iteration, gradient):
        if iteration % 25 == 0:
            roc_auc = roc_auc_score(y_valid, predict(params, X_valid))

            if roc_auc > best_score[0]:
                best_score[0] = roc_auc
                best_params[0] = copy.deepcopy(params)

                fd = open(filename_model, "wb")
                pickle.dump(best_params[0], fd)
                fd.close()

            logging.info(
                "%5d\t~loss(train)=%.4f\tloss(valid)=%.4f"
                "\troc_auc(valid)=%.4f\tbest_roc_auc(valid)=%.4f" % (
                    iteration,
                    loss(X_train[:5000], y_train[:5000], params),
                    loss(X_valid, y_valid, params),
                    roc_auc,
                    best_score[0]))

    for i in range(n_epochs):
        logging.info("epoch = %d" % i)
        logging.info("step_size = %.4f" % step_size)

        trained_params = adam(ag.grad(objective),
                              trained_params,
                              step_size=step_size,
                              num_iters=1 * n_batches,
                              callback=callback)
        step_size = step_size * decay

if __name__ == "__main__":
    train()
<commit_msg>n_events_train for jets<commit_after>import autograd as ag
import click
import copy
import numpy as np
import logging
import pickle

from sklearn.cross_validation import train_test_split
from sklearn.metrics import roc_auc_score
from sklearn.preprocessing import RobustScaler
from sklearn.utils import check_random_state

from recnn.preprocessing import rewrite_content
from recnn.preprocessing import permute_by_pt
from recnn.preprocessing import extract
from recnn.recnn import log_loss
from recnn.recnn import adam
from recnn.recnn import grnn_init_simple
from recnn.recnn import grnn_predict_simple
from recnn.recnn import grnn_init_gated
from recnn.recnn import grnn_predict_gated

logging.basicConfig(level=logging.INFO,
                    format="[%(asctime)s %(levelname)s] %(message)s")

@click.command()
@click.argument("filename_train")
@click.argument("filename_model")
@click.option("--n_events_train", default=-1)
@click.option("--simple", is_flag=True, default=False)
@click.option("--n_features", default=7)
@click.option("--n_hidden", default=40)
@click.option("--n_epochs", default=20)
@click.option("--batch_size", default=64)
@click.option("--step_size", default=0.0005)
@click.option("--decay", default=0.9)
@click.option("--random_state", default=1)
def train(filename_train,
          filename_model,
          n_events_train=-1,
          simple=False,
          n_features=7,
          n_hidden=30,
          n_epochs=5,
          batch_size=64,
          step_size=0.01,
          decay=0.7,
          random_state=1):
    # Initialization
    gated = not simple
    logging.info("Calling with...")
    logging.info("\tfilename_train = %s" % filename_train)
    logging.info("\tfilename_model = %s" % filename_model)
    logging.info("\tn_events_train = %d" % n_events_train)
    logging.info("\tgated = %s" % gated)
    logging.info("\tn_features = %d" % n_features)
    logging.info("\tn_hidden = %d" % n_hidden)
    logging.info("\tn_epochs = %d" % n_epochs)
    logging.info("\tbatch_size = %d" % batch_size)
    logging.info("\tstep_size = %f" % step_size)
    logging.info("\tdecay = %f" % decay)
    logging.info("\trandom_state = %d" % random_state)
    rng = check_random_state(random_state)

    # Make data
    logging.info("Loading data...")

    fd = open(filename_train, "rb")
    X, y = pickle.load(fd)
    fd.close()
    y = np.array(y)

    if n_events_train > 0:
        indices = rng.permutation(len(X))[:n_events_train]
        X = [X[i] for i in indices]
        y = y[indices]

    logging.info("\tfilename = %s" % filename_train)
    logging.info("\tX size = %d" % len(X))
    logging.info("\ty size = %d" % len(y))

    # Preprocessing
    logging.info("Preprocessing...")
    X = [extract(permute_by_pt(rewrite_content(jet))) for jet in X]
    tf = RobustScaler().fit(np.vstack([jet["content"] for jet in X]))

    for jet in X:
        jet["content"] = tf.transform(jet["content"])

    # Split into train+test
    logging.info("Splitting into train and validation...")

    X_train, X_valid, y_train, y_valid = train_test_split(X, y,
                                                          test_size=5000,
                                                          random_state=rng)

    # Training
    logging.info("Training...")

    if gated:
        predict = grnn_predict_gated
        init = grnn_init_gated
    else:
        predict = grnn_predict_simple
        init = grnn_init_simple

    trained_params = init(n_features, n_hidden, random_state=rng)
    n_batches = int(np.ceil(len(X_train) / batch_size))
    best_score = [-np.inf]  # yuck, but works
    best_params = [trained_params]

    def loss(X, y, params):
        y_pred = predict(params, X)
        l = log_loss(y, y_pred).mean()
        return l

    def objective(params, iteration):
        rng = check_random_state(iteration % n_batches)
        start = rng.randint(len(X_train) - batch_size)
        idx = slice(start, start+batch_size)
        return loss(X_train[idx], y_train[idx], params)

    def callback(params, iteration, gradient):
        if iteration % 25 == 0:
            roc_auc = roc_auc_score(y_valid, predict(params, X_valid))

            if roc_auc > best_score[0]:
                best_score[0] = roc_auc
                best_params[0] = copy.deepcopy(params)

                fd = open(filename_model, "wb")
                pickle.dump(best_params[0], fd)
                fd.close()

            logging.info(
                "%5d\t~loss(train)=%.4f\tloss(valid)=%.4f"
                "\troc_auc(valid)=%.4f\tbest_roc_auc(valid)=%.4f" % (
                    iteration,
                    loss(X_train[:5000], y_train[:5000], params),
                    loss(X_valid, y_valid, params),
                    roc_auc,
                    best_score[0]))

    for i in range(n_epochs):
        logging.info("epoch = %d" % i)
        logging.info("step_size = %.4f" % step_size)

        trained_params = adam(ag.grad(objective),
                              trained_params,
                              step_size=step_size,
                              num_iters=1 * n_batches,
                              callback=callback)
        step_size = step_size * decay

if __name__ == "__main__":
    train()
<|endoftext|>