<commit_before># -*- coding: utf-8 -*-
#
# Copyright (C) 2005 Edgewall Software
# Copyright (C) 2005 Christopher Lenz <cmlenz@gmx.de>
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.com/license.html.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://projects.edgewall.com/trac/.
#
# Author: Christopher Lenz <cmlenz@gmx.de>

import os
import sys

try:
    import pkg_resources
except ImportError:
    pkg_resources = None

TRAC_META = 'trac_plugin.txt'

__all__ = ['load_components']

def normalize_path(s):
  return os.path.normcase(os.path.abspath(s))

def paths_equal(path1, path2):
  return normalize_path(path1) == normalize_path(path2)

def load_components(env):

    loaded_components = []

    def load_module(name):
         if name not in loaded_components:
             try:
                 module = __import__(name)
                 loaded_components.append(name)
                 return module
             except ImportError, e:
                 env.log.error('Component module %s not found',
                               name, exc_info=True)

    plugins_dir = os.path.join(env.path, 'plugins')

    def enable_modules(egg_path, modules):
        """Automatically enable any components provided by plugins loaded from
        the environment plugins directory."""
        if paths_equal(os.path.dirname(egg_path), os.path.realpath(plugins_dir)):
            for module in modules:
                env.config.setdefault('components', module + '.*', 'enabled')

    # Load components from the environment plugins directory
    if pkg_resources is not None: # But only if setuptools is installed!
        if hasattr(pkg_resources, 'Environment'):
            # setuptools >= 0.6
            pkg_env = pkg_resources.Environment([plugins_dir] + sys.path)
            for name in pkg_env:
                egg = pkg_env[name][0]
                modules = []

                for name in egg.get_entry_map('trac.plugins'):
                    # Load plugins declared via the `trac.plugins` entry point.
                    # This is the only supported option going forward, the
                    # others will be dropped at some point in the future.
                    env.log.debug('Loading plugin %s from %s', name,
                                  egg.location)
                    egg.activate()
                    try:
                        entry_point = egg.get_entry_info('trac.plugins', name)
                        if entry_point.module_name not in loaded_components:
                            entry_point.load()
                            modules.append(entry_point.module_name)
                            loaded_components.append(entry_point.module_name)
                    except ImportError, e:
                        env.log.error('Failed to load plugin %s from %s', name,
                                      egg.location, exc_info=True)

                else:
                    # Support for pre-entry-point plugins
                    if egg.has_metadata('trac_plugin.txt'):
                        env.log.debug('Loading plugin %s from %s', name,
                                      egg.location)
                        egg.activate()
                        for module in egg.get_metadata_lines('trac_plugin.txt'):
                            if load_module(module):
                                modules.append(module)

                if modules:
                    enable_modules(egg.location, modules)

        else:
            # setuptools < 0.6
            distributions = pkg_resources.AvailableDistributions([plugins_dir] \
                                                                 + sys.path)
            for name in distributions:
                egg = distributions[name][0]
                modules = []
                if egg.metadata.has_metadata(TRAC_META):
                    egg.install_on()
                    for module in egg.metadata.get_metadata_lines(TRAC_META):
                        if load_module(module):
                            modules.append(module)

                if modules:
                    enable_modules(egg.path, modules)

    elif os.path.exists(plugins_dir) and os.listdir(plugins_dir):
        env.log.warning('setuptools is required for plugin deployment')

    # Load default components
    from trac.db_default import default_components
    for module in default_components:
        if not module in loaded_components:
            __import__(module)
<commit_msg>Drop support for Setuptools < 0.6, and other clean-up in `trac.loader`.<commit_after># -*- coding: utf-8 -*-
#
# Copyright (C) 2005-2006 Edgewall Software
# Copyright (C) 2005-2006 Christopher Lenz <cmlenz@gmx.de>
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.com/license.html.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://projects.edgewall.com/trac/.
#
# Author: Christopher Lenz <cmlenz@gmx.de>

import os
import sys

try:
    import pkg_resources
except ImportError:
    pkg_resources = None

__all__ = ['load_components']

def load_components(env):
    loaded_components = []

    plugins_dir = os.path.normcase(os.path.realpath(os.path.join(env.path,
                                                                 'plugins')))

    # Load components from the environment plugins directory
    if pkg_resources is not None: # But only if setuptools is installed!
        pkg_env = pkg_resources.Environment([plugins_dir] + sys.path)
        for name in pkg_env:
            egg = pkg_env[name][0]
            modules = []

            for name in egg.get_entry_map('trac.plugins'):
                # Load plugins declared via the `trac.plugins` entry point.
                # This is the only supported option going forward, the
                # others will be dropped at some point in the future.
                env.log.debug('Loading plugin %s from %s', name,
                              egg.location)
                egg.activate()
                try:
                    entry_point = egg.get_entry_info('trac.plugins', name)
                    if entry_point.module_name not in loaded_components:
                        entry_point.load()
                        modules.append(entry_point.module_name)
                        loaded_components.append(entry_point.module_name)
                except ImportError, e:
                    env.log.error('Failed to load plugin %s from %s', name,
                                  egg.location, exc_info=True)

            else:
                # Support for pre-entry-point plugins
                if egg.has_metadata('trac_plugin.txt'):
                    env.log.debug('Loading plugin %s from %s', name,
                                  egg.location)
                    egg.activate()
                    for modname in egg.get_metadata_lines('trac_plugin.txt'):
                        module = None
                        if modname not in loaded_components:
                            try:
                                module = __import__(modname)
                                loaded_components.append(modname)
                                modules.append(modname)
                            except ImportError, e:
                                env.log.error('Component module %s not found',
                                              modname, exc_info=True)

            if modules:
                # Automatically enable any components provided by plugins
                # loaded from the environment plugins directory.
                if os.path.dirname(egg.location) == plugins_dir:
                    for module in modules:
                        env.config.setdefault('components', module + '.*',
                                              'enabled')

    elif os.path.exists(plugins_dir) and os.listdir(plugins_dir):
        env.log.warning('setuptools is required for plugin deployment')

    # Load default components
    from trac.db_default import default_components
    for module in default_components:
        if not module in loaded_components:
            __import__(module)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright (C) 2005-2009 Edgewall Software
# Copyright (C) 2005-2006 Christopher Lenz <cmlenz@gmx.de>
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.org/wiki/TracLicense.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://trac.edgewall.org/log/.
#
# Author: Christopher Lenz <cmlenz@gmx.de>

from glob import glob
import imp
import pkg_resources
from pkg_resources import working_set, DistributionNotFound, VersionConflict, \
                          UnknownExtra
import os
import sys

from trac.util.text import exception_to_unicode

__all__ = ['load_components']

def _enable_plugin(env, module):
    """Enable the given plugin module by adding an entry to the configuration.
    """
    if module + '.*' not in env.config['components']:
        env.config['components'].set(module + '.*', 'enabled')

def load_eggs(entry_point_name):
    """Loader that loads any eggs on the search path and `sys.path`."""
    def _load_eggs(env, search_path, auto_enable=None):
        # Note that the following doesn't seem to support unicode search_path
        distributions, errors = working_set.find_plugins(
            pkg_resources.Environment(search_path)
        )
        for dist in distributions:
            env.log.debug('Adding plugin %s from %s', dist, dist.location)
            working_set.add(dist)

        def _log_error(item, e):
            ue = exception_to_unicode(e)
            if isinstance(e, DistributionNotFound):
                env.log.debug('Skipping "%s": ("%s" not found)', item, ue)
            elif isinstance(e, VersionConflict):
                env.log.error('Skipping "%s": (version conflict "%s")',
                              item, ue)
            elif isinstance(e, UnknownExtra):
                env.log.error('Skipping "%s": (unknown extra "%s")', item, ue)
            elif isinstance(e, ImportError):
                env.log.error('Skipping "%s": (can\'t import "%s")', item, ue)
            else:
                env.log.error('Skipping "%s": (error "%s")', item, ue)

        for dist, e in errors.iteritems():
            _log_error(dist, e)

        for entry in working_set.iter_entry_points(entry_point_name):
            env.log.debug('Loading %s from %s', entry.name,
                          entry.dist.location)
            try:
                entry.load(require=True)
            except (ImportError, DistributionNotFound, VersionConflict,
                    UnknownExtra), e:
                _log_error(entry, e)
            else:
                if os.path.dirname(entry.dist.location) == auto_enable:
                    _enable_plugin(env, entry.module_name)
    return _load_eggs

def load_py_files():
    """Loader that look for Python source files in the plugins directories,
    which simply get imported, thereby registering them with the component
    manager if they define any components.
    """
    def _load_py_files(env, search_path, auto_enable=None):
        for path in search_path:
            plugin_files = glob(os.path.join(path, '*.py'))
            for plugin_file in plugin_files:
                try:
                    plugin_name = os.path.basename(plugin_file[:-3])
                    env.log.debug('Loading file plugin %s from %s' % \
                                  (plugin_name, plugin_file))
                    if plugin_name not in sys.modules:
                        module = imp.load_source(plugin_name, plugin_file)
                    if path == auto_enable:
                        _enable_plugin(env, plugin_name)
                except Exception, e:
                    env.log.error('Failed to load plugin from %s', plugin_file,
                                  exception_to_unicode(e, traceback=True))

    return _load_py_files

def load_components(env, extra_path=None, loaders=(load_eggs('trac.plugins'),
                                                   load_py_files())):
    """Load all plugin components found on the given search path."""
    plugins_dir = os.path.normcase(os.path.realpath(
        os.path.join(env.path, 'plugins')
    ))
    search_path = [plugins_dir]
    if extra_path:
        search_path += list(extra_path)

    for loadfunc in loaders:
        loadfunc(env, search_path, auto_enable=plugins_dir)
<commit_msg>0.12dev: Merged [8061] from 0.11-stable.<commit_after># -*- coding: utf-8 -*-
#
# Copyright (C) 2005-2009 Edgewall Software
# Copyright (C) 2005-2006 Christopher Lenz <cmlenz@gmx.de>
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.org/wiki/TracLicense.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://trac.edgewall.org/log/.
#
# Author: Christopher Lenz <cmlenz@gmx.de>

from glob import glob
import imp
import pkg_resources
from pkg_resources import working_set, DistributionNotFound, VersionConflict, \
                          UnknownExtra
import os
import sys

from trac.util.text import exception_to_unicode

__all__ = ['load_components']

def _enable_plugin(env, module):
    """Enable the given plugin module by adding an entry to the configuration.
    """
    if module + '.*' not in env.config['components']:
        env.config['components'].set(module + '.*', 'enabled')

def load_eggs(entry_point_name):
    """Loader that loads any eggs on the search path and `sys.path`."""
    def _load_eggs(env, search_path, auto_enable=None):
        # Note that the following doesn't seem to support unicode search_path
        distributions, errors = working_set.find_plugins(
            pkg_resources.Environment(search_path)
        )
        for dist in distributions:
            env.log.debug('Adding plugin %s from %s', dist, dist.location)
            working_set.add(dist)

        def _log_error(item, e):
            ue = exception_to_unicode(e)
            if isinstance(e, DistributionNotFound):
                env.log.debug('Skipping "%s": ("%s" not found)', item, ue)
            elif isinstance(e, VersionConflict):
                env.log.error('Skipping "%s": (version conflict "%s")',
                              item, ue)
            elif isinstance(e, UnknownExtra):
                env.log.error('Skipping "%s": (unknown extra "%s")', item, ue)
            elif isinstance(e, ImportError):
                env.log.error('Skipping "%s": (can\'t import "%s")', item, ue)
            else:
                env.log.error('Skipping "%s": (error "%s")', item, ue)

        for dist, e in errors.iteritems():
            _log_error(dist, e)

        for entry in working_set.iter_entry_points(entry_point_name):
            env.log.debug('Loading %s from %s', entry.name,
                          entry.dist.location)
            try:
                entry.load(require=True)
            except (ImportError, DistributionNotFound, VersionConflict,
                    UnknownExtra), e:
                _log_error(entry, e)
            else:
                if os.path.dirname(entry.dist.location) == auto_enable:
                    _enable_plugin(env, entry.module_name)
    return _load_eggs

def load_py_files():
    """Loader that look for Python source files in the plugins directories,
    which simply get imported, thereby registering them with the component
    manager if they define any components.
    """
    def _load_py_files(env, search_path, auto_enable=None):
        for path in search_path:
            plugin_files = glob(os.path.join(path, '*.py'))
            for plugin_file in plugin_files:
                try:
                    plugin_name = os.path.basename(plugin_file[:-3])
                    env.log.debug('Loading file plugin %s from %s' % \
                                  (plugin_name, plugin_file))
                    if plugin_name not in sys.modules:
                        module = imp.load_source(plugin_name, plugin_file)
                    if path == auto_enable:
                        _enable_plugin(env, plugin_name)
                except Exception, e:
                    env.log.error('Failed to load plugin from %s: %s',
                                  plugin_file,
                                  exception_to_unicode(e, traceback=True))

    return _load_py_files

def load_components(env, extra_path=None, loaders=(load_eggs('trac.plugins'),
                                                   load_py_files())):
    """Load all plugin components found on the given search path."""
    plugins_dir = os.path.normcase(os.path.realpath(
        os.path.join(env.path, 'plugins')
    ))
    search_path = [plugins_dir]
    if extra_path:
        search_path += list(extra_path)

    for loadfunc in loaders:
        loadfunc(env, search_path, auto_enable=plugins_dir)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import sys
from datetime import datetime, timedelta
import email
from imapclient import IMAPClient

## http://stackoverflow.com/a/6618585/1839451
## Put your info here
EMAIL = "netfabbcloud@gmail.com"
HOST = "imap.gmail.com"
USERNAME = EMAIL
PASSWORD = "******************"
ssl = True

today = datetime.today()

## Connect, login and select the INBOX
server = IMAPClient(HOST, use_uid=True, ssl=ssl)
server.login(USERNAME, PASSWORD)
select_info = server.select_folder('INBOX')

## Search for relevant messages
messages = server.search(['FROM "cloud@netfabb.com"', 'SINCE %s' % today.strftime('%d-%b-%Y')])
response = server.fetch(messages, ['RFC822'])

for msgid, data in response.iteritems():
	msg_string = data['RFC822']
	msg = email.message_from_string(msg_string)
	#print '%s' % (msg['text'])
	print msg
<commit_msg>Sending request from python<commit_after>#!/usr/bin/env python

import sys, re, urllib2, MultipartPostHandler, os.path, email
from datetime import datetime, timedelta
from imapclient import IMAPClient

## Chcek if we have exactly one argument
if len(sys.argv) != 2:
	print 'Usage: %s file.stl' % sys.argv[0]
	exit(1)

## Check if the argument is file
stlfile = sys.argv[1]
if not os.path.isfile(stlfile):
	print '%s is not a file' % stlfile
	exit(1)

## Put your info here
EMAIL = 'netfabbcloud@gmail.com'
HOST = 'imap.gmail.com'
USERNAME = EMAIL
PASSWORD = '******************'
ssl = True
UNITS = 'mm'

## Send the file to Netfabb Cloud
url = 'http://cloud.netfabb.com/index.php'
data = {}
data['file'] = open(stlfile, 'rb')
data['email'] = EMAIL
data['units'] = UNITS
data['filerepair'] = '1'
data['termsandconditions'] = '1'
data['upload'] = 'Upload to Cloud'
opener = urllib2.build_opener(MultipartPostHandler.MultipartPostHandler)
urllib2.install_opener(opener)
request = urllib2.Request(url, data)
try: page = urllib2.urlopen(request)
except URLError, e:
	print e.reason
	exit(1)

## It seems only english letters (low or capital), numbers and dots are left in the filename
crippledname = re.sub('[^0-9,a-z,A-Z,.]+', '', stlfile)

try:
	## Connect, login and select the INBOX
	server = IMAPClient(HOST, use_uid=True, ssl=ssl)
	server.login(USERNAME, PASSWORD)
	select_info = server.select_folder('INBOX')
	## Search for relevant messages
	messages = server.search(['FROM "cloud@netfabb.com"', 'SINCE %s' % datetime.today().strftime('%d-%b-%Y')])
	response = server.fetch(messages, ['RFC822'])

	for msgid, data in response.iteritems():
		msg_string = data['RFC822']
		msg = email.message_from_string(msg_string)
		print '%s' % msg['Subject']
		#print msg
except IMAPClient.Error, e:
	print e
	exit(1)

## Error string:
# Unfortunately, your uploaded file "FILE" was not processed successfully. The system logged the short error message "ERROR".
## OK string:
# Congratulations! Your uploaded file "FILE" was processed successfully. The results are located at
<|endoftext|>


<commit_before>from __future__ import print_function
import numpy as np
np.random.seed(1337)  # for reproducibility
SEED = 1337

import os
import pickle
from time import localtime, strftime

import keras
from keras.datasets import mnist
from keras.utils import np_utils
from keras import backend as K
from keras.optimizers import SGD

from bird.models.resnet import ResNetBuilder
from bird.generators.sound import SoundDataGenerator

train_path = "./datasets/birdClef2016Subset/train";
valid_path = "./datasets/birdClef2016Subset/valid";
basename = strftime("%Y_%m_%d_%H:%M:%S_", localtime()) + "cuberun"
weight_file_path = os.path.join("./weights", basename + ".h5")
history_file_path = os.path.join("./history", basename + ".pkl")

batch_size = 8
nb_classes = 20
samples_per_epoch = 12
nb_epoch = 1

# input image dimensions
img_rows, img_cols = 256, 512
# number of channels
nb_channels = 1

# Setup Callbacks for History
class HistoryCollector(keras.callbacks.Callback):
    def __init__(self, name):
        self.name = name
        self.data = []

    def on_epoch_end(self, batch, logs={}):
        self.data.append(logs.get(self.name))

trainLossHistory = HistoryCollector('loss')
validLossHistory = HistoryCollector('val_loss')
trainAccHistory = HistoryCollector('acc')
validAccHistory = HistoryCollector('val_acc')

model = ResNetBuilder.build_resnet_18((img_rows, img_cols, nb_channels), nb_classes)
sgd = SGD(lr=0.001, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy',
              optimizer=sgd,
              metrics=['accuracy'])

train_datagen = SoundDataGenerator(
    rescale=1./255,
    width_shift_range=0.5,
    height_shift_range=0.05,
    augment_with_same_class=True,
    augment_with_noise=True)

valid_datagen = SoundDataGenerator(
    rescale=1./255)

# Generator for training data
print("Loading training data...")
train_generator = train_datagen.flow_from_directory(
    train_path,
    target_size=(img_rows, img_cols),
    batch_size=batch_size,
    class_mode='categorical',
    color_mode='grayscale',
    seed=SEED)

# Generator for validation data
print("Loading validation data...")
valid_generator = valid_datagen.flow_from_directory(
    valid_path,
    target_size=(img_rows, img_cols),
    batch_size=batch_size,
    class_mode='categorical',
    color_mode='grayscale',
    seed=SEED)

# Fit model to generated training data
model.fit_generator(
    train_generator,
    samples_per_epoch=samples_per_epoch,
    nb_epoch=nb_epoch,
    validation_data=valid_generator,
    nb_val_samples=613,
    callbacks=[trainLossHistory, validLossHistory, trainAccHistory, validAccHistory])

# save the weights
model.save_weights(weight_file_path)
print (strftime("%a, %d %b %Y %H:%M:%S +0000", localtime()))
print ("The weights have been saved in: " + weight_file_path)

# save history to file
with open(history_file_path, 'wb') as output:
    pickle.dump(trainLossHistory.data, output, pickle.HIGHEST_PROTOCOL)
    pickle.dump(validLossHistory.data, output, pickle.HIGHEST_PROTOCOL)
    pickle.dump(trainAccHistory.data, output, pickle.HIGHEST_PROTOCOL)
    pickle.dump(validAccHistory.data, output, pickle.HIGHEST_PROTOCOL)
<commit_msg>Add print for history save<commit_after>from __future__ import print_function
import numpy as np
np.random.seed(1337)  # for reproducibility
SEED = 1337

import os
import pickle
from time import localtime, strftime

import keras
from keras.datasets import mnist
from keras.utils import np_utils
from keras import backend as K
from keras.optimizers import SGD

from bird.models.resnet import ResNetBuilder
from bird.generators.sound import SoundDataGenerator

train_path = "./datasets/birdClef2016Subset/train";
valid_path = "./datasets/birdClef2016Subset/valid";
basename = strftime("%Y_%m_%d_%H:%M:%S_", localtime()) + "cuberun"
weight_file_path = os.path.join("./weights", basename + ".h5")
history_file_path = os.path.join("./history", basename + ".pkl")

batch_size = 8
nb_classes = 20
samples_per_epoch = 12
nb_epoch = 1

# input image dimensions
img_rows, img_cols = 256, 512
# number of channels
nb_channels = 1

# Setup Callbacks for History
class HistoryCollector(keras.callbacks.Callback):
    def __init__(self, name):
        self.name = name
        self.data = []

    def on_epoch_end(self, batch, logs={}):
        self.data.append(logs.get(self.name))

trainLossHistory = HistoryCollector('loss')
validLossHistory = HistoryCollector('val_loss')
trainAccHistory = HistoryCollector('acc')
validAccHistory = HistoryCollector('val_acc')

model = ResNetBuilder.build_resnet_18((img_rows, img_cols, nb_channels), nb_classes)
sgd = SGD(lr=0.001, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy',
              optimizer=sgd,
              metrics=['accuracy'])

train_datagen = SoundDataGenerator(
    rescale=1./255,
    width_shift_range=0.5,
    height_shift_range=0.05,
    augment_with_same_class=True,
    augment_with_noise=True)

valid_datagen = SoundDataGenerator(
    rescale=1./255)

# Generator for training data
print("Loading training data...")
train_generator = train_datagen.flow_from_directory(
    train_path,
    target_size=(img_rows, img_cols),
    batch_size=batch_size,
    class_mode='categorical',
    color_mode='grayscale',
    seed=SEED)

# Generator for validation data
print("Loading validation data...")
valid_generator = valid_datagen.flow_from_directory(
    valid_path,
    target_size=(img_rows, img_cols),
    batch_size=batch_size,
    class_mode='categorical',
    color_mode='grayscale',
    seed=SEED)

# Fit model to generated training data
model.fit_generator(
    train_generator,
    samples_per_epoch=samples_per_epoch,
    nb_epoch=nb_epoch,
    validation_data=valid_generator,
    nb_val_samples=613,
    callbacks=[trainLossHistory, validLossHistory, trainAccHistory, validAccHistory])

# save the weights
model.save_weights(weight_file_path)
print (strftime("%a, %d %b %Y %H:%M:%S +0000", localtime()))
print ("The weights have been saved in: " + weight_file_path)

# save history to file
with open(history_file_path, 'wb') as output:
    pickle.dump(trainLossHistory.data, output, pickle.HIGHEST_PROTOCOL)
    pickle.dump(validLossHistory.data, output, pickle.HIGHEST_PROTOCOL)
    pickle.dump(trainAccHistory.data, output, pickle.HIGHEST_PROTOCOL)
    pickle.dump(validAccHistory.data, output, pickle.HIGHEST_PROTOCOL)
print ("The history has been saved in: " + history_file_path)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- encoding: utf-8 -*-
# pylint: disable=expression-not-assigned

""" track-cli - CLI interface to track

"""

import os
import sys
from contextlib import suppress
import argparse
import json
from typing import List

import zmq

from .core import util, common, ActiveApplications
from .core.util import log

def parse_arguments(argv: List[str]) -> argparse.Namespace:
    """parse command line arguments and return argument object"""
    parser = argparse.ArgumentParser(description=__doc__)
    common.setup_argument_parser(parser)

    subparsers = parser.add_subparsers(help='available commands', metavar="CMD")

    parser_help = subparsers.add_parser('help', help='show this help')
    parser_help.set_defaults(func=lambda *_: parser.print_help())

    parser_list = subparsers.add_parser('list', help='list basic logs')
    parser_list.set_defaults(func=fn_info)

    parser_show = subparsers.add_parser('show', help='show content of one log file')
    parser_show.set_defaults(func=fn_show)
    parser_show.add_argument("element", nargs="+")

    parser.set_defaults(func=lambda *a: parser.print_usage())

    return parser.parse_args(argv)

def send_request(request):
    context = zmq.Context()
    req_socket = context.socket(zmq.REQ)

    req_socket.connect('tcp://127.0.0.1:3456')

    req_socket.send_json(request)
    return req_socket.recv_json()

def handle_result(result):
    if 'type' in result and result['type'] == 'error':
        raise Exception('server replied with error: "%s"' % result['what'])
    print(result)

def convert(data):
    return data if "tracker_data" in data else {"tracker_data": data}

def to_time(value):
    return "%2d:%.2d" % (value // 60, value % 60)

def fn_show(args) -> None:
    log_dir = args.data_dir

    log().info("Show infos for %r", args.element)
    for file in args.element:
        data = convert(json.load(open(os.path.join(log_dir, file))))
        apps = ActiveApplications(data["tracker_data"])
        daily_note = data.get("daily_note") or ""
        print("%s: %s - %s = %s => %s" % (
            file,
            to_time(apps.begin_index()),
            to_time(apps.end_index()),
            to_time(apps.end_index() - apps.begin_index()),
            to_time(apps.end_index() - apps.begin_index() - 60)))
        for time in (t for t in range(apps.begin_index(), apps.end_index()) if t in apps._minutes):
            print(to_time(time))
        print(daily_note)

def fn_info(args) -> None:
    log_dir = args.data_dir
    log().info("List recorded data")
    for file in (
            f
            for f in sorted(os.listdir(log_dir))
            if '-log-' not in f and "rules" not in f and f.endswith(".json")
            ):
        data = convert(json.load(open(os.path.join(log_dir, file))))
        if "20200503" in file:
            print(file)
        apps = ActiveApplications(data["tracker_data"])
        daily_note = data.get("daily_note") or ""
        print("%s: %s - %s = %s => %s (note: %r)" % (
            file,
            to_time(apps.begin_index()),
            to_time(apps.end_index()),
            to_time(apps.end_index() - apps.begin_index()),
            to_time(apps.end_index() - apps.begin_index() - 60),
            daily_note.split("\n")[0]))
        # print("".join(("X" if minute in apps._minutes else " ")
        #        for minute in range(apps.begin_index(), apps.end_index() + 1)))

def main(argv=None) -> int:
    """read command line arguments, configure application and run command
    specified on command line"""

    args = parse_arguments(argv or sys.argv[1:])
    util.setup_logging(args)
    util.log_system_info(args)

    args.func(args)

    '''
    args = sys.argv[1:]

    if args == []:
        print('no command provided')
        return
    elif args == ['quit']:
        request = {'type': 'quit'}

    elif args == ['version']:
        request = {'type': 'version'}

    elif args == ['apps']:
        request = {'type': 'apps'}

    elif args == ['current']:
        request = {'type': 'current'}

    elif args == ['rules']:
        request = {'type': 'rules'}

    elif args == ['save']:
        request = {'type': 'save'}

    elif args == ['help']:
        print(['quit', 'version', 'apps', 'current', 'rules'])
        sys.exit()

    else:
        raise Exception('command not handled: %s' % args)

    try:
        result = send_request(request)
        handle_result(result)
    except zmq.ZMQError as e:
        log.error(e)
        return
    except KeyboardInterrupt:
        log.info("got keyboard interrupt - exit")
    '''

if __name__ == "__main__":
    with suppress(KeyboardInterrupt, BrokenPipeError):
        raise SystemExit(main())
<commit_msg>cli: fix and allow server commands<commit_after>#!/usr/bin/env python3
# -*- encoding: utf-8 -*-
# pylint: disable=expression-not-assigned

""" track-cli - CLI interface to track

"""

import os
import sys
from contextlib import suppress
import argparse
import json
from typing import List

import zmq

from .core import util, common, ActiveApplications
from .core.util import log

def parse_arguments(argv: List[str]) -> argparse.Namespace:
    """parse command line arguments and return argument object"""
    parser = argparse.ArgumentParser(description=__doc__)
    common.setup_argument_parser(parser)

    subparsers = parser.add_subparsers(help='available commands', metavar="CMD")

    parser_help = subparsers.add_parser('help', help='show this help')
    parser_help.set_defaults(func=lambda *_: parser.print_help())

    parser_list = subparsers.add_parser('list', help='list basic logs')
    parser_list.set_defaults(func=fn_info)

    parser_server = subparsers.add_parser('server', help='send command to server')
    parser_server.set_defaults(func=fn_server)
    parser_server.add_argument("command")

    parser_show = subparsers.add_parser('show', help='show content of one log file')
    parser_show.set_defaults(func=fn_show)
    parser_show.add_argument("element", nargs="+")

    parser.set_defaults(func=lambda *a: parser.print_usage())

    return parser.parse_args(argv)

def send_request(request):
    context = zmq.Context()
    req_socket = context.socket(zmq.REQ)

    req_socket.connect('tcp://127.0.0.1:3456')

    req_socket.send_json(request)
    return req_socket.recv_json()

def handle_result(result):
    if 'type' in result and result['type'] == 'error':
        raise Exception('server replied with error: "%s"' % result['what'])
    print(result)

def convert(data):
    return data if "tracker_data" in data else {"tracker_data": data}

def to_time(value):
    return "%2d:%.2d" % (value // 60, value % 60)

def fn_server(args) -> None:
    if args.command not in {"quit", "version", "apps", "current", "rules", "save", "note"}:
        log().error("Command not known: %r", args.command)
        return

    try:
        result = send_request({"cmd": args.command})
        handle_result(result)
    except zmq.ZMQError as e:
        log.error(e)
        return

def fn_show(args) -> None:
    log_dir = args.data_dir

    log().info("Show infos for %r", args.element)
    for file in args.element:
        data = convert(json.load(open(os.path.join(log_dir, file))))
        apps = ActiveApplications(data["tracker_data"])
        daily_note = data.get("daily_note") or ""
        print("%s: %s - %s = %s => %s" % (
            file,
            to_time(apps.begin_index()),
            to_time(apps.end_index()),
            to_time(apps.end_index() - apps.begin_index()),
            to_time(apps.end_index() - apps.begin_index() - 60)))
        for time in (t for t in range(apps.begin_index(), apps.end_index()) if t in apps._minutes):
            print(to_time(time))
        print(daily_note)

def fn_info(args) -> None:
    log_dir = args.data_dir
    log().info("List recorded data")
    for file in (
            f
            for f in sorted(os.listdir(log_dir))
            if '-log-' not in f and "rules" not in f and f.endswith(".json")
            ):
        data = convert(json.load(open(os.path.join(log_dir, file))))
        if "20200503" in file:
            print(file)
        apps = ActiveApplications(data["tracker_data"])
        daily_note = data.get("daily_note") or ""
        print("%s: %s - %s = %s => %s (note: %r)" % (
            file,
            to_time(apps.begin_index()),
            to_time(apps.end_index()),
            to_time(apps.end_index() - apps.begin_index()),
            to_time(apps.end_index() - apps.begin_index() - 60),
            daily_note.split("\n")[0]))
        # print("".join(("X" if minute in apps._minutes else " ")
        #        for minute in range(apps.begin_index(), apps.end_index() + 1)))

def main(argv=None) -> int:
    """read command line arguments, configure application and run command
    specified on command line"""

    args = parse_arguments(argv or sys.argv[1:])
    util.setup_logging(args)
    common.log_system_info(args)

    args.func(args)

if __name__ == "__main__":
    with suppress(KeyboardInterrupt, BrokenPipeError):
        raise SystemExit(main())
<|endoftext|>


<commit_before>import numpy as np
import collections as col
from functools import reduce

from mast.selection import CoordArray, CoordArraySelection, \
    Point, IndexedSelection, SelectionDict, SelectionList

DIM_NUM_3D = 3

class Atom(Point):
    def __init__(self, coords=None, atom_array=None, array_idx=None, element=None):

        if not coords is None:
            assert coords.shape[-1] == DIM_NUM_3D, \
                "coords must have 3-dimensions, not {}".format(
                    coords.shape[-1])

        if atom_array:
            assert atom_array.shape[-1] == DIM_NUM_3D, \
                "atom_array must have 3-dimensions, not {}".format(
                    atom_array.shape[-1])

        super().__init__(coords=coords, coord_array=atom_array, array_idx=array_idx)
        self.element = element

        # stubs
        self.force_field = None

    def __repr__(self):
        return "Atom<{0}>{1}".format(str(self.element), self.coords)

class MoleculeType(object):
    def __init__(self):
        self._molecule = None
        self.features = None

class RDKitMoleculeType(MoleculeType):
    def __init__(self, rdkit_molecule):
        super().__init__()
        self.molecule = rdkit_molecule

    @property
    def atoms(self):
        return [atom for atom in self.molecule.GetAtoms()]

    @property
    def bonds(self):
        return [bond for bond in self.molecule.GetBonds()]

    def to_molecule(self, conformer_idx):
        """Construct a Molecule using a coordinates from a conformer of this
   rdchem.Mol.

        """
        assert self.molecule.GetNumConformers() > 0, \
            "{0} has no conformers".format(self)

        conformer = self.molecule.GetConformer(conformer_idx)
        atom_idxs = range(self.molecule.GetNumAtoms())
        # make the CoordArray
        positions = []
        for atom_idx in atom_idxs:
            position = conformer.GetAtomPosition(atom_idx)
            position = np.array([position.x, position.y, position.z])
            positions.append(position)
        positions = np.array(positions)
        coord_array = CoordArray(positions)
        # Make atoms out of the coord array
        atoms = []
        for atom_idx in atom_idxs:
            # TODO additional encapsualtion of atom data needed
            atom = Atom(atom_array=coord_array, array_idx=atom_idx)
            atoms.append(atom)

        # TODO handle bonds
        bonds = list(self.molecule.GetBonds())

        # TODO handle and create angles
        angles = None

        return Molecule(atoms, bonds, angles, mol_type=RDKitMoleculeType)

class Molecule(SelectionDict):
    def __init__(self, mol_input, *args, **kwargs):

        if 'mol_type' not in kwargs.keys():
            mol_type = None
        else:
            mol_type = kwargs.pop('mol_type')

        if issubclass(type(mol_input), MoleculeType):
            molecule_dict = Molecule.type_constructor(mol_input, *args, **kwargs)
        elif issubclass(type(mol_input), col.Sequence):
            print("atoms constructor")
            molecule_dict = Molecule.atoms_constructor(mol_input, *args, **kwargs)

        else:
            raise TypeError("mol_input must be either a MoleculeType or a sequence of Atoms")
        print(molecule_dict)
        super().__init__(selection_dict=molecule_dict)
        self._molecule_type = mol_type

    @classmethod
    def type_constructor(cls, mol_type):
        raise NotImplementedError

    @classmethod
    def atoms_constructor(cls, atoms, bonds, angles):
        assert atoms, "atoms must exist, {}".format(atoms)
        assert issubclass(type(atoms), col.Sequence), \
            "atoms must be a subclass of collections.Sequence, not {}".format(
                type(atoms))
        assert all([(lambda x: True if issubclass(type(x), Atom) else False)(atom)
                    for atom in atoms]), \
            "all elements in atoms must be a subclass of type Atom"
        molecule_dict= {'atoms' : atoms, 'bonds' : bonds, 'angles': angles}
        return molecule_dict

    @property
    def molecule_type(self):
        return self._molecule_type

    @molecule_type.setter
    def molecule_type(self, mol_type):
        assert issubclass(type(mol_type), MoleculeType), \
            "mol_type must be a subclass of MoleculeType, not {}".format(
                type(mol_type))
        self._molecule_type = mol_type

    @property
    def atoms(self):
        return self.data['atoms']

    @property
    def bonds(self):
        return self.data['bonds']

    @property
    def angles(self):
        return self.data['angles']

if __name__ == "__main__":

    print("making an CoordArray for atoms")
    array = np.array([[0,0,0], [0,0,1], [1,0,0]])
    atom_array = CoordArray(array)
    print(atom_array)

    print("making atoms")
    atom1 = Atom(np.array([5,5,5]))
    print(atom1)
    print(atom1.coords)
    atom2 = Atom(np.array([6,6,6]), element='C', atom_array=atom_array)
    print(atom2)
    print(atom2.coords)
    atom3 = Atom(atom_array=atom_array, array_idx=0)
    print(atom3)
    print(atom3.coords)
    atoms = [atom1, atom2, Atom(np.array([0,1,0]))]
    # # make a selection of those atoms
    atomsel = IndexedSelection(atoms, [0,1])
    print(atomsel)

    from rdkit import Chem
    import os.path as osp
    tspo_dir = osp.expanduser("~/Dropbox/lab/tspo")
    PKA_pdb_path = osp.join(tspo_dir, "PKA.pdb")
    pka = Chem.MolFromPDBFile(PKA_pdb_path, removeHs=False)
    print(pka)

    pka_type = RDKitMoleculeType(pka)
    print(pka_type)

    def func(arg1, *args, **kwargs):
        print(arg1)
        print(args)
        print(*args)
        print(kwargs)
        print(*kwargs)

    # make a selection of atoms for bonds, and angle
    print("making a molecule")
    bonds = [IndexedSelection(atoms, [0,1]), IndexedSelection(atoms, [1,2])]
    angles = [IndexedSelection(atoms, [0,1,2])]
    mol = Molecule(atoms, bonds, angles)
    print(mol)

    pka_mol = pka_type.to_molecule(0)
    # pka_mol = Molecule(mol_type=pka_type, coords=pka_coords)
    print(pka_mol)
    print(pka_mol.molecule_type)
<commit_msg>Tried metaclasses but failed<commit_after>import numpy as np
import collections as col
from functools import reduce
import os.path as osp

from mast.selection import CoordArray, CoordArraySelection, \
    Point, IndexedSelection, SelectionDict, SelectionList

DIM_NUM_3D = 3

class Atom(Point):
    def __init__(self, coords=None, atom_array=None, array_idx=None, element=None):

        if not coords is None:
            assert coords.shape[-1] == DIM_NUM_3D, \
                "coords must have 3-dimensions, not {}".format(
                    coords.shape[-1])

        if atom_array:
            assert atom_array.shape[-1] == DIM_NUM_3D, \
                "atom_array must have 3-dimensions, not {}".format(
                    atom_array.shape[-1])

        super().__init__(coords=coords, coord_array=atom_array, array_idx=array_idx)
        self.element = element

        # stubs
        self.force_field = None

    def __repr__(self):
        return "Atom<{0}>{1}".format(str(self.element), self.coords)

class MoleculeType(object):
    def __init__(self):
        self._molecule = None

    # @classmethod
    # def create_molecule_type(cls, mol, mol_type=None):
    #     cls_name = str(super().__self_class__).strip("'>").split('.')[-1]
    #     return type(mol_type, (cls_name,), cls(mol).__dict__)

class RDKitMoleculeType(MoleculeType):
    def __init__(self, rdkit_molecule, mol_type=None):
        super().__init__()
        self.molecule = rdkit_molecule
        # TODO convert this attribute to a class factory
        assert isinstance(mol_type, str)
        self.molecule_type = mol_type
        self._features = None

    @property
    def atoms(self):
        return [atom for atom in self.molecule.GetAtoms()]

    @property
    def bonds(self):
        return [bond for bond in self.molecule.GetBonds()]

    @property
    def features(self):
        return self._features

    def find_features(self, fdef="BaseFeatures.fdef"):
        """Uses a feature definition (fdef) database to to find features in
the molecule.

        """
        from rdkit import RDConfig
        from rdkit.Chem import ChemicalFeatures

        assert isinstance(fdef, str)
        fdef_path = osp.join(RDConfig.RDDataDir, fdef)
        feature_factory = ChemicalFeatures.BuildFeatureFactory(fdef_path)
        factory_features = feature_factory.GetFeaturesForMol(self.molecule)
        features = {}
        for feature in factory_features:
            feature_info = {'family' : feature.GetFamily(),
                            'type' : feature.GetType(),
                            'atom_ids' : feature.GetAtomIds(),
                            'position' : feature.GetPos()}
            features[feature.GetId()] = feature_info

        return features

    def to_molecule(self, conformer_idx):
        """Construct a Molecule using a coordinates from a conformer of this
   rdchem.Mol.

        """
        assert self.molecule.GetNumConformers() > 0, \
            "{0} has no conformers".format(self)

        conformer = self.molecule.GetConformer(conformer_idx)
        atom_idxs = range(self.molecule.GetNumAtoms())
        # make the CoordArray
        positions = []
        for atom_idx in atom_idxs:
            position = conformer.GetAtomPosition(atom_idx)
            position = np.array([position.x, position.y, position.z])
            positions.append(position)
        positions = np.array(positions)
        coord_array = CoordArray(positions)
        # Make atoms out of the coord array
        atoms = []
        for atom_idx in atom_idxs:
            # TODO additional encapsualtion of atom data needed
            atom = Atom(atom_array=coord_array, array_idx=atom_idx)
            atoms.append(atom)

        # TODO handle bonds
        bonds = list(self.molecule.GetBonds())

        # TODO handle and create angles
        angles = None

        return Molecule(atoms, bonds, angles, mol_type=self.molecule_type, external_mol_rep=(RDKitMoleculeType, self))

class Molecule(SelectionDict):
    def __init__(self, mol_input, *args, **kwargs):

        if 'mol_type' not in kwargs.keys():
            mol_type = None
        else:
            mol_type = kwargs.pop('mol_type')

        if 'external_mol_rep' not in kwargs.keys():
            external_mol_rep = None
        else:
            assert isinstance(kwargs['external_mol_rep'], tuple), \
                "An external_mol_rep must be a tuple (external_type, external_mol), not {}".format(
                    kwargs['external_mol_rep'])
            external_mol_rep = kwargs.pop('external_mol_rep')

        if issubclass(type(mol_input), MoleculeType):
            molecule_dict = Molecule.type_constructor(mol_input, *args, **kwargs)
        elif issubclass(type(mol_input), col.Sequence):
            print("atoms constructor")
            molecule_dict = Molecule.atoms_constructor(mol_input, *args, **kwargs)
        else:
            raise TypeError("mol_input must be either a MoleculeType or a sequence of Atoms")

        super().__init__(selection_dict=molecule_dict)
        self._features = None
        self._molecule_type = mol_type
        # a dictionary of molecular representations from other libraries
        if external_mol_rep:
            self._external_mol_reps = {external_mol_rep[0] : external_mol_rep[1]}

    @classmethod
    def type_constructor(cls, mol_type):
        raise NotImplementedError

    @classmethod
    def atoms_constructor(cls, atoms, bonds, angles):
        assert atoms, "atoms must exist, {}".format(atoms)
        assert issubclass(type(atoms), col.Sequence), \
            "atoms must be a subclass of collections.Sequence, not {}".format(
                type(atoms))
        assert all([(lambda x: True if issubclass(type(x), Atom) else False)(atom)
                    for atom in atoms]), \
            "all elements in atoms must be a subclass of type Atom"
        molecule_dict= {'atoms' : atoms, 'bonds' : bonds, 'angles': angles}
        return molecule_dict

    @property
    def molecule_type(self):
        return self._molecule_type

    @molecule_type.setter
    def molecule_type(self, mol_type):
        assert issubclass(type(mol_type), MoleculeType), \
            "mol_type must be a subclass of MoleculeType, not {}".format(
                type(mol_type))
        self._molecule_type = mol_type

    @property
    def atoms(self):
        return self.data['atoms']

    @property
    def bonds(self):
        return self.data['bonds']

    @property
    def angles(self):
        return self.data['angles']

    @property
    def external_mol_reps(self):
        return self._external_mol_reps

    @property
    def features(self):
        return self._features

    # TODO should be in the Type class for the molecule not the
    # molecule itself
    def find_features(self):
        features = self._external_mol_reps[RDKitMoleculeType].find_features()
        self._features = features

if __name__ == "__main__":

    print("making an CoordArray for atoms")
    array = np.array([[0,0,0], [0,0,1], [1,0,0]])
    atom_array = CoordArray(array)
    print(atom_array)

    print("making atoms")
    atom1 = Atom(np.array([5,5,5]))
    print(atom1)
    print(atom1.coords)
    atom2 = Atom(np.array([6,6,6]), element='C', atom_array=atom_array)
    print(atom2)
    print(atom2.coords)
    atom3 = Atom(atom_array=atom_array, array_idx=0)
    print(atom3)
    print(atom3.coords)
    atoms = [atom1, atom2, Atom(np.array([0,1,0]))]
    # # make a selection of those atoms
    atomsel = IndexedSelection(atoms, [0,1])
    print(atomsel)

    from rdkit import Chem
    import os.path as osp
    tspo_dir = osp.expanduser("~/Dropbox/lab/tspo")
    PKA_pdb_path = osp.join(tspo_dir, "PKA.pdb")
    pka = Chem.MolFromPDBFile(PKA_pdb_path, removeHs=False)
    print(pka)

    # PKA = RDKitMoleculeType.create_molecule_type(pka, mol_type='PKA')

    # PKA = RDKitMoleculeType.create_molecule_type(
    pka_type = RDKitMoleculeType(pka, mol_type="PKA")
    print(pka_type)

    # make a selection of atoms for bonds, and angle
    print("making a molecule")
    bonds = [IndexedSelection(atoms, [0,1]), IndexedSelection(atoms, [1,2])]
    angles = [IndexedSelection(atoms, [0,1,2])]
    mol = Molecule(atoms, bonds, angles)
    print(mol)

    print("Making a mast.Molecule from the RDKitMoleculeType")
    pka_mol = pka_type.to_molecule(0)
    # pka_mol = Molecule(mol_type=pka_type, coords=pka_coords)
    print(pka_mol)
    print(pka_mol.molecule_type)

    print("Getting features for the molecule from it's RDKit representation")
    pka_mol_rdmol = pka_mol.external_mol_reps[RDKitMoleculeType]
    pka_rdfeatures = pka_mol_rdmol.find_features()
    print(pka_rdfeatures)

    print("finding features using amst.molecule method")
    pka_mol.find_features()
    print(pka_mol.features)
<|endoftext|>


<commit_before>"""
Error handlers for errors originating from the Submission systems.
"""

from __future__ import division

__author__ = "Michiel van Setten"
__copyright__ = " "
__version__ = "0.9"
__maintainer__ = "Michiel van Setten"
__email__ = "mjvansetten@gmail.com"
__date__ = "Mar 24, 2014"

import re
from abc import ABCMeta, abstractmethod, abstractproperty
from pymatgen.io.gwsetup.scheduler_error_parsers import AbstractError
from pymatgen.io.gwsetup.scheduler_error_parsers import TimeCancelError, MemoryCancelError, NodeFailureError

<commit_msg>cleaning up<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python3

from __future__ import print_function
import numpy as np
# np.random.seed(1337)  # for reproducibility
# SEED = 1337

import os
import ast
import pickle
import configparser
from time import localtime, strftime
from optparse import OptionParser

import keras
from keras.datasets import mnist
from keras.utils import np_utils
from keras import backend as K
from keras.optimizers import SGD

from bird.models.cuberun import CubeRun
from bird.models.resnet import ResNetBuilder
from bird.generators.sound import SoundDataGenerator

# Setup Callbacks for History
class HistoryCollector(keras.callbacks.Callback):
    def __init__(self, name):
        self.name = name
        self.data = []

    def on_epoch_end(self, batch, logs={}):
        self.data.append(logs.get(self.name))

def train_model(config_file, weight_file_path, history_file_path, first_epoch, lock_file):

    config_parser = configparser.ConfigParser()
    config_parser.read(config_file)

    # model
    batch_size = int(config_parser['MODEL']['BatchSize'])
    nb_classes = int(config_parser['MODEL']['NumberOfClasses'])
    nb_epoch = int(config_parser['MODEL']['NumberOfEpochs'])
    nb_iterations = int(config_parser['MODEL']['NumberOfIterations'])
    nb_val_samples = int(config_parser['MODEL']['NumberOfValidationSamplesPerEpoch'])
    samples_per_epoch = int(config_parser['MODEL']['NumberOfTrainingSamplesPerEpoch'])
    input_shape = ast.literal_eval(config_parser['MODEL']['InputShape'])
    model_name = config_parser['MODEL']['ModelName']
    audio_mode = config_parser['MODEL']['InputDataMode']

    # paths
    noise_path = config_parser['PATHS']['NoiseDataDir']
    train_path = config_parser['PATHS']['TrainingDataDir']
    valid_path = config_parser['PATHS']['ValidationDataDir']

    # training
    optimizer = config_parser['TRAINING']['Optimizer']
    learning_rate = float(config_parser['TRAINING']['LearningRate'])
    decay = float(config_parser['TRAINING']['Decay'])
    momentum = float(config_parser['TRAINING']['Momentum'])
    nesterov = config_parser['TRAINING']['Nesterov'] == 'True'
    loss_function = config_parser['TRAINING']['LossFunction']
    time_shift = config_parser['TRAINING']['TimeShiftAugmentation'] == 'True'
    pitch_shift = config_parser['TRAINING']['PitchShiftAugmentation'] == 'True'
    same_class_augmentation = config_parser['TRAINING']['SameClassAugmentation'] == 'True'
    noise_augmentation = config_parser['TRAINING']['NoiseAugmentation'] == 'True'

    img_rows, img_cols, nb_channels = input_shape

    model = None
    if model_name == 'cuberun':
	    model = CubeRun(nb_classes, input_shape)
    elif model_name == 'resnet_18':
            model = ResNetBuilder.build_resnet_18(input_shape, nb_classes)
    elif model_name == 'resnet_34':
            model = ResNetBuilder.build_resnet_34(input_shape, nb_classes)
    elif model_name == 'resnet_50':
            model = ResNetBuilder.build_resnet_50(input_shape, nb_classes)
    elif model_name == 'resnet_101':
            model = ResNetBuilder.build_resnet_101(input_shape, nb_classes)
    elif model_name == 'resnet_152':
            model = ResNetBuilder.build_resnet_152(input_shape, nb_classes)
    else:
        raise ValueError("Can not find model ", model_name, ".")

    if optimizer == 'sgd':
        sgd = SGD(lr=learning_rate, decay=decay, momentum=momentum,
                  nesterov=nesterov)
        model.compile(loss=loss_function,
                      optimizer=sgd,
                      metrics=['accuracy'])
    else:
        model.compile(loss=loss_function,
                      optimizer=optimizer,
                      metrics=['accuracy'])

    if first_epoch=='False':
        # load weights
        model.load_weights(weight_file_path)
        print("loading weigths from: " + weight_file_path)
    else:
        print("using initial weights")

    # Callback history collectors
    trainLossHistory = HistoryCollector('loss')
    validLossHistory = HistoryCollector('val_loss')
    trainAccHistory = HistoryCollector('acc')
    validAccHistory = HistoryCollector('val_acc')
    checkpoint = keras.callbacks.ModelCheckpoint(weight_file_path,
                                                monitor='val_acc', verbose=0,
                                                save_best_only=True,
                                                save_weights_only=True,
                                                mode='auto')

    # train data generator
    train_datagen = SoundDataGenerator(
        rescale=1./255,
        time_shift=time_shift,
        pitch_shift=pitch_shift,
        augment_with_same_class=same_class_augmentation,
        augment_with_noise=noise_augmentation)

    # validation data generator
    valid_datagen = SoundDataGenerator(
        rescale=1./255)

    # Generator for training data
    print("Loading training data...")
    train_generator = train_datagen.flow_from_directory(
        train_path,
        noise_path,
        target_size=(img_rows, img_cols),
        batch_size=batch_size,
        class_mode='categorical',
        audio_mode=audio_mode
        #save_to_dir='./visuals/augmented_samples'
        )

    # Generator for validation data
    print("Loading validation data...")
    valid_generator = valid_datagen.flow_from_directory(
        valid_path,
        noise_path,
        target_size=(img_rows, img_cols),
        batch_size=batch_size,
        class_mode='categorical',
        audio_mode=audio_mode
        #save_to_dir='./visuals/validation_samples',
        )

    # Fit model to generated training data
    model.fit_generator(
        train_generator,
        samples_per_epoch=samples_per_epoch,
        nb_epoch=nb_epoch,
        validation_data=valid_generator,
        nb_val_samples=nb_val_samples,
        callbacks=[trainLossHistory, validLossHistory, trainAccHistory,
                   validAccHistory, checkpoint])

    # save the weights
    model.save_weights(weight_file_path)
    print (strftime("%a, %d %b %Y %H:%M:%S +0000", localtime()))
    print ("The weights have been saved in: " + weight_file_path)

    # save history to file
    with open(history_file_path, 'wb') as output:
        pickle.dump(trainLossHistory.data, output, pickle.HIGHEST_PROTOCOL)
        pickle.dump(validLossHistory.data, output, pickle.HIGHEST_PROTOCOL)
        pickle.dump(trainAccHistory.data, output, pickle.HIGHEST_PROTOCOL)
        pickle.dump(validAccHistory.data, output, pickle.HIGHEST_PROTOCOL)
    print ("The history has been saved in: " + history_file_path)

    os.remove(lock_file)

    return weight_file_path

parser = OptionParser()
parser.add_option("--config_file", dest="config_file")
parser.add_option("--history_path", dest="history_path")
parser.add_option("--weight_path", dest="weight_path")
parser.add_option("--first_epoch", dest="first_epoch")
parser.add_option("--lock_file", dest="lock_file")

(options, args) = parser.parse_args()
config_file = options.config_file
history_file_path = options.history_path
weight_file_path = options.weight_path
first_epoch = options.first_epoch
lock_file = options.lock_file

train_model(config_file, weight_file_path, history_file_path, first_epoch, lock_file)
<commit_msg>Save best weights to separate file<commit_after>#!/usr/bin/env python3

from __future__ import print_function
import numpy as np
# np.random.seed(1337)  # for reproducibility
# SEED = 1337

import os
import ast
import pickle
import configparser
from time import localtime, strftime
from optparse import OptionParser

import keras
from keras.datasets import mnist
from keras.utils import np_utils
from keras import backend as K
from keras.optimizers import SGD

from bird.models.cuberun import CubeRun
from bird.models.resnet import ResNetBuilder
from bird.generators.sound import SoundDataGenerator

# Setup Callbacks for History
class HistoryCollector(keras.callbacks.Callback):
    def __init__(self, name):
        self.name = name
        self.data = []

    def on_epoch_end(self, batch, logs={}):
        self.data.append(logs.get(self.name))

def train_model(config_file, weight_file_path, history_file_path, first_epoch, lock_file):

    config_parser = configparser.ConfigParser()
    config_parser.read(config_file)

    # model
    batch_size = int(config_parser['MODEL']['BatchSize'])
    nb_classes = int(config_parser['MODEL']['NumberOfClasses'])
    nb_epoch = int(config_parser['MODEL']['NumberOfEpochs'])
    nb_iterations = int(config_parser['MODEL']['NumberOfIterations'])
    nb_val_samples = int(config_parser['MODEL']['NumberOfValidationSamplesPerEpoch'])
    samples_per_epoch = int(config_parser['MODEL']['NumberOfTrainingSamplesPerEpoch'])
    input_shape = ast.literal_eval(config_parser['MODEL']['InputShape'])
    model_name = config_parser['MODEL']['ModelName']
    audio_mode = config_parser['MODEL']['InputDataMode']

    # paths
    noise_path = config_parser['PATHS']['NoiseDataDir']
    train_path = config_parser['PATHS']['TrainingDataDir']
    valid_path = config_parser['PATHS']['ValidationDataDir']

    # training
    optimizer = config_parser['TRAINING']['Optimizer']
    learning_rate = float(config_parser['TRAINING']['LearningRate'])
    decay = float(config_parser['TRAINING']['Decay'])
    momentum = float(config_parser['TRAINING']['Momentum'])
    nesterov = config_parser['TRAINING']['Nesterov'] == 'True'
    loss_function = config_parser['TRAINING']['LossFunction']
    time_shift = config_parser['TRAINING']['TimeShiftAugmentation'] == 'True'
    pitch_shift = config_parser['TRAINING']['PitchShiftAugmentation'] == 'True'
    same_class_augmentation = config_parser['TRAINING']['SameClassAugmentation'] == 'True'
    noise_augmentation = config_parser['TRAINING']['NoiseAugmentation'] == 'True'

    img_rows, img_cols, nb_channels = input_shape

    model = None
    if model_name == 'cuberun':
	    model = CubeRun(nb_classes, input_shape)
    elif model_name == 'resnet_18':
            model = ResNetBuilder.build_resnet_18(input_shape, nb_classes)
    elif model_name == 'resnet_34':
            model = ResNetBuilder.build_resnet_34(input_shape, nb_classes)
    elif model_name == 'resnet_50':
            model = ResNetBuilder.build_resnet_50(input_shape, nb_classes)
    elif model_name == 'resnet_101':
            model = ResNetBuilder.build_resnet_101(input_shape, nb_classes)
    elif model_name == 'resnet_152':
            model = ResNetBuilder.build_resnet_152(input_shape, nb_classes)
    else:
        raise ValueError("Can not find model ", model_name, ".")

    if optimizer == 'sgd':
        sgd = SGD(lr=learning_rate, decay=decay, momentum=momentum,
                  nesterov=nesterov)
        model.compile(loss=loss_function,
                      optimizer=sgd,
                      metrics=['accuracy'])
    else:
        model.compile(loss=loss_function,
                      optimizer=optimizer,
                      metrics=['accuracy'])

    if first_epoch=='False':
        # load weights
        model.load_weights(weight_file_path)
        print("loading weigths from: " + weight_file_path)
    else:
        print("using initial weights")

    # Callback history collectors
    trainLossHistory = HistoryCollector('loss')
    validLossHistory = HistoryCollector('val_loss')
    trainAccHistory = HistoryCollector('acc')
    validAccHistory = HistoryCollector('val_acc')
    best_weight_file_path = os.path.join(os.path.dirname(weight_file_path), "best_weights.h5")
    checkpoint = keras.callbacks.ModelCheckpoint(best_weight_file_path,
                                                monitor='val_acc', verbose=0,
                                                save_best_only=True,
                                                save_weights_only=True,
                                                mode='auto')

    # train data generator
    train_datagen = SoundDataGenerator(
        rescale=1./255,
        time_shift=time_shift,
        pitch_shift=pitch_shift,
        augment_with_same_class=same_class_augmentation,
        augment_with_noise=noise_augmentation)

    # validation data generator
    valid_datagen = SoundDataGenerator(
        rescale=1./255)

    # Generator for training data
    print("Loading training data...")
    train_generator = train_datagen.flow_from_directory(
        train_path,
        noise_path,
        target_size=(img_rows, img_cols),
        batch_size=batch_size,
        class_mode='categorical',
        audio_mode=audio_mode
        #save_to_dir='./visuals/augmented_samples'
        )

    # Generator for validation data
    print("Loading validation data...")
    valid_generator = valid_datagen.flow_from_directory(
        valid_path,
        noise_path,
        target_size=(img_rows, img_cols),
        batch_size=batch_size,
        class_mode='categorical',
        audio_mode=audio_mode
        #save_to_dir='./visuals/validation_samples',
        )

    # Fit model to generated training data
    model.fit_generator(
        train_generator,
        samples_per_epoch=samples_per_epoch,
        nb_epoch=nb_epoch,
        validation_data=valid_generator,
        nb_val_samples=nb_val_samples,
        callbacks=[trainLossHistory, validLossHistory, trainAccHistory,
                   validAccHistory, checkpoint])

    # save the weights
    model.save_weights(weight_file_path)
    print (strftime("%a, %d %b %Y %H:%M:%S +0000", localtime()))
    print ("The weights have been saved in: " + weight_file_path)

    # save history to file
    with open(history_file_path, 'wb') as output:
        pickle.dump(trainLossHistory.data, output, pickle.HIGHEST_PROTOCOL)
        pickle.dump(validLossHistory.data, output, pickle.HIGHEST_PROTOCOL)
        pickle.dump(trainAccHistory.data, output, pickle.HIGHEST_PROTOCOL)
        pickle.dump(validAccHistory.data, output, pickle.HIGHEST_PROTOCOL)
    print ("The history has been saved in: " + history_file_path)

    os.remove(lock_file)

    return weight_file_path

parser = OptionParser()
parser.add_option("--config_file", dest="config_file")
parser.add_option("--history_path", dest="history_path")
parser.add_option("--weight_path", dest="weight_path")
parser.add_option("--first_epoch", dest="first_epoch")
parser.add_option("--lock_file", dest="lock_file")

(options, args) = parser.parse_args()
config_file = options.config_file
history_file_path = options.history_path
weight_file_path = options.weight_path
first_epoch = options.first_epoch
lock_file = options.lock_file

train_model(config_file, weight_file_path, history_file_path, first_epoch, lock_file)
<|endoftext|>


<commit_before>"""Applies an OSMChange file to the database"""

import sys
import os
import time
from datetime import datetime
from xml.sax import make_parser
from xml.sax.handler import ContentHandler
from pymongo import Connection

class OsmChangeHandler(ContentHandler):
    """This ContentHandler works with the OSMChange XML file"""
    def __init__(self, client):
        """Initializes the OsmChange object"""
        self.action = ""
        self.record = {}
        self.nodes = []
        self.ways = []
        self.relations = []
        self.client = client

    def fillDefault(self, attrs):
        """Fills in default attributes for new records"""
        self.record['id'] = long(attrs['id'])
        self.record['timestamp'] = self.isoToTimestamp(attrs['timestamp'])
        self.record['tags'] = {}
        if attrs.has_key('user'):
            self.record['user'] = attrs['user']
        if attrs.has_key('uid'):
            self.record['uid'] = long(attrs['uid'])
        if attrs.has_key('version'):
            self.record['version'] = int(attrs['version'])
        if attrs.has_key('changeset'):
            self.record['changeset'] = long(attrs['changeset'])

    def isoToTimestamp(self, isotime):
        """Returns an time tuple from the time string"""
        t = datetime.strptime(isotime, "%Y-%m-%dT%H:%M:%SZ")
        return time.mktime(t.timetuple())

    def startElement(self, name, attrs):
        """Parse the XML element at the start"""
        if name in ['create', 'modify', 'delete']:
            self.action = name
        elif name == 'node':
            self.fillDefault(attrs)
            self.record['loc'] = {'lat': float(attrs['lat']),                                  'lon': float(attrs['lon'])}
        elif name == 'tag':
            # MongoDB doesn't let us have dots in the key names.
            k = attrs['k']
            k = k.replace('.', ',,')
            self.record['tags'][k] = attrs['v']
        elif name == 'way':
            self.fillDefault(attrs)
            self.record['nodes'] = []
        elif name == 'relation':
            self.fillDefault(attrs)
            self.record['members'] = []
        elif name == 'nd':
            ref = long(attrs['ref'])
            self.record['nodes'].append(ref)

            nodes2ways = self.client.osm.nodes.find_one({ 'id' : ref })
            if nodes2ways:
                if 'ways' not in nodes2ways:
                    nodes2ways['ways'] = []
                nodes2ways['ways'].append(self.record['id'])
                self.client.osm.nodes.save(nodes2ways)
            else:
                print "Node %d ref'd by way %d not in file." % \
                    (ref, self.record['id'])
        elif name == 'member':
            ref = long(attrs['ref'])
            member = {'type': attrs['type'],
                      'ref':  ref,
                      'role': attrs['role']}
            self.record['members'].append(member)

            if attrs['type'] == 'way':
                ways2relations = self.client.osm.ways.find_one({ 'id' : ref})
                if ways2relations:
                    if 'relations' not in ways2relations:
                        ways2relations['relations'] = []
                    ways2relations['relations'].append(self.record['id'])
                    self.client.osm.ways.save(ways2relations)
            elif attrs['type'] == 'node':
                nodes2relations = self.client.osm.nodes.find_one({ 'id' : ref})
                if nodes2relations:
                    if 'relations' not in nodes2relations:
                        nodes2relations['relations'] = []
                    nodes2relations['relations'].append(self.record['id'])
                    self.client.osm.nodes.save(nodes2relations)
        elif name == 'node':
            self.record['loc'] = {'lat': float(attrs['lat']),
                                  'lon': float(attrs['lon'])}
            self.fillDefault(attrs)

        elif name == 'tag':
            # MongoDB doesn't let us have dots in the key names.
            k = attrs['k']
            k = k.replace('.', ',,')
            self.record['tags'][k] = attrs['v']
        elif name == 'way':
            self.fillDefault(attrs)
            self.record['nodes'] = []
        elif name == 'relation':
            self.fillDefault(attrs)
            self.record['members'] = []
        elif name == 'nd':
            ref = long(attrs['ref'])
            self.record['nodes'].append(ref)
        elif name == 'member':
            ref = long(attrs['ref'])
            member = {'type': attrs['type'],
                      'ref':  ref,
                      'role': attrs['role']}
            self.record['members'].append(member)

    def create(self):
        """Takes the current items in the record queue and creates
        them in the database"""
        pass
    def modify(self):
        """Takes the current items in the record queue and modifies
        them in the database.

        Currently that means to make a new copy of the record,
        ignoring the old version"""
        pass
    def delete(self):
        """Takes the current items in the record queue and delete them
        from the database.

        Currently that means to set visable=false, rather than
        actually delete them."""

    def endElement(self, name):
        """Finish parsing osm objects or actions"""
        if name in ('node', 'way', 'relation'):
            getattr(self, name + 's').append(self.record)
            self.record = {}
        elif name == 'delete':
            ## This is all doable with list comprehensions but this is
            ## easier to read
            for coll in ('nodes', 'ways', 'relations'):
                for rec in getattr(self, coll):
                    rec['visable'] = False
        elif name in ('create', 'modify', 'delete'):
            self.action = None
            for coll in ('nodes', 'ways', 'relations'):
                if getattr(self, coll):
                    getattr(self.client.osm, coll).insert(getattr(self, coll))
                    setattr(self, coll, [])

if __name__ == "__main__":
    filename = sys.argv[1]

    if not os.path.exists(filename):
        print "Path %s doesn't exist." % (filename)
        sys.exit(-1)

    client = Connection()
    parser = make_parser()
    handler = OsmChangeHandler(client)
    parser.setContentHandler(handler)
    parser.parse(open(filename))
    client.disconnect()
<commit_msg>Removed unused methods<commit_after>"""Applies an OSMChange file to the database"""

import sys
import os
import time
from datetime import datetime
from xml.sax import make_parser
from xml.sax.handler import ContentHandler
from pymongo import Connection

class OsmChangeHandler(ContentHandler):
    """This ContentHandler works with the OSMChange XML file"""
    def __init__(self, client):
        """Initializes the OsmChange object"""
        self.action = ""
        self.record = {}
        self.nodes = []
        self.ways = []
        self.relations = []
        self.client = client

    def fillDefault(self, attrs):
        """Fills in default attributes for new records"""
        self.record['id'] = long(attrs['id'])
        self.record['timestamp'] = self.isoToTimestamp(attrs['timestamp'])
        self.record['tags'] = {}
        if attrs.has_key('user'):
            self.record['user'] = attrs['user']
        if attrs.has_key('uid'):
            self.record['uid'] = long(attrs['uid'])
        if attrs.has_key('version'):
            self.record['version'] = int(attrs['version'])
        if attrs.has_key('changeset'):
            self.record['changeset'] = long(attrs['changeset'])

    def isoToTimestamp(self, isotime):
        """Returns an time tuple from the time string"""
        t = datetime.strptime(isotime, "%Y-%m-%dT%H:%M:%SZ")
        return time.mktime(t.timetuple())

    def startElement(self, name, attrs):
        """Parse the XML element at the start"""
        if name in ['create', 'modify', 'delete']:
            self.action = name
        elif name == 'node':
            self.fillDefault(attrs)
            self.record['loc'] = {'lat': float(attrs['lat']),                                  'lon': float(attrs['lon'])}
        elif name == 'tag':
            # MongoDB doesn't let us have dots in the key names.
            k = attrs['k']
            k = k.replace('.', ',,')
            self.record['tags'][k] = attrs['v']
        elif name == 'way':
            self.fillDefault(attrs)
            self.record['nodes'] = []
        elif name == 'relation':
            self.fillDefault(attrs)
            self.record['members'] = []
        elif name == 'nd':
            ref = long(attrs['ref'])
            self.record['nodes'].append(ref)

            nodes2ways = self.client.osm.nodes.find_one({ 'id' : ref })
            if nodes2ways:
                if 'ways' not in nodes2ways:
                    nodes2ways['ways'] = []
                nodes2ways['ways'].append(self.record['id'])
                self.client.osm.nodes.save(nodes2ways)
            else:
                print "Node %d ref'd by way %d not in file." % \
                    (ref, self.record['id'])
        elif name == 'member':
            ref = long(attrs['ref'])
            member = {'type': attrs['type'],
                      'ref':  ref,
                      'role': attrs['role']}
            self.record['members'].append(member)

            if attrs['type'] == 'way':
                ways2relations = self.client.osm.ways.find_one({ 'id' : ref})
                if ways2relations:
                    if 'relations' not in ways2relations:
                        ways2relations['relations'] = []
                    ways2relations['relations'].append(self.record['id'])
                    self.client.osm.ways.save(ways2relations)
            elif attrs['type'] == 'node':
                nodes2relations = self.client.osm.nodes.find_one({ 'id' : ref})
                if nodes2relations:
                    if 'relations' not in nodes2relations:
                        nodes2relations['relations'] = []
                    nodes2relations['relations'].append(self.record['id'])
                    self.client.osm.nodes.save(nodes2relations)
        elif name == 'node':
            self.record['loc'] = {'lat': float(attrs['lat']),
                                  'lon': float(attrs['lon'])}
            self.fillDefault(attrs)

        elif name == 'tag':
            # MongoDB doesn't let us have dots in the key names.
            k = attrs['k']
            k = k.replace('.', ',,')
            self.record['tags'][k] = attrs['v']
        elif name == 'way':
            self.fillDefault(attrs)
            self.record['nodes'] = []
        elif name == 'relation':
            self.fillDefault(attrs)
            self.record['members'] = []
        elif name == 'nd':
            ref = long(attrs['ref'])
            self.record['nodes'].append(ref)
        elif name == 'member':
            ref = long(attrs['ref'])
            member = {'type': attrs['type'],
                      'ref':  ref,
                      'role': attrs['role']}
            self.record['members'].append(member)

    def endElement(self, name):
        """Finish parsing osm objects or actions"""
        if name in ('node', 'way', 'relation'):
            getattr(self, name + 's').append(self.record)
            self.record = {}
        elif name == 'delete':
            ## This is all doable with list comprehensions but this is
            ## easier to read
            for coll in ('nodes', 'ways', 'relations'):
                for rec in getattr(self, coll):
                    rec['visable'] = False
        elif name in ('create', 'modify', 'delete'):
            self.action = None
            for coll in ('nodes', 'ways', 'relations'):
                if getattr(self, coll):
                    getattr(self.client.osm, coll).insert(getattr(self, coll))
                    setattr(self, coll, [])

if __name__ == "__main__":
    filename = sys.argv[1]

    if not os.path.exists(filename):
        print "Path %s doesn't exist." % (filename)
        sys.exit(-1)

    client = Connection()
    parser = make_parser()
    handler = OsmChangeHandler(client)
    parser.setContentHandler(handler)
    parser.parse(open(filename))
    client.disconnect()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright © 2013 Jens Schmer, Michael Engelhard

from Position import Position

class Move(object):
    """description of class"""

    def __init__(self, fr, to):
        self.start = fr
        self.end   = to

    def __str__(self):
        return str(self.start) + " -> " + str(self.end)

<commit_msg>position import not needed...<commit_after># -*- coding: utf-8 -*-
# Copyright © 2013 Jens Schmer, Michael Engelhard

class Move(object):
    """description of class"""

    def __init__(self, fr, to):
        self.start = fr
        self.end   = to

    def __str__(self):
        return str(self.start) + " -> " + str(self.end)

<|endoftext|>


<commit_before>
# Copyright (C) 2012 Victor Semionov
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#  * Redistributions of source code must retain the above copyright notice, this
#    list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#  * Neither the name of the copyright holder nor the names of the contributors
#    may be used to endorse or promote products derived from this software
#    without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import model

version = 1.0

# physical parameters:

lasing_wavelen = 1064.0e-9

dopant_xsection = 2.8e-23
dopant_upper_lifetime = 230.0e-6
dopant_lower_lifetime = (115.0e-12 + 225.0e-12) / 2.0
dopant_branching_ratio = 0.56
dopant_concentration = 1.38e26

medium_radius = 2.0e-3
medium_length = 100.0e-3
medium_refr_idx = 1.82

initial_inversion = 0.0

pump_wavelen = 808.0e-9
pump_duration = 250.0e-6
pump_power = 4000.0
pump_efficiency = 0.5

depop_model_class = model.depop.RossApproximateASEModel

beam_class = model.beam.GaussianBeam
beam_radius = 2.0e-3

pulse_class = model.pulse.GaussianPulse
pulse_energy = 1.0e-6
pulse_duration = 6.0e-12

train_pulse_count = 1
train_pulse_period = 1.0e-9

# numerical parameters:

depop_rate_rtol = 5.0e-2
depop_rate_min_samples = 4096
depop_model_extra_args = {}

inverter_class = model.inverter.RungeKuttaInverter # RungeKuttaInverter usually makes fewer derivative evaluations for RossNumericalASEModel; for other depopulation models EulerInverter seems faster
inversion_rtol = 1.0e-2
inversion_min_count_t = 0
inversion_validate = False

ext_depop_models = [model.depop.FluorescenceModel, model.depop.RossApproximateASEModel, model.depop.RossNumericalASEModel]
ext_alt_depop_model = model.depop.FluorescenceModel
ext_opt_inversion_rdiff_max = 0.1
ext_opt_fluence_max = 1.0e4
ext_opt_pump_duration = (dopant_upper_lifetime / 2.0, dopant_upper_lifetime * 2.0)
ext_opt_pump_power = (2.0e3, 8.0e3)
ext_opt_pump_resolution = (24, 24)
ext_opt_geom_mediumradius = (medium_radius / 2.0, medium_radius * 2.0)
ext_opt_geom_beamradius = (beam_radius / 2.0, beam_radius * 2.0)
ext_opt_geom_resolution = (24, 24)

time_trunc_rtol = 1.0e-2
fluence_rtol = 1.0e-6
energy_rtol = 1.0e-2
amp_rtol = 2.0e-2

min_count_rho = 0
min_count_phi = 0
min_count_z = 0
min_count_t = 0

integrator_classes = [model.integrator.SimpsonIntegrator]
amplifier_classes = [model.amplifier.HybridAmplifier, model.amplifier.NSFDAmplifier] # HybridAmplifier is faster and may produce less error in some cases; NSFDAmplifier is unconditionally stable and produces qualitatively correct solutions (with all expected properties)

# output parameters:

amplification = True

extended_mode = False
graphs = False
output_data = False

verbose = False

num_tasks = 0

lower_process_priority = True

pulse_num_stride = 100
extended_status_strides = (4, 0)

out_markers_step_divisor = 5
out_rho_steps_divisor = 33
out_phi_steps_divisor = 65
out_z_steps_divisor = 33
out_t_steps_divisor = 33

num_auto_contours = 5

output_rel_time = False
output_color = True
output_styled_lines = False
output_plot_titles = True

output_file_format = "png"
<commit_msg>adjustment<commit_after>
# Copyright (C) 2012 Victor Semionov
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#  * Redistributions of source code must retain the above copyright notice, this
#    list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#  * Neither the name of the copyright holder nor the names of the contributors
#    may be used to endorse or promote products derived from this software
#    without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import model

version = 1.0

# physical parameters:

lasing_wavelen = 1064.0e-9

dopant_xsection = 2.8e-23
dopant_upper_lifetime = 230.0e-6
dopant_lower_lifetime = (115.0e-12 + 225.0e-12) / 2.0
dopant_branching_ratio = 0.56
dopant_concentration = 1.38e26

medium_radius = 2.0e-3
medium_length = 100.0e-3
medium_refr_idx = 1.82

initial_inversion = 0.0

pump_wavelen = 808.0e-9
pump_duration = 250.0e-6
pump_power = 4000.0
pump_efficiency = 0.5

depop_model_class = model.depop.RossApproximateASEModel

beam_class = model.beam.GaussianBeam
beam_radius = 2.0e-3

pulse_class = model.pulse.GaussianPulse
pulse_energy = 1.0e-6
pulse_duration = 6.0e-12

train_pulse_count = 1
train_pulse_period = 1.0e-9

# numerical parameters:

depop_rate_rtol = 5.0e-2
depop_rate_min_samples = 4096
depop_model_extra_args = {}

inverter_class = model.inverter.RungeKuttaInverter # RungeKuttaInverter usually makes fewer derivative evaluations for RossNumericalASEModel; for other depopulation models EulerInverter seems faster
inversion_rtol = 1.0e-2
inversion_min_count_t = 0
inversion_validate = False

ext_depop_models = [model.depop.FluorescenceModel, model.depop.RossApproximateASEModel, model.depop.RossNumericalASEModel]
ext_alt_depop_model = model.depop.FluorescenceModel
ext_opt_inversion_rdiff_max = 0.1
ext_opt_fluence_max = 1.0e4
ext_opt_pump_duration = (dopant_upper_lifetime / 2.0, dopant_upper_lifetime * 2.0)
ext_opt_pump_power = (2.0e3, 8.0e3)
ext_opt_pump_resolution = (24, 24)
ext_opt_geom_mediumradius = (medium_radius / 2.0, medium_radius * 2.0)
ext_opt_geom_beamradius = (beam_radius / 2.0, beam_radius * 2.0)
ext_opt_geom_resolution = (24, 24)

time_trunc_rtol = 1.0e-2
fluence_rtol = 1.0e-6
energy_rtol = 1.0e-2
amp_rtol = 2.0e-2

min_count_rho = 0
min_count_phi = 0
min_count_z = 0
min_count_t = 0

integrator_classes = [model.integrator.SimpsonIntegrator]
amplifier_classes = [model.amplifier.HybridAmplifier, model.amplifier.NSFDAmplifier] # HybridAmplifier is faster and may produce less error in some cases; NSFDAmplifier is unconditionally stable and produces qualitatively correct solutions (with all expected properties)

# output parameters:

amplification = True

extended_mode = False
graphs = False
output_data = False

verbose = False

num_tasks = 0

lower_process_priority = True

pulse_num_stride = 100
extended_status_strides = (4, 0)

out_markers_step_divisor = 5
out_rho_steps_divisor = 33
out_phi_steps_divisor = 33
out_z_steps_divisor = 33
out_t_steps_divisor = 33

num_auto_contours = 5

output_rel_time = False
output_color = True
output_styled_lines = False
output_plot_titles = True

output_file_format = "png"
<|endoftext|>


<commit_before>import os
import logging
import re
import typing
from typing import Any, Dict, Optional, Text

from rasa.nlu import utils
from rasa.nlu.components import Component
from rasa.nlu.training_data import Message

logger = logging.getLogger(__name__)

if typing.TYPE_CHECKING:
    from rasa.nlu.config import RasaNLUModelConfig
    from rasa.nlu.training_data import TrainingData
    from rasa.nlu.model import Metadata
    from rasa.nlu.training_data import Message

class KeywordIntentClassifier(Component):
    """Intent classifier using simple keyword matching.

    The classifier takes a list of keywords and associated intents as an input.
    A input sentence is checked for the keywords and the intent is returned.

    """

    provides = ["intent"]

    defaults = {"case_sensitive": True}

    def __init__(self,
                 component_config: Dict[Text, Any] = None,
                 intent_keyword_map: Optional[Dict] = None):

        super(KeywordIntentClassifier, self).__init__(component_config)

        if intent_keyword_map is None:
            self.intent_keyword_map = {}
        else:
            self.intent_keyword_map = intent_keyword_map

    def train(
        self,
        training_data: "TrainingData",
        cfg: Optional["RasaNLUModelConfig"] = None,
        **kwargs: Any
    ) -> None:

        for intent in training_data.intents:
            self.intent_keyword_map[intent] = [
                ex.text
                for ex in training_data.training_examples
                if ex.get("intent") == intent
            ]

    def process(self, message: Message, **kwargs: Any) -> None:
        intent_name = self._map_keyword_to_intent(message.text)
        if intent_name is not None:
            intent = {"name": intent_name, "confidence": 1.0}
            message.set("intent", intent, add_to_output=True)

    def _map_keyword_to_intent(self, text: Text) -> Optional[Text]:
        re_flags = 0 if self.component_config["case_sensitive"] else re.IGNORECASE
        for intent, keywords in self.intent_keyword_map.items():
            for string in keywords:
                if re.search(r"\b" + string + r"\b", text, flags=re_flags):
                    return intent

        return None

    def persist(self, file_name: Text, model_dir: Text) -> Dict[Text, Any]:
        """Persist this model into the passed directory.

        Return the metadata necessary to load the model again.
        """

        file_name = file_name + ".json"
        keyword_file = os.path.join(model_dir, file_name)
        utils.write_json_to_file(keyword_file, self.intent_keyword_map)

        return {"file": file_name}

    @classmethod
    def load(
        cls,
        meta: Dict[Text, Any],
        model_dir: Optional[Text] = None,
        model_metadata: "Metadata" = None,
        cached_component: Optional["KeywordIntentClassifier"] = None,
        **kwargs: Any
    ) -> "KeywordIntentClassifier":

        if model_dir and meta.get("file"):
            file_name = meta.get("file")
            keyword_file = os.path.join(model_dir, file_name)
            if os.path.exists(keyword_file):
                intent_keyword_map = utils.read_json_file(keyword_file)
            else:
                logger.warning(
                    "Failed to load IntentKeywordClassifier, maybe "
                    "{} does not exist.".format(keyword_file)
                )
        return cls(meta, intent_keyword_map)
<commit_msg>replaced regex.search with string equality<commit_after>import os
import logging
import typing
from typing import Any, Dict, Optional, Text

from rasa.nlu import utils
from rasa.nlu.components import Component
from rasa.nlu.training_data import Message

logger = logging.getLogger(__name__)

if typing.TYPE_CHECKING:
    from rasa.nlu.config import RasaNLUModelConfig
    from rasa.nlu.training_data import TrainingData
    from rasa.nlu.model import Metadata
    from rasa.nlu.training_data import Message

class KeywordIntentClassifier(Component):
    """Intent classifier using simple keyword matching.

    The classifier takes a list of keywords and associated intents as an input.
    A input sentence is checked for the keywords and the intent is returned.

    """

    provides = ["intent"]

    defaults = {"case_sensitive": True}

    def __init__(self,
                 component_config: Dict[Text, Any] = None,
                 intent_keyword_map: Optional[Dict] = None):

        super(KeywordIntentClassifier, self).__init__(component_config)

        if intent_keyword_map is None:
            self.intent_keyword_map = {}
        else:
            self.intent_keyword_map = intent_keyword_map

    def train(
        self,
        training_data: "TrainingData",
        cfg: Optional["RasaNLUModelConfig"] = None,
        **kwargs: Any
    ) -> None:

        for intent in training_data.intents:
            self.intent_keyword_map[intent] = [
                ex.text
                for ex in training_data.training_examples
                if ex.get("intent") == intent
            ]

    def process(self, message: Message, **kwargs: Any) -> None:
        intent_name = self._map_keyword_to_intent(message.text)
        if intent_name is not None:
            intent = {"name": intent_name, "confidence": 1.0}
            message.set("intent", intent, add_to_output=True)

    def _map_keyword_to_intent(self, text: Text) -> Optional[Text]:
        for intent, examples in self.intent_keyword_map.items():
            for example in examples:
                if self.component_config["case_sensitive"]:
                    if example == text:
                        return intent
                else:
                    if example.lower() == text.lower():
                        return intent
        return None

    def persist(self, file_name: Text, model_dir: Text) -> Dict[Text, Any]:
        """Persist this model into the passed directory.

        Return the metadata necessary to load the model again.
        """

        file_name = file_name + ".json"
        keyword_file = os.path.join(model_dir, file_name)
        utils.write_json_to_file(keyword_file, self.intent_keyword_map)

        return {"file": file_name}

    @classmethod
    def load(
        cls,
        meta: Dict[Text, Any],
        model_dir: Optional[Text] = None,
        model_metadata: "Metadata" = None,
        cached_component: Optional["KeywordIntentClassifier"] = None,
        **kwargs: Any
    ) -> "KeywordIntentClassifier":

        if model_dir and meta.get("file"):
            file_name = meta.get("file")
            keyword_file = os.path.join(model_dir, file_name)
            if os.path.exists(keyword_file):
                intent_keyword_map = utils.read_json_file(keyword_file)
            else:
                logger.warning(
                    "Failed to load IntentKeywordClassifier, maybe "
                    "{} does not exist.".format(keyword_file)
                )
        return cls(meta, intent_keyword_map)
<|endoftext|>