<commit_before>import time
import random
import logging
import functools

import numpy as np
import tensorflow as tf
from opts import distributed

def log_execution_time(func):
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        start_time = time.time()
        ret = func(*args, **kwargs)
        logging.info("Time: {}".format(time.time() - start_time))
        return ret
    return wrapper

class Trainer:
    def __init__(self):
        self.model = None
        self.mode = None
        self.bid = None
        self.session = None
        self.run_op = None

        FORMAT = '%(asctime)-15s epoch=%(epoch)d;batch=%(batch)d; %(message)s'
        logging.basicConfig(format=FORMAT)

        self.logger = logging.getLogger('trainer')
        self.logger.setLevel(logging.INFO)

    def setup_and_run(self, model, mode, session=None, **kwargs):
        self.model = model
        self.mode = mode
        self.bid = 0
        self.session = session or tf.get_default_session()

        if kwargs['verbose'] == 2:
            self.logger.setLevel(15)
        elif kwargs['verbose'] == 0:
            self.logger.setLevel(30)

        def extract(opt_name, key, pkey='inference'):
            inf = model.ops[opt_name][pkey]

            if hasattr(model, 'devices'):
                inf = list(inf.values())[0]

            return inf.get(key, [])

        self.run_op = {opt_name: [
            extract(opt_name, 'states'),
            extract(opt_name, 0, pkey='losses'),
            extract(opt_name, 'values'),
            extract(opt_name, 'norms'),
            extract(opt_name, 'cosines'),
            model.ops[opt_name]['summaries']
        ] for opt_name in model.optimizees}

        if mode == 'train':
            for opt_name in model.optimizees:
                self.run_op[opt_name].append(model.ops[opt_name]['train_op'])

        return getattr(self, mode)(**kwargs)

    def test(self, eid, n_batches, n_steps=20, opt_name=None):
        self.model.restore(eid)

        sample_optimizee = (opt_name is None)
        opt_names = list(self.model.optimizees.keys())

        rets = []
        for batch in range(n_batches):
            #self.logger.info("Batch: {}".format(batch))
            if sample_optimizee:
                opt_name = random.choice(opt_names)

            ret = self.run_iter(opt_name, n_steps, batch_size=1, train=False)
            rets.append(ret)

        return rets

    def train(self, n_epochs, n_batches,
              batch_size=100, n_steps=20,
              train_lr=1e-4, momentum=0.9,
              eid=0, test=True, verbose=1):

        self.logger.info("Training model: {}".format(self.model.name), extra={'epoch': 0, 'batch': 0})

        self.lr = train_lr
        self.mu = momentum

        if eid > 0:
            self.model.restore(eid)
            self.bid = eid * n_batches

        train_rets = []
        test_rets = []

        sample_steps = bool(n_steps == 0)

        try:
            for epoch in range(eid, n_epochs):
                self.epoch = epoch
                #self.logger.info("Epoch: {}".format(epoch))
                epoch_time = time.time()

                loss = None

                for batch in range(n_batches):
                    self.batch = batch
                    #self.logger.info("Batch: {}".format(batch))
                    if sample_steps:
                        #n_steps = int(np.random.exponential(scale=200)) + 50

                        exp_scale = min(50, epoch)
                        n_steps = int(np.random.exponential(scale=exp_scale)) + 1
                        n_steps *= self.n_bptt_steps
                        self.log("n_steps: {}".format(n_steps), level=15)

                    opt_name = random.choice(list(self.model.optimizees.keys()))
                    ret = self.run_iter(opt_name, n_steps, batch_size, train=True)

                    if np.isnan(ret['loss']):
                        self.log("Loss is NaN", level=40)
                        #print(ret['fxs'])

                    if loss is not None:
                        loss = 0.9 * loss + 0.1 * ret['loss']
                    else:
                        loss = ret['loss']

                    train_rets.append(ret)

                self.log("Epoch time: {}".format(time.time() - epoch_time))
                self.log("Epoch loss: {}".format(loss / np.log(10) / self.model.n_bptt_steps))

                if test and (epoch + 1) % 10 == 0:
                    self.model.save(epoch + 1)

                    self.log("Test epoch: {}".format(epoch))
                    test_epoch_time = time.time()

                    for batch in range(n_batches):
                        self.log("Test batch: {}".format(batch), level=15)

                        opt_name = random.choice(list(self.model.optimizees.keys()))
                        ret = self.run_iter(opt_name, n_steps, batch_size=1, train=False)

                        test_rets.append(ret)

                    test_epoch_time = time.time() - test_epoch_time
                    self.log("Epoch time: {}".format(test_epoch_time))
        except KeyboardInterrupt:
            print("Stopped training early")

        return train_rets, test_rets

    @log_execution_time
    def run_iter(self, opt_name, n_steps, batch_size, train=True):
        self.bid += 1

        optimizee = self.model.optimizees[opt_name]
        x = optimizee.get_initial_x(batch_size)
        state = self.session.run(self.model.initial_state, feed_dict={self.model.x: x})

        optimizee_params = optimizee.get_new_params(batch_size)

        losses, fxs, norms, lrs, cosines = [], [], [], [], []

        self.log("Optimizee: {}".format(opt_name), level=15)

        for i in range(n_steps // self.model.n_bptt_steps):
            feed_dict = optimizee_params
            #feed_dict.update({inp: init for inp, init in zip(self.model.input_state, state)})
            feed_dict.update({inp: state[name] for name, inp in self.model.input_state.items()})
            feed_dict.update(optimizee.get_next_dict(self.model.n_bptt_steps, batch_size))
            feed_dict.update({
                self.model.train_lr: self.lr,
                self.model.momentum: self.mu,
            })

            states, loss, fx, g_norm, cos_step_adam, summaries_str = self.session.run(self.run_op[opt_name], feed_dict=feed_dict)[:5]
            state = states[-1]

            losses.append(loss)
            fxs.extend(fx)
            lrs.extend([s.get('loglr', 0) for s in states])
            norms.extend(g_norm)
            cosines.extend(cos_step_adam)

        self.log("First function value: {}".format(fxs[0][0]), level=15)
        self.log("Last function value: {}".format(fxs[-1][0]), level=15)
        self.log("Loss: {}".format(np.mean(losses / np.log(10))), level=15)

        if self.model.save_tf_data:
            if train:
                for summary_str in summaries_str:
                    self.model.train_writer.add_summary(summary_str, self.bid)
                self.model.train_writer.flush()
            else:
                for summary_str in summaries_str:
                    self.model.test_writer.add_summary(summary_str, self.bid)
                self.model.test_writer.flush()

        return {
            'optimizee_name': opt_name,
            'loss': np.nanmean(losses),
            'fxs': np.array(fxs),
            'lrs': np.array(lrs).mean(axis=1),
            'norms': np.array(norms),
            'cosines': np.array(cosines),
        }

    def log(self, message, level=logging.INFO):
        extra = {
            'batch': self.batch,
            'epoch': self.epoch,
        }
        self.logger.log(level, message, extra=extra)
<commit_msg>refactored<commit_after>import time
import random
import logging
import functools

import numpy as np
import tensorflow as tf
from opts import distributed

def log_execution_time(func):
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        start_time = time.time()
        ret = func(*args, **kwargs)
        logging.info("Time: {}".format(time.time() - start_time))
        return ret
    return wrapper

class Trainer:
    def __init__(self):
        self.model = None
        self.mode = None
        self.bid = None
        self.session = None
        self.run_op = None

        FORMAT = '%(asctime)-15s epoch=%(epoch)d;batch=%(batch)d; %(message)s'
        logging.basicConfig(format=FORMAT)

        self.logger = logging.getLogger('trainer')
        self.logger.setLevel(logging.INFO)

    def setup_and_run(self, model, mode, session=None, **kwargs):
        self.model = model
        self.mode = mode
        self.bid = 0
        self.session = session or tf.get_default_session()

        if kwargs['verbose'] == 2:
            self.logger.setLevel(15)
        elif kwargs['verbose'] == 0:
            self.logger.setLevel(30)

        self.run_op = {}

        for opt_name in model.optimizees:
            inf = model.ops[opt_name]['inference']
            losses = model.ops[opt_name]['losses']

            if hasattr(model, 'devices'):
                inf = list(inf.values())[0]
                losses = list(losses.values())[0]

            self.run_op[opt_name] = {
                'loss': losses[0],
                'summaries': model.ops[opt_name]['summaries'],
                'states': inf['states'],
                'values': inf['values'],
                'norms': inf['norms'],
            }

            if mode == 'train':
                self.run_op[opt_name]['train_op'] = model.ops[opt_name]['train_op']

        return getattr(self, mode)(**kwargs)

    def test(self, eid, n_batches, n_steps=20, opt_name=None):
        self.model.restore(eid)

        sample_optimizee = (opt_name is None)
        opt_names = list(self.model.optimizees.keys())

        rets = []
        for batch in range(n_batches):
            #self.logger.info("Batch: {}".format(batch))
            if sample_optimizee:
                opt_name = random.choice(opt_names)

            ret = self.run_iter(opt_name, n_steps, batch_size=1, train=False)
            rets.append(ret)

        return rets

    def train(self, n_epochs, n_batches,
              batch_size=100, n_steps=20,
              train_lr=1e-4, momentum=0.9,
              eid=0, test=True, verbose=1):

        self.logger.info("Training model: {}".format(self.model.name), extra={'epoch': 0, 'batch': 0})

        self.lr = train_lr
        self.mu = momentum

        if eid > 0:
            self.model.restore(eid)
            self.bid = eid * n_batches

        train_rets = []
        test_rets = []

        sample_steps = bool(n_steps == 0)

        try:
            for epoch in range(eid, n_epochs):
                self.epoch = epoch
                #self.logger.info("Epoch: {}".format(epoch))
                epoch_time = time.time()

                loss = None

                for batch in range(n_batches):
                    self.batch = batch
                    #self.logger.info("Batch: {}".format(batch))
                    if sample_steps:
                        #n_steps = int(np.random.exponential(scale=200)) + 50

                        exp_scale = min(50, epoch)
                        n_steps = int(np.random.exponential(scale=exp_scale)) + 1
                        n_steps *= self.n_bptt_steps
                        self.log("n_steps: {}".format(n_steps), level=15)

                    opt_name = random.choice(list(self.model.optimizees.keys()))
                    ret = self.run_iter(opt_name, n_steps, batch_size, train=True)

                    if np.isnan(ret['loss']):
                        self.log("Loss is NaN", level=40)
                        raise "Loss is NaN"
                    elif np.isinf(ret['loss']):
                        self.log("Loss is +-INF", level=40)
                        raise "Loss is +-INF"
                        #print(ret['fxs'])

                    if loss is not None:
                        loss = 0.9 * loss + 0.1 * ret['loss']
                    else:
                        loss = ret['loss']

                    train_rets.append(ret)

                self.log("Epoch time: {}".format(time.time() - epoch_time))
                self.log("Epoch loss: {}".format(loss / np.log(10) / self.model.n_bptt_steps))

                if test and (epoch + 1) % 10 == 0:
                    self.model.save(epoch + 1)

                    self.log("Test epoch: {}".format(epoch))
                    test_epoch_time = time.time()

                    for batch in range(n_batches):
                        self.log("Test batch: {}".format(batch), level=15)

                        opt_name = random.choice(list(self.model.optimizees.keys()))
                        ret = self.run_iter(opt_name, n_steps, batch_size=1, train=False)

                        test_rets.append(ret)

                    test_epoch_time = time.time() - test_epoch_time
                    self.log("Epoch time: {}".format(test_epoch_time))
        except KeyboardInterrupt:
            print("Stopped training early")

        return train_rets, test_rets

    @log_execution_time
    def run_iter(self, opt_name, n_steps, batch_size, train=True):
        self.bid += 1

        optimizee = self.model.optimizees[opt_name]
        x = optimizee.get_initial_x(batch_size)
        state = self.session.run(self.model.initial_state, feed_dict={self.model.x: x})

        optimizee_params = optimizee.get_new_params(batch_size)

        steps_info = {
            'values': [],
            'norms': [],
        }

        losses = []
        fxs = []

        self.log("Optimizee: {}".format(opt_name), level=15)

        for i in range(n_steps // self.model.n_bptt_steps):
            feed_dict = optimizee_params
            feed_dict.update({inp: state[name] for name, inp in self.model.input_state.items()})
            feed_dict.update(optimizee.get_next_dict(self.model.n_bptt_steps, batch_size))
            feed_dict.update({
                self.model.train_lr: self.lr,
                self.model.momentum: self.mu,
            })

            info = self.session.run(self.run_op[opt_name], feed_dict=feed_dict)
            state = info['states'][-1]
            summaries_str = info['summaries']

            losses.append(info['loss'])
            fxs.extend(info['values'])

        self.log("First function value: {}".format(fxs[0][0]), level=15)
        self.log("Last function value: {}".format(fxs[-1][0]), level=15)
        self.log("Loss: {}".format(np.mean(losses / np.log(10))), level=15)

        if self.model.save_tf_data:
            if train:
                for summary_str in summaries_str:
                    self.model.train_writer.add_summary(summary_str, self.bid)
                self.model.train_writer.flush()
            else:
                for summary_str in summaries_str:
                    self.model.test_writer.add_summary(summary_str, self.bid)
                self.model.test_writer.flush()

        return {
            'optimizee_name': opt_name,
            'loss': np.nanmean(losses),
            'fxs': np.array(fxs),
        }

    def log(self, message, level=logging.INFO):
        extra = {
            'batch': self.batch,
            'epoch': self.epoch,
        }
        self.logger.log(level, message, extra=extra)
<|endoftext|>


<commit_before>import cPickle
import functools
import os

import fuel
import numpy
from fuel.schemes import (
    ConstantScheme, ShuffledExampleScheme, SequentialExampleScheme)
from fuel.streams import DataStream
from fuel.transformers import (
    SortMapping, Padding, ForceFloatX, Batch, Mapping, Unpack, Filter,
    FilterSources, Transformer)

from lvsr.datasets.h5py import H5PYAudioDataset
from lvsr.preprocessing import log_spectrogram

def switch_first_two_axes(batch):
    result = []
    for array in batch:
        if array.ndim == 2:
            result.append(array.transpose(1, 0))
        else:
            result.append(array.transpose(1, 0, 2))
    return tuple(result)

def _length(example):
    return len(example[0])

def apply_preprocessing(preprocessing, example):
    recording, label = example
    return (numpy.asarray(preprocessing(recording)), label)

class _AddLabel(object):

    def __init__(self, label, append=True, times=1):
        self.label = label
        self.append = append
        self.times = times

    def __call__(self, example):
        example = list(example)
        if self.append:
            # Not using `list.append` to avoid having weird mutable
            # example objects.
            example[1] = numpy.hstack([example[1], self.times * [self.label]])
        else:
            example[1] = numpy.hstack([self.times * [self.label], example[1]])
        return example

class _LengthFilter(object):

    def __init__(self, max_length):
        self.max_length = max_length

    def __call__(self, example):
        if self.max_length:
            return len(example[0]) <= self.max_length
        return True

class ForceCContiguous(Transformer):
    """Force all floating point numpy arrays to be floatX."""
    def __init__(self, data_stream):
        super(ForceCContiguous, self).__init__(
            data_stream, axis_labels=data_stream.axis_labels)

    def get_data(self, request=None):
        if request is not None:
            raise ValueError
        data = next(self.child_epoch_iterator)
        result = []
        for piece in data:
            if isinstance(piece, numpy.ndarray):
                result.append(numpy.ascontiguousarray(piece))
            else:
                result.append(piece)
        return tuple(result)

class Data(object):
    """Dataset manager.

    This class is in charge of accessing different datasets
    and building preprocessing pipelines.

    Parameters
    ----------
    dataset : str
        Dataset name.
    recordings_source : str
        Source name for recording.
    labels_source : str
        Source name for labels.
    batch_size : int
        Batch size.
    sort_k_batches : int
    max_length : int
        Maximum length of input, longer sequences will be filtered.
    normalization : str
        Normalization file name to use.
    uttid_source : str
        Utterance id source name.
    feature_name : str
        `wav` or `fbank_and_delta_delta`.
    preprocess_features : str
        Now supports only `log_spectrogram` value.
    add_eos : bool
        Add end of sequence symbol.
    add_bos : int
        Add this many beginning-of-sequence tokens.
    eos_label : int
        Label to use for eos symbol.
    preprocess_text : bool
        Preprocess text for WSJ.

    """
    def __init__(self, dataset, recordings_source, labels_source,
                 batch_size, sort_k_batches=None,
                 max_length=None, normalization=None,
                 uttid_source='uttids',
                 feature_name='wav', preprocess_features=None,
                 add_eos=True, eos_label=None,
                 add_bos=0, prepend_eos=False,
                 preprocess_text=False):
        assert not prepend_eos

        # We used to support more datasets, but only WSJ is left after
        # a cleanup.
        if not dataset in ('WSJ'):
            raise ValueError()

        if normalization:
            with open(normalization, "rb") as src:
                normalization = cPickle.load(src)

        self.dataset = dataset
        self.recordings_source = recordings_source
        self.labels_source = labels_source
        self.uttid_source = uttid_source
        self.normalization = normalization
        self.batch_size = batch_size
        self.sort_k_batches = sort_k_batches
        self.feature_name = feature_name
        self.max_length = max_length
        self.add_eos = add_eos
        self.prepend_eos = prepend_eos
        self._eos_label = eos_label
        self.add_bos = add_bos
        self.preprocess_text = preprocess_text
        self.preprocess_features = preprocess_features
        self.dataset_cache = {}

        self.length_filter = _LengthFilter(self.max_length)

    @property
    def info_dataset(self):
        return self._get_dataset("train")

    @property
    def num_labels(self):
        return self.info_dataset.num_characters

    @property
    def character_map(self):
        return self.info_dataset.char2num

    @property
    def num_features(self):
        return self.info_dataset.num_features

    @property
    def eos_label(self):
        if self._eos_label:
            return self._eos_label
        return self.info_dataset.eos_label

    @property
    def bos_label(self):
        return self.info_dataset.bos_label

    def decode(self, labels):
        return self.info_dataset.decode(labels)

    def pretty_print(self, labels):
        return self.info_dataset.pretty_print(labels)

    def get_dataset(self, part, add_sources=()):
        """Returns dataset from the cache or creates a new one"""
        key = (part, add_sources)
        if key not in self.dataset_cache:
            self.dataset_cache[key] = self._get_dataset(*key)
        return self.dataset_cache[key]

    def _get_dataset(self, part, add_sources=()):
        wsj_name_mapping = {"train": "train_si284", "valid": "test_dev93",
                            "test": "test_eval92"}

        return H5PYAudioDataset(
            os.path.join(fuel.config.data_path, "wsj.h5"),
            which_sets=(wsj_name_mapping.get(part, part),),
            sources=(self.recordings_source,
                     self.labels_source) + tuple(add_sources))

    def get_stream(self, part, batches=True, shuffle=True, add_sources=(),
                   num_examples=None, rng=None, seed=None):
        if not rng:
            rng = numpy.random.RandomState(seed)
        dataset = self.get_dataset(part, add_sources=add_sources)
        if num_examples and shuffle:
            examples = list(range(dataset.num_examples))
            examples = rng.choice(examples, num_examples)
        else:
            examples = num_examples

        if not examples:
            examples = dataset.num_examples

        if shuffle:
            stream = DataStream(
                dataset, iteration_scheme=ShuffledExampleScheme(examples))
        else:
            stream = DataStream(
                dataset, iteration_scheme=SequentialExampleScheme(examples))

        stream = FilterSources(stream, (self.recordings_source,
                                        self.labels_source)+tuple(add_sources))
        if self.add_eos:
            stream = Mapping(stream, _AddLabel(self.eos_label))
        if self.add_bos:
            stream = Mapping(stream, _AddLabel(self.bos_label, append=False,
                                               times=self.add_bos))
        if self.preprocess_text:
            stream = Mapping(stream, lvsr.datasets.wsj.preprocess_text)
        stream = Filter(stream, self.length_filter)
        if self.sort_k_batches and batches:
            stream = Batch(stream,
                           iteration_scheme=ConstantScheme(
                               self.batch_size * self.sort_k_batches))
            stream = Mapping(stream, SortMapping(_length))
            stream = Unpack(stream)

        if self.preprocess_features == 'log_spectrogram':
            stream = Mapping(
                stream, functools.partial(apply_preprocessing,
                                          log_spectrogram))
        if self.normalization:
            stream = self.normalization.wrap_stream(stream)
        stream = ForceFloatX(stream)
        if not batches:
            return stream

        stream = Batch(stream, iteration_scheme=ConstantScheme(self.batch_size))
        stream = Padding(stream)
        stream = Mapping(stream, switch_first_two_axes)
        stream = ForceCContiguous(stream)
        return stream
<commit_msg>Fix typo<commit_after>import cPickle
import functools
import os

import fuel
import numpy
from fuel.schemes import (
    ConstantScheme, ShuffledExampleScheme, SequentialExampleScheme)
from fuel.streams import DataStream
from fuel.transformers import (
    SortMapping, Padding, ForceFloatX, Batch, Mapping, Unpack, Filter,
    FilterSources, Transformer)

from lvsr.datasets.h5py import H5PYAudioDataset
from lvsr.preprocessing import log_spectrogram

def switch_first_two_axes(batch):
    result = []
    for array in batch:
        if array.ndim == 2:
            result.append(array.transpose(1, 0))
        else:
            result.append(array.transpose(1, 0, 2))
    return tuple(result)

def _length(example):
    return len(example[0])

def apply_preprocessing(preprocessing, example):
    recording, label = example
    return (numpy.asarray(preprocessing(recording)), label)

class _AddLabel(object):

    def __init__(self, label, append=True, times=1):
        self.label = label
        self.append = append
        self.times = times

    def __call__(self, example):
        example = list(example)
        if self.append:
            # Not using `list.append` to avoid having weird mutable
            # example objects.
            example[1] = numpy.hstack([example[1], self.times * [self.label]])
        else:
            example[1] = numpy.hstack([self.times * [self.label], example[1]])
        return example

class _LengthFilter(object):

    def __init__(self, max_length):
        self.max_length = max_length

    def __call__(self, example):
        if self.max_length:
            return len(example[0]) <= self.max_length
        return True

class ForceCContiguous(Transformer):
    """Force all floating point numpy arrays to be floatX."""
    def __init__(self, data_stream):
        super(ForceCContiguous, self).__init__(
            data_stream, axis_labels=data_stream.axis_labels)

    def get_data(self, request=None):
        if request is not None:
            raise ValueError
        data = next(self.child_epoch_iterator)
        result = []
        for piece in data:
            if isinstance(piece, numpy.ndarray):
                result.append(numpy.ascontiguousarray(piece))
            else:
                result.append(piece)
        return tuple(result)

class Data(object):
    """Dataset manager.

    This class is in charge of accessing different datasets
    and building preprocessing pipelines.

    Parameters
    ----------
    dataset : str
        Dataset name.
    recordings_source : str
        Source name for recording.
    labels_source : str
        Source name for labels.
    batch_size : int
        Batch size.
    sort_k_batches : int
    max_length : int
        Maximum length of input, longer sequences will be filtered.
    normalization : str
        Normalization file name to use.
    uttid_source : str
        Utterance id source name.
    feature_name : str
        `wav` or `fbank_and_delta_delta`.
    preprocess_features : str
        Now supports only `log_spectrogram` value.
    add_eos : bool
        Add end of sequence symbol.
    add_bos : int
        Add this many beginning-of-sequence tokens.
    eos_label : int
        Label to use for eos symbol.
    preprocess_text : bool
        Preprocess text for WSJ.

    """
    def __init__(self, dataset, recordings_source, labels_source,
                 batch_size, sort_k_batches=None,
                 max_length=None, normalization=None,
                 uttid_source='uttids',
                 feature_name='wav', preprocess_features=None,
                 add_eos=True, eos_label=None,
                 add_bos=0, prepend_eos=False,
                 preprocess_text=False):
        assert not prepend_eos

        # We used to support more datasets, but only WSJ is left after
        # a cleanup.
        if not dataset in ('WSJ'):
            raise ValueError()

        if normalization:
            with open(normalization, "rb") as src:
                normalization = cPickle.load(src)

        self.dataset = dataset
        self.recordings_source = recordings_source
        self.labels_source = labels_source
        self.uttid_source = uttid_source
        self.normalization = normalization
        self.batch_size = batch_size
        self.sort_k_batches = sort_k_batches
        self.feature_name = feature_name
        self.max_length = max_length
        self.add_eos = add_eos
        self.prepend_eos = prepend_eos
        self._eos_label = eos_label
        self.add_bos = add_bos
        self.preprocess_text = preprocess_text
        self.preprocess_features = preprocess_features
        self.dataset_cache = {}

        self.length_filter = _LengthFilter(self.max_length)

    @property
    def info_dataset(self):
        return self.get_dataset("train")

    @property
    def num_labels(self):
        return self.info_dataset.num_characters

    @property
    def character_map(self):
        return self.info_dataset.char2num

    @property
    def num_features(self):
        return self.info_dataset.num_features

    @property
    def eos_label(self):
        if self._eos_label:
            return self._eos_label
        return self.info_dataset.eos_label

    @property
    def bos_label(self):
        return self.info_dataset.bos_label

    def decode(self, labels):
        return self.info_dataset.decode(labels)

    def pretty_print(self, labels):
        return self.info_dataset.pretty_print(labels)

    def get_dataset(self, part, add_sources=()):
        """Returns dataset from the cache or creates a new one"""
        key = (part, add_sources)
        if key not in self.dataset_cache:
            self.dataset_cache[key] = self._get_dataset(*key)
        return self.dataset_cache[key]

    def _get_dataset(self, part, add_sources=()):
        wsj_name_mapping = {"train": "train_si284", "valid": "test_dev93",
                            "test": "test_eval92"}

        return H5PYAudioDataset(
            os.path.join(fuel.config.data_path, "wsj.h5"),
            which_sets=(wsj_name_mapping.get(part, part),),
            sources=(self.recordings_source,
                     self.labels_source) + tuple(add_sources))

    def get_stream(self, part, batches=True, shuffle=True, add_sources=(),
                   num_examples=None, rng=None, seed=None):
        if not rng:
            rng = numpy.random.RandomState(seed)
        dataset = self.get_dataset(part, add_sources=add_sources)
        if num_examples and shuffle:
            examples = list(range(dataset.num_examples))
            examples = rng.choice(examples, num_examples)
        else:
            examples = num_examples

        if not examples:
            examples = dataset.num_examples

        if shuffle:
            stream = DataStream(
                dataset, iteration_scheme=ShuffledExampleScheme(examples))
        else:
            stream = DataStream(
                dataset, iteration_scheme=SequentialExampleScheme(examples))

        stream = FilterSources(stream, (self.recordings_source,
                                        self.labels_source)+tuple(add_sources))
        if self.add_eos:
            stream = Mapping(stream, _AddLabel(self.eos_label))
        if self.add_bos:
            stream = Mapping(stream, _AddLabel(self.bos_label, append=False,
                                               times=self.add_bos))
        if self.preprocess_text:
            stream = Mapping(stream, lvsr.datasets.wsj.preprocess_text)
        stream = Filter(stream, self.length_filter)
        if self.sort_k_batches and batches:
            stream = Batch(stream,
                           iteration_scheme=ConstantScheme(
                               self.batch_size * self.sort_k_batches))
            stream = Mapping(stream, SortMapping(_length))
            stream = Unpack(stream)

        if self.preprocess_features == 'log_spectrogram':
            stream = Mapping(
                stream, functools.partial(apply_preprocessing,
                                          log_spectrogram))
        if self.normalization:
            stream = self.normalization.wrap_stream(stream)
        stream = ForceFloatX(stream)
        if not batches:
            return stream

        stream = Batch(stream, iteration_scheme=ConstantScheme(self.batch_size))
        stream = Padding(stream)
        stream = Mapping(stream, switch_first_two_axes)
        stream = ForceCContiguous(stream)
        return stream
<|endoftext|>


<commit_before>"""Holds various Version strings of MPF.

This modules holds the MPF version strings, including the version of BCP it
needs and the config file version it needs.

It's used internally for all sorts of things, from printing the output of the
`mpf --version` command, to making sure any processes connected via BCP are
the proper versions, to automatically triggering new builds and deployments to
PyPI.

"""

__version__ = '0.54.0-dev.49'
'''The full version of MPF.'''

__short_version__ = '0.54'
'''The major.minor version of MPF.'''

__bcp_version__ = '1.1'
'''The version of BCP this build of MPF uses.'''

__config_version__ = '5'
'''The config file version this build of MPF uses.'''

__show_version__ = '5'
'''The show format version this build of MPF uses.'''

# pylint: disable-msg=invalid-name
version = "MPF v{}".format(__version__)
'''A friendly version string for this build of MPF.'''

# pylint: disable-msg=invalid-name
extended_version = "MPF v{}, Config version:{}, Show version: {}, " \
                   "BCP version:{}".format(__version__, __config_version__,
                                           __show_version__, __bcp_version__)
'''An extended version string that includes the MPF version, show version,
and BCP versions used in this build of MPF.'''

if "dev" in __version__:
    # pylint: disable-msg=invalid-name
    log_url = "https://docs.missionpinball.org/en/dev/logs/{}.html"
else:
    # pylint: disable-msg=invalid-name
    log_url = "https://docs.missionpinball.org/en/{}/logs/{{}}.html".format(__short_version__)

__api__ = ['version',
           '__short_version__',
           '__bcp_version__',
           '__config_version__',
           '__show_version__',
           'version',
           'extended_version',
           'log_url']
<commit_msg>0.54.0-dev.50<commit_after>"""Holds various Version strings of MPF.

This modules holds the MPF version strings, including the version of BCP it
needs and the config file version it needs.

It's used internally for all sorts of things, from printing the output of the
`mpf --version` command, to making sure any processes connected via BCP are
the proper versions, to automatically triggering new builds and deployments to
PyPI.

"""

__version__ = '0.54.0-dev.50'
'''The full version of MPF.'''

__short_version__ = '0.54'
'''The major.minor version of MPF.'''

__bcp_version__ = '1.1'
'''The version of BCP this build of MPF uses.'''

__config_version__ = '5'
'''The config file version this build of MPF uses.'''

__show_version__ = '5'
'''The show format version this build of MPF uses.'''

# pylint: disable-msg=invalid-name
version = "MPF v{}".format(__version__)
'''A friendly version string for this build of MPF.'''

# pylint: disable-msg=invalid-name
extended_version = "MPF v{}, Config version:{}, Show version: {}, " \
                   "BCP version:{}".format(__version__, __config_version__,
                                           __show_version__, __bcp_version__)
'''An extended version string that includes the MPF version, show version,
and BCP versions used in this build of MPF.'''

if "dev" in __version__:
    # pylint: disable-msg=invalid-name
    log_url = "https://docs.missionpinball.org/en/dev/logs/{}.html"
else:
    # pylint: disable-msg=invalid-name
    log_url = "https://docs.missionpinball.org/en/{}/logs/{{}}.html".format(__short_version__)

__api__ = ['version',
           '__short_version__',
           '__bcp_version__',
           '__config_version__',
           '__show_version__',
           'version',
           'extended_version',
           'log_url']
<|endoftext|>


<commit_before>#!/usr/bin/env python -OO
# encoding: utf-8

###########
# ORP - Open Robotics Platform
#
# Copyright (c) 2010 John Harrison, William Woodall
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
##########

"""
rpcserver.py - Contains the RPCServer class which defines the available \
functions in the rpc server.

Created by William Woodall on 2010-09-18.
"""
__author__ = "William Woodall"
__copyright__ = "Copyright (c) 2010 John Harrison, William Woodall"

###  Imports  ###

# Standard Python Libraries
import os
import sys
import types
import signal
from multiprocessing import Process, Lock, Value, Queue
from threading import Timer
from SimpleXMLRPCServer import SimpleXMLRPCServer
import time
import logging
import ctypes

# Other Libraries
import orpd
from orpd import network_handlers
import orpdaemon
from logerror import logError
from sandbox import Sandbox

###  Class  ###

class RPCServer(SimpleXMLRPCServer):
    """Defines the available xmlrpc functions"""
    def __init__(self, connection, daemon):
        """docstring for __init__"""
        # Setup logging
        self.log = logging.getLogger('ORPD')
        self.log_servers = []
        self.daemon = daemon
        self.sandbox = None
        self.sandbox_proc = None
        self.sandbox_lock = None
        self.sandbox_queue = None
        self.sandbox_running = None
        self.sandbox_paused = False
        self.client_addr = None
        self.network_handlers = network_handlers
        SimpleXMLRPCServer.__init__(self,
                        connection, logRequests=False, allow_none=True)

    def _dispatch(self, method, params):
        """Overrides the dispatch method"""
        try:
            # We are forcing the 'xmlrpc_' prefix on methods that are
            # callable through XML-RPC
            func = getattr(self, 'xmlrpc_'+method)
        except AttributeError:
            try:
                func = self.funcs[method]
                if func is not None:
                    try:
                        return func(*params)
                    except Exception as err:
                        logError(sys.exc_info(), self.log.error, 'Exception in RPC call %s' % method, orpdaemon.HWM_MAGIC_LINENO)
                else:
                    raise Exception("Method '%s' is not supported by this Server" % method)
            except Exception:
                return SimpleXMLRPCServer._dispatch(self, method, params)
        else:
            try:
                return func(*params)
            except Exception as err:
                logError(sys.exc_info(), self.log.error, "Exception in RPC call %s" % method)

    def registerFunctions(self, functions):
        """Takes a dictionary, functions, and registers them"""
        for func_name in functions:
            self.register_function(functions[func_name], func_name)

    def verify_request(self, request, client_addr):
        """Overrides SimpleXMLRPCServer.verify_request to
        capture the address of the latest request"""
        self.client_addr = client_addr
        return True

    def xmlrpc_listFunctions(self):
        """Lists the Available Functions"""
        methods = self.funcs.keys()
        for x in ['system.listMethods', 'system.methodHelp', 'system.methodSignature']:
            if x in methods:
                methods.remove(x)
        return methods

    ####  Functions Related to Executing Control Code  ####
    def xmlrpc_runControlCode(self, control_code=None):
        """Runs control code in the files dir"""
        # Check for correct parameters
        if control_code == None:
            self.log.error("No Control Code passed to function runControlCode")
            return False
        if self.sandbox_proc != None and self.sandbox_proc.is_alive():
            self.log.error("Control Code already running")
            return False
        if not os.path.exists(control_code):
            self.log.error('Control Code not found: ' + control_code)
            return False
        # Spawn the sandbox
        self.sandbox = Sandbox()
        self.sandbox_lock = Lock()
        self.sandbox_running = Value(ctypes.c_bool, True)
        self.sandbox_queue = Queue()
        self.sandbox_proc = \
            Process("Sandbox", target=self.sandbox.startUp,
                args=(self.daemon.device_objects, self.daemon.work_directory,
                        control_code, self.sandbox_lock, self.sandbox_running,
                        self.sandbox_queue))
        self.sandbox_proc.start()
        Timer(0.5, self.joinControlCode).start()
        # Attempt to call start() on each of the devices that are running
        for device in self.daemon.device_objects:
            try:
                device.start()
            except Exception as error:
                logError(sys.exc_info(), self.log.error, "Error executing start() of the %s device:" % device.name, orpdaemon.HWM_MAGIC_LINENO)
        return True

    def joinControlCode(self):
        """This function trys to join the sandbox and periodically checks to see if it has crashed"""
        # Wait for the sandbox to stop
        try:
            sandbox_running = True
            while sandbox_running:
                if self.sandbox_proc != None and self.sandbox_proc.exitcode == None:
                    pass
                else:
                    sandbox_running = False
                    self.sandbox_running.value = False
                    break
                self.sandbox_proc.join(1)
        finally:
            if self.sandbox_proc.exitcode == -signal.SIGTERM:
                result = False
                self.log.info('Control Code did not exit cleanly')
            self.sandbox = None
            self.sandbox_proc = None
            self.sandbox_lock = None
            self.sandbox_running.value = False
            self.sandbox_paused = False
            for device in self.daemon.device_objects:
                try:
                    device.stop()
                except Exception as error:
                    logError(sys.exc_info(), self.log.error, "Error executing stop() of the %s device:" % device.name, orpdaemon.HWM_MAGIC_LINENO)
            self.log.info('Control Code Stopped')

    def xmlrpc_stopControlCode(self):
        """Stops the sandbox, returns false if it times out"""
        self.log.info('Control Code Stopping')
        # Check to see if it is already stopped
        result = True
        if self.sandbox_proc == None or not self.sandbox_proc.is_alive():
            return result
        # Start a timer to kill the sandbox is necessary
        Timer(2, self.sandbox_proc.terminate).start()
        # Stop the sandbox
        self.sandbox_running.value = False
        self.sandbox_proc.join()
        if self.sandbox_proc.exitcode == -signal.SIGTERM:
            result = False
        return result

    def xmlrpc_pauseControlCode(self):
        """Pauses the current running control, if some is running"""
        # Check that control code is running, else return False
        if self.sandbox_proc == None:
            return False
        # Pause the control code
        if not self.sandbox_paused:
            self.sandbox_lock.acquire()
            self.sandbox_paused = True
        return True

    def xmlrpc_resumeControlCode(self):
        """Pauses the current running control, if some is running"""
        # Check that control code is running, else return False
        if self.sandbox_proc == None or not self.sandbox_paused:
            return False
        # Resume the control code
        # self.sandbox_lock.acquire(0)
        self.sandbox_lock.release()
        self.sandbox_paused = False
        return True

    ####  Functions related to logging  ####
    def xmlrpc_log(self, level, msg):
        """Allows for people to inject logs over rpc"""
        if type(level) == str:
            levels = {'notset':0,'debug':10,'info':20,
                      'warning':30,'error':40,'critical':50}
            self.log.log(levels[level.lower()], msg)
        elif type(level) == int:
            self.log.log(level, msg)

    def xmlrpc_debug(self, msg):
        """Creates a debug log"""
        self.log.debug(msg)

    def xmlrpc_info(self, msg):
        """Creates an info log"""
        self.log.info(msg)

    def xmlrpc_warning(self, msg):
        """Creates a warning log"""
        self.log.warning(msg)

    def xmlrpc_error(self, msg):
        """Creates an error log"""
        self.log.error(msg)

    def xmlrpc_critical(self, msg):
        """Creates a critical log"""
        self.log.critical(msg)

    ####  File Management Functions  ####
    def xmlrpc_write(self, file_name, file_contents, timestamp=None):
        """Writes a file sent to the daemon,
        returns True on success and False on failure"""
        file_dir, _ = os.path.split(file_name)
        if file_dir and not os.path.exists(file_dir):
            os.makedirs(file_dir)
        result = True
        try:
            save_file = open(file_name, 'w+')
            save_file.write(file_contents)
            save_file.close()
            if timestamp:
                os.utime(file_name, (timestamp, timestamp))
            self.log.info('Wrote file '+file_name)
        except IOError as error:
            self.log.error('Cannot write file: '+file_name+\
                                    '\n'+str(error))
            result = False
        return result

    def xmlrpc_getFileContents(self, file_name):
        """Returns the contents of the given file"""
        file_handler = open(file_name, 'r')
        return file_handler.read()

    def xmlrpc_getConfig(self):
        """This function returns the config file as a string"""
        config_file = open(self.daemon.config_file_name, 'r')
        return config_file.read()

    def xmlrpc_sendConfig(self, config):
        """Writes the given data to the config file"""
        config_file_name = self.daemon.config_file_name
        _, temp_name = os.path.split(config_file_name)
        if temp_name == 'orpd.cfg.default':
            self.daemon.config_file_name = config_file_name = config_file_name[:-8]
        config_file = open(config_file_name, 'w+')
        config_file.write(config)
        config_file.close()

    ####  Misc Functions  ####
    def xmlrpc_ping(self):
        """Ping, Pong."""
        self.log.info('Pong')
        return True

    def xmlrpc_connect(self):
        """Called when a dashboard connects"""
        self.log.info('Receiving connection from %s' % self.client_addr[0])
        if self.client_addr[0] in self.network_handlers:
            self.log.debug('Dashboard Already Connected, not opening a new logging socket')
        else:
            # Setup Network handler
            network = logging.handlers.SocketHandler(self.client_addr[0],
                                logging.handlers.DEFAULT_TCP_LOGGING_PORT)
            network.setLevel(logging.DEBUG)
            root = logging.getLogger('')
            root.addHandler(network)
            self.log_servers.append(network)
            self.network_handlers[self.client_addr[0]] = network

    def xmlrpc_fileSync(self):
        "Synchronize the files between the server and the client"
        return self.__buildListing()

    def __buildListing(self):
        """Build a listing of the files in files and modules for connection"""
        cc_list = self.__walkDirectory(os.path.join(
            self.daemon.work_directory, 'files'))
        module_list = self.__walkDirectory(os.path.join(
            self.daemon.work_directory, 'modules'))
        return (cc_list, module_list)

    def __walkDirectory(self, path):
        """Walks the Directory"""
        listing = {}
        for root, _, files in os.walk(path):
            # if root[0] == '.':
                # continue
            file_path = root.replace(path, '')[1:]
            if file_path and filter(lambda x: x[0] == '.', file_path.split(os.sep)):
                continue
            for f in files:
                if f[0] != '.' and not f.endswith(('.hwmo', '.hwmc', '.pyo', '.pyc', '.cco', '.ccc')):
                    full_path = os.path.join(path, file_path, f)
                    listing_name = os.path.join(file_path, f)
                    listing[listing_name] = os.stat(full_path)[8]
        return listing

    def xmlrpc_disconnect(self):
        """docstring for xmlrpc_disconnect"""
        if self.client_addr[0] in self.network_handlers:
            root = logging.getLogger()
            root.removeHandler(self.network_handlers[self.client_addr[0]])
            del self.network_handlers[self.client_addr[0]]

    def xmlrpc_shutdown(self):
        """Shutsdown the system"""
        # I do this so that the xmlrpc function can return properly.
        # If you just xmlrpc.shutdown() then,
        # the return None line never gets executed.
        # So I just have it call that 100mS later to allow it to return first.
        Timer(0.1, self.__shutdown).start()
        return None

    def xmlrpc_restart(self):
        """Restarts the server"""
        self.log.info('Server Restarted')
        return self.xmlrpc_changeWorkDirectory(self.daemon.work_directory)

    def xmlrpc_changeWorkDirectory(self, work_dir):
        """This stops the services, changes the work directory,
        and then reloads everything that can be configured through config files"""
        if os.path.exists(work_dir):
            self.daemon.new_work_dir = work_dir
            self.xmlrpc_shutdown()
            return True
        return False

    def __shutdown(self):
        """Shuts down"""
        self.shutdown()
        self.socket.close()
<commit_msg>Minor bug fix where a required library was not included.<commit_after>#!/usr/bin/env python -OO
# encoding: utf-8

###########
# ORP - Open Robotics Platform
#
# Copyright (c) 2010 John Harrison, William Woodall
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
##########

"""
rpcserver.py - Contains the RPCServer class which defines the available \
functions in the rpc server.

Created by William Woodall on 2010-09-18.
"""
__author__ = "William Woodall"
__copyright__ = "Copyright (c) 2010 John Harrison, William Woodall"

###  Imports  ###

# Standard Python Libraries
import sys
import os
import sys
import types
import signal
from multiprocessing import Process, Lock, Value, Queue
from threading import Timer
from SimpleXMLRPCServer import SimpleXMLRPCServer
import time
import logging
import ctypes

# Other Libraries
import orpd
from orpd import network_handlers
import orpdaemon
from logerror import logError
from sandbox import Sandbox

###  Class  ###

class RPCServer(SimpleXMLRPCServer):
    """Defines the available xmlrpc functions"""
    def __init__(self, connection, daemon):
        """docstring for __init__"""
        # Setup logging
        self.log = logging.getLogger('ORPD')
        self.log_servers = []
        self.daemon = daemon
        self.sandbox = None
        self.sandbox_proc = None
        self.sandbox_lock = None
        self.sandbox_queue = None
        self.sandbox_running = None
        self.sandbox_paused = False
        self.client_addr = None
        self.network_handlers = network_handlers
        SimpleXMLRPCServer.__init__(self,
                        connection, logRequests=False, allow_none=True)

    def _dispatch(self, method, params):
        """Overrides the dispatch method"""
        try:
            # We are forcing the 'xmlrpc_' prefix on methods that are
            # callable through XML-RPC
            func = getattr(self, 'xmlrpc_'+method)
        except AttributeError:
            try:
                func = self.funcs[method]
                if func is not None:
                    try:
                        return func(*params)
                    except Exception as err:
                        logError(sys.exc_info(), self.log.error, 'Exception in RPC call %s' % method, orpdaemon.HWM_MAGIC_LINENO)
                else:
                    raise Exception("Method '%s' is not supported by this Server" % method)
            except Exception:
                return SimpleXMLRPCServer._dispatch(self, method, params)
        else:
            try:
                return func(*params)
            except Exception as err:
                logError(sys.exc_info(), self.log.error, "Exception in RPC call %s" % method)

    def registerFunctions(self, functions):
        """Takes a dictionary, functions, and registers them"""
        for func_name in functions:
            self.register_function(functions[func_name], func_name)

    def verify_request(self, request, client_addr):
        """Overrides SimpleXMLRPCServer.verify_request to
        capture the address of the latest request"""
        self.client_addr = client_addr
        return True

    def xmlrpc_listFunctions(self):
        """Lists the Available Functions"""
        methods = self.funcs.keys()
        for x in ['system.listMethods', 'system.methodHelp', 'system.methodSignature']:
            if x in methods:
                methods.remove(x)
        return methods

    ####  Functions Related to Executing Control Code  ####
    def xmlrpc_runControlCode(self, control_code=None):
        """Runs control code in the files dir"""
        # Check for correct parameters
        if control_code == None:
            self.log.error("No Control Code passed to function runControlCode")
            return False
        if self.sandbox_proc != None and self.sandbox_proc.is_alive():
            self.log.error("Control Code already running")
            return False
        if not os.path.exists(control_code):
            self.log.error('Control Code not found: ' + control_code)
            return False
        # Spawn the sandbox
        self.sandbox = Sandbox()
        self.sandbox_lock = Lock()
        self.sandbox_running = Value(ctypes.c_bool, True)
        self.sandbox_queue = Queue()
        self.sandbox_proc = \
            Process("Sandbox", target=self.sandbox.startUp,
                args=(self.daemon.device_objects, self.daemon.work_directory,
                        control_code, self.sandbox_lock, self.sandbox_running,
                        self.sandbox_queue))
        self.sandbox_proc.start()
        Timer(0.5, self.joinControlCode).start()
        # Attempt to call start() on each of the devices that are running
        for device in self.daemon.device_objects:
            try:
                device.start()
            except Exception as error:
                logError(sys.exc_info(), self.log.error, "Error executing start() of the %s device:" % device.name, orpdaemon.HWM_MAGIC_LINENO)
        return True

    def joinControlCode(self):
        """This function trys to join the sandbox and periodically checks to see if it has crashed"""
        # Wait for the sandbox to stop
        try:
            sandbox_running = True
            while sandbox_running:
                if self.sandbox_proc != None and self.sandbox_proc.exitcode == None:
                    pass
                else:
                    sandbox_running = False
                    self.sandbox_running.value = False
                    break
                self.sandbox_proc.join(1)
        finally:
            if self.sandbox_proc.exitcode == -signal.SIGTERM:
                result = False
                self.log.info('Control Code did not exit cleanly')
            self.sandbox = None
            self.sandbox_proc = None
            self.sandbox_lock = None
            self.sandbox_running.value = False
            self.sandbox_paused = False
            for device in self.daemon.device_objects:
                try:
                    device.stop()
                except Exception as error:
                    logError(sys.exc_info(), self.log.error, "Error executing stop() of the %s device:" % device.name, orpdaemon.HWM_MAGIC_LINENO)
            self.log.info('Control Code Stopped')

    def xmlrpc_stopControlCode(self):
        """Stops the sandbox, returns false if it times out"""
        self.log.info('Control Code Stopping')
        # Check to see if it is already stopped
        result = True
        if self.sandbox_proc == None or not self.sandbox_proc.is_alive():
            return result
        # Start a timer to kill the sandbox is necessary
        Timer(2, self.sandbox_proc.terminate).start()
        # Stop the sandbox
        self.sandbox_running.value = False
        self.sandbox_proc.join()
        if self.sandbox_proc.exitcode == -signal.SIGTERM:
            result = False
        return result

    def xmlrpc_pauseControlCode(self):
        """Pauses the current running control, if some is running"""
        # Check that control code is running, else return False
        if self.sandbox_proc == None:
            return False
        # Pause the control code
        if not self.sandbox_paused:
            self.sandbox_lock.acquire()
            self.sandbox_paused = True
        return True

    def xmlrpc_resumeControlCode(self):
        """Pauses the current running control, if some is running"""
        # Check that control code is running, else return False
        if self.sandbox_proc == None or not self.sandbox_paused:
            return False
        # Resume the control code
        # self.sandbox_lock.acquire(0)
        self.sandbox_lock.release()
        self.sandbox_paused = False
        return True

    ####  Functions related to logging  ####
    def xmlrpc_log(self, level, msg):
        """Allows for people to inject logs over rpc"""
        if type(level) == str:
            levels = {'notset':0,'debug':10,'info':20,
                      'warning':30,'error':40,'critical':50}
            self.log.log(levels[level.lower()], msg)
        elif type(level) == int:
            self.log.log(level, msg)

    def xmlrpc_debug(self, msg):
        """Creates a debug log"""
        self.log.debug(msg)

    def xmlrpc_info(self, msg):
        """Creates an info log"""
        self.log.info(msg)

    def xmlrpc_warning(self, msg):
        """Creates a warning log"""
        self.log.warning(msg)

    def xmlrpc_error(self, msg):
        """Creates an error log"""
        self.log.error(msg)

    def xmlrpc_critical(self, msg):
        """Creates a critical log"""
        self.log.critical(msg)

    ####  File Management Functions  ####
    def xmlrpc_write(self, file_name, file_contents, timestamp=None):
        """Writes a file sent to the daemon,
        returns True on success and False on failure"""
        file_dir, _ = os.path.split(file_name)
        if file_dir and not os.path.exists(file_dir):
            os.makedirs(file_dir)
        result = True
        try:
            save_file = open(file_name, 'w+')
            save_file.write(file_contents)
            save_file.close()
            if timestamp:
                os.utime(file_name, (timestamp, timestamp))
            self.log.info('Wrote file '+file_name)
        except IOError as error:
            self.log.error('Cannot write file: '+file_name+\
                                    '\n'+str(error))
            result = False
        return result

    def xmlrpc_getFileContents(self, file_name):
        """Returns the contents of the given file"""
        file_handler = open(file_name, 'r')
        return file_handler.read()

    def xmlrpc_getConfig(self):
        """This function returns the config file as a string"""
        config_file = open(self.daemon.config_file_name, 'r')
        return config_file.read()

    def xmlrpc_sendConfig(self, config):
        """Writes the given data to the config file"""
        config_file_name = self.daemon.config_file_name
        _, temp_name = os.path.split(config_file_name)
        if temp_name == 'orpd.cfg.default':
            self.daemon.config_file_name = config_file_name = config_file_name[:-8]
        config_file = open(config_file_name, 'w+')
        config_file.write(config)
        config_file.close()

    ####  Misc Functions  ####
    def xmlrpc_ping(self):
        """Ping, Pong."""
        self.log.info('Pong')
        return True

    def xmlrpc_connect(self):
        """Called when a dashboard connects"""
        self.log.info('Receiving connection from %s' % self.client_addr[0])
        if self.client_addr[0] in self.network_handlers:
            self.log.debug('Dashboard Already Connected, not opening a new logging socket')
        else:
            # Setup Network handler
            network = logging.handlers.SocketHandler(self.client_addr[0],
                                logging.handlers.DEFAULT_TCP_LOGGING_PORT)
            network.setLevel(logging.DEBUG)
            root = logging.getLogger('')
            root.addHandler(network)
            self.log_servers.append(network)
            self.network_handlers[self.client_addr[0]] = network

    def xmlrpc_fileSync(self):
        "Synchronize the files between the server and the client"
        return self.__buildListing()

    def __buildListing(self):
        """Build a listing of the files in files and modules for connection"""
        cc_list = self.__walkDirectory(os.path.join(
            self.daemon.work_directory, 'files'))
        module_list = self.__walkDirectory(os.path.join(
            self.daemon.work_directory, 'modules'))
        return (cc_list, module_list)

    def __walkDirectory(self, path):
        """Walks the Directory"""
        listing = {}
        for root, _, files in os.walk(path):
            # if root[0] == '.':
                # continue
            file_path = root.replace(path, '')[1:]
            if file_path and filter(lambda x: x[0] == '.', file_path.split(os.sep)):
                continue
            for f in files:
                if f[0] != '.' and not f.endswith(('.hwmo', '.hwmc', '.pyo', '.pyc', '.cco', '.ccc')):
                    full_path = os.path.join(path, file_path, f)
                    listing_name = os.path.join(file_path, f)
                    listing[listing_name] = os.stat(full_path)[8]
        return listing

    def xmlrpc_disconnect(self):
        """docstring for xmlrpc_disconnect"""
        if self.client_addr[0] in self.network_handlers:
            root = logging.getLogger()
            root.removeHandler(self.network_handlers[self.client_addr[0]])
            del self.network_handlers[self.client_addr[0]]

    def xmlrpc_shutdown(self):
        """Shutsdown the system"""
        # I do this so that the xmlrpc function can return properly.
        # If you just xmlrpc.shutdown() then,
        # the return None line never gets executed.
        # So I just have it call that 100mS later to allow it to return first.
        Timer(0.1, self.__shutdown).start()
        return None

    def xmlrpc_restart(self):
        """Restarts the server"""
        self.log.info('Server Restarted')
        return self.xmlrpc_changeWorkDirectory(self.daemon.work_directory)

    def xmlrpc_changeWorkDirectory(self, work_dir):
        """This stops the services, changes the work directory,
        and then reloads everything that can be configured through config files"""
        if os.path.exists(work_dir):
            self.daemon.new_work_dir = work_dir
            self.xmlrpc_shutdown()
            return True
        return False

    def __shutdown(self):
        """Shuts down"""
        self.shutdown()
        self.socket.close()
<|endoftext|>


<commit_before>import logging
import atexit, sys, os, time

from rest_framework.parsers import JSONParser
from rest_framework.renderers import JSONRenderer
from rest_framework import serializers

from .client import BaseClient, MQTTException

from ambulance.models import Ambulance
from ambulance.models import Call
from ambulance.serializers import AmbulanceSerializer

from login.views import SettingsView

from hospital.models import Equipment, HospitalEquipment, Hospital
from hospital.serializers import HospitalSerializer, \
    HospitalEquipmentSerializer, EquipmentSerializer

from login.serializers import ExtendedProfileSerializer

logger = logging.getLogger(__name__)

# MessagePublishClient class

class MessagePublishClient():

    def publish_profile(self, profile, **kwargs):
        pass

    def publish_ambulance(self, ambulance, **kwargs):
        pass

    def remove_ambulance(self, ambulance, **kwargs):
        pass

    def publish_hospital(self, hospital, **kwargs):
        pass

    def remove_hospital(self, hospital, **kwargs):
        pass

    def publish_hospital_metadata(self, hospital, **kwargs):
        pass

    def publish_hospital_equipment(self, hospital, **kwargs):
        pass

    def remove_hospital_equipment(self, hospital, **kwargs):
        pass

    # For call

    def publish_call(self, call, **kwargs):
        pass

    def remove_call(self, call, **kwargs):
        pass

class PublishClient(BaseClient):

    def __init__(self, broker, **kwargs):

        # call super
        super().__init__(broker, **kwargs)

        # set as active
        self.active = True

    def on_disconnect(self, client, userdata, rc):
        # Exception is generated only if never connected
        if not self.connected and rc:
            raise MQTTException('Disconnected',
                                rc)
        # call super
        super().on_disconnect(client, userdata, rc)

    def publish_topic(self, topic, payload, qos=0, retain=False):

        if self.active:

            # serializer?
            if isinstance(payload, serializers.BaseSerializer):
                payload = JSONRenderer().render(payload.data)
            else:
                payload = JSONRenderer().render(payload)

            # Publish to topic
            self.publish(topic,
                         payload,
                         qos=qos,
                         retain=retain)

    def remove_topic(self, topic, qos=0):

        if self.active:

            # Publish null to retained topic
            self.publish(topic,
                         None,
                         qos=qos,
                         retain=True)

    def publish_settings(self, qos=2, retain=True):
        self.publish_topic('settings',
                           SettingsView.get_settings(),
                           qos=qos,
                           retain=retain)

    def publish_profile(self, profile, qos=2, retain=True):
        self.publish_topic('user/{}/profile'.format(profile.user.username),
                          ExtendedProfileSerializer(profile),
                          qos=qos,
                          retain=retain)

    def publish_ambulance(self, ambulance, qos=2, retain=True):
        self.publish_topic('ambulance/{}/data'.format(ambulance.id),
                          AmbulanceSerializer(ambulance),
                          qos=qos,
                          retain=retain)

    def remove_ambulance(self, ambulance):
        self.remove_topic('ambulance/{}/data'.format(ambulance.id))

    def publish_hospital(self, hospital, qos=2, retain=True):
        self.publish_topic('hospital/{}/data'.format(hospital.id),
                          HospitalSerializer(hospital),
                          qos=qos,
                          retain=retain)

    def remove_hospital(self, hospital):
        self.remove_topic('hospital/{}/data'.format(hospital.id))
        self.remove_topic('hospital/{}/metadata'.format(hospital.id))

    def publish_hospital_metadata(self, hospital, qos=2, retain=True):
        hospital_equipment = hospital.hospitalequipment_set.values('equipment')
        equipment = Equipment.objects.filter(id__in=hospital_equipment)
        self.publish_topic('hospital/{}/metadata'.format(hospital.id),
                          EquipmentSerializer(equipment, many=True),
                          qos=qos,
                          retain=retain)django app strucutredjango app strucutre

    def publish_hospital_equipment(self, equipment, qos=2, retain=True):
        self.publish_topic('hospital/{}/equipment/{}/data'.format(equipment.hospital.id,
                                                                 equipment.equipment.name),
                          HospitalEquipmentSerializer(equipment),
                          qos=qos,
                          retain=retain)

    def remove_hospital_equipment(self, equipment):
        self.remove_topic('hospital/{}/equipment/{}/data'.format(equipment.hospital.id,
                                                                 equipment.equipment.name))

    # Method to publish calls, NO SERIALIZER IMPLEMENTED, uses empty string
    # for now.
    def publish_call(self, call, qos=2, retain=True):
        self.publish_topic('call/{}/data'.format(call.id),
                           "",  #NO SERIALIZER IMPLEMENTED
                           qos = qos,
                           retain=retain)

    # Method to remove calls, no metadata?
    def remove_call(self, call):
        self.remove_topic('call/{}/data'.format(call.id))

# Uses Alex Martelli's Borg for making PublishClient act like a singleton

class SingletonPublishClient(PublishClient):

    _shared_state = {}

    def __init__(self, **kwargs):

        # Makes sure it is a singleton
        self.__dict__ = self._shared_state

        # Do not initialize if already initialized
        if not self.__dict__ == {}:
            # skip initialization
            return

        # initialization
        from django.conf import settings

        broker = {
            'HOST': 'localhost',
            'PORT': 1883,
            'KEEPALIVE': 60,
            'CLEAN_SESSION': True
        }
        broker.update(settings.MQTT)

        # override client_id
        broker['CLIENT_ID'] = 'mqtt_publish_' + str(os.getpid())

        try:

            # try to connect
            logger.info('>> Connecting to MQTT brocker...')

            # initialize PublishClient
            super().__init__(broker, **kwargs)

            # wait for connection
            while not self.connected:
                self.loop()

            # start loop
            self.loop_start()

            # register atexit handler to make sure it disconnects at exit
            atexit.register(self.disconnect)

        except MQTTException as e:

            self.active = False

            logger.info('>> Failed to connect to MQTT brocker. Will not publish updates to MQTT...')
            logger.info('>> Generated exception: {}'.format(e))

<commit_msg>Added serializer call to publish_call<commit_after>import logging
import atexit, sys, os, time

from rest_framework.parsers import JSONParser
from rest_framework.renderers import JSONRenderer
from rest_framework import serializers

from .client import BaseClient, MQTTException

from ambulance.models import Ambulance
from ambulance.models import Call
from ambulance.serializers import AmbulanceSerializer

from login.views import SettingsView

from hospital.models import Equipment, HospitalEquipment, Hospital
from hospital.serializers import HospitalSerializer, \
    HospitalEquipmentSerializer, EquipmentSerializer

from login.serializers import ExtendedProfileSerializer

logger = logging.getLogger(__name__)

# MessagePublishClient class

class MessagePublishClient():

    def publish_profile(self, profile, **kwargs):
        pass

    def publish_ambulance(self, ambulance, **kwargs):
        pass

    def remove_ambulance(self, ambulance, **kwargs):
        pass

    def publish_hospital(self, hospital, **kwargs):
        pass

    def remove_hospital(self, hospital, **kwargs):
        pass

    def publish_hospital_metadata(self, hospital, **kwargs):
        pass

    def publish_hospital_equipment(self, hospital, **kwargs):
        pass

    def remove_hospital_equipment(self, hospital, **kwargs):
        pass

    # For call

    def publish_call(self, call, **kwargs):
        pass

    def remove_call(self, call, **kwargs):
        pass

class PublishClient(BaseClient):

    def __init__(self, broker, **kwargs):

        # call super
        super().__init__(broker, **kwargs)

        # set as active
        self.active = True

    def on_disconnect(self, client, userdata, rc):
        # Exception is generated only if never connected
        if not self.connected and rc:
            raise MQTTException('Disconnected',
                                rc)
        # call super
        super().on_disconnect(client, userdata, rc)

    def publish_topic(self, topic, payload, qos=0, retain=False):

        if self.active:

            # serializer?
            if isinstance(payload, serializers.BaseSerializer):
                payload = JSONRenderer().render(payload.data)
            else:
                payload = JSONRenderer().render(payload)

            # Publish to topic
            self.publish(topic,
                         payload,
                         qos=qos,
                         retain=retain)

    def remove_topic(self, topic, qos=0):

        if self.active:

            # Publish null to retained topic
            self.publish(topic,
                         None,
                         qos=qos,
                         retain=True)

    def publish_settings(self, qos=2, retain=True):
        self.publish_topic('settings',
                           SettingsView.get_settings(),
                           qos=qos,
                           retain=retain)

    def publish_profile(self, profile, qos=2, retain=True):
        self.publish_topic('user/{}/profile'.format(profile.user.username),
                          ExtendedProfileSerializer(profile),
                          qos=qos,
                          retain=retain)

    def publish_ambulance(self, ambulance, qos=2, retain=True):
        self.publish_topic('ambulance/{}/data'.format(ambulance.id),
                          AmbulanceSerializer(ambulance),
                          qos=qos,
                          retain=retain)

    def remove_ambulance(self, ambulance):
        self.remove_topic('ambulance/{}/data'.format(ambulance.id))

    def publish_hospital(self, hospital, qos=2, retain=True):
        self.publish_topic('hospital/{}/data'.format(hospital.id),
                          HospitalSerializer(hospital),
                          qos=qos,
                          retain=retain)

    def remove_hospital(self, hospital):
        self.remove_topic('hospital/{}/data'.format(hospital.id))
        self.remove_topic('hospital/{}/metadata'.format(hospital.id))

    def publish_hospital_metadata(self, hospital, qos=2, retain=True):
        hospital_equipment = hospital.hospitalequipment_set.values('equipment')
        equipment = Equipment.objects.filter(id__in=hospital_equipment)
        self.publish_topic('hospital/{}/metadata'.format(hospital.id),
                          EquipmentSerializer(equipment, many=True),
                          qos=qos,
                          retain=retain)django app strucutredjango app strucutre

    def publish_hospital_equipment(self, equipment, qos=2, retain=True):
        self.publish_topic('hospital/{}/equipment/{}/data'.format(equipment.hospital.id,
                                                                 equipment.equipment.name),
                          HospitalEquipmentSerializer(equipment),
                          qos=qos,
                          retain=retain)

    def remove_hospital_equipment(self, equipment):
        self.remove_topic('hospital/{}/equipment/{}/data'.format(equipment.hospital.id,
                                                                 equipment.equipment.name))

    def publish_call(self, call, qos=2, retain=True):
        self.publish_topic('call/{}/data'.format(call.id),
                           CallSerializer(call),
                           qos = qos,
                           retain=retain)

    # Method to remove calls, no metadata?
    def remove_call(self, call):
        self.remove_topic('call/{}/data'.format(call.id))

# Uses Alex Martelli's Borg for making PublishClient act like a singleton

class SingletonPublishClient(PublishClient):

    _shared_state = {}

    def __init__(self, **kwargs):

        # Makes sure it is a singleton
        self.__dict__ = self._shared_state

        # Do not initialize if already initialized
        if not self.__dict__ == {}:
            # skip initialization
            return

        # initialization
        from django.conf import settings

        broker = {
            'HOST': 'localhost',
            'PORT': 1883,
            'KEEPALIVE': 60,
            'CLEAN_SESSION': True
        }
        broker.update(settings.MQTT)

        # override client_id
        broker['CLIENT_ID'] = 'mqtt_publish_' + str(os.getpid())

        try:

            # try to connect
            logger.info('>> Connecting to MQTT brocker...')

            # initialize PublishClient
            super().__init__(broker, **kwargs)

            # wait for connection
            while not self.connected:
                self.loop()

            # start loop
            self.loop_start()

            # register atexit handler to make sure it disconnects at exit
            atexit.register(self.disconnect)

        except MQTTException as e:

            self.active = False

            logger.info('>> Failed to connect to MQTT brocker. Will not publish updates to MQTT...')
            logger.info('>> Generated exception: {}'.format(e))

<|endoftext|>


<commit_before>"""Global constants for osg-build"""
import os
import sys

WD_RESULTS = '_build_results'
WD_PREBUILD = '_final_srpm_contents'
WD_UNPACKED = '_upstream_srpm_contents'
WD_UNPACKED_TARBALL = '_upstream_tarball_contents'
WD_QUILT = '_quilt'
AFS_CACHE_PATH = '/p/vdt/public/html/upstream'
AFS_CACHE_PREFIX = 'file://' + AFS_CACHE_PATH
WEB_CACHE_PREFIX = 'http://vdt.cs.wisc.edu/upstream'

KOJI_USER_CONFIG_DIR = os.path.expanduser("~/.koji")
OSG_KOJI_USER_CONFIG_DIR = os.path.expanduser("~/.osg-koji")
KOJI_CLIENT_CERT = os.path.join(OSG_KOJI_USER_CONFIG_DIR, "client.crt")

KOJI_CONF = "osg-koji-site.conf"
OLD_KOJI_CONF = "osg-koji.conf"
DATA_DIR = "/usr/share/osg-build"

KOJI_HUB = "http://koji.opensciencegrid.org"
KOJI_WEB = "https://koji.opensciencegrid.org"

DATA_FILE_SEARCH_PATH = [os.path.abspath(os.path.dirname(__file__) + "/../data")]
if "OSG_LOCATION" in os.environ:
    DATA_FILE_SEARCH_PATH.append(os.environ["OSG_LOCATION"] + DATA_DIR)
DATA_FILE_SEARCH_PATH.append(DATA_DIR)

SVN_ROOT = "https://vdt.cs.wisc.edu/svn"
SVN_REDHAT_PATH = "/native/redhat"

SVN_RESTRICTED_BRANCHES = {
    r'^trunk$'                             : 'main',
    r'^branches/upcoming$'                 : 'upcoming',
    r'^branches/osg-internal$'             : 'internal',
    r'^branches/devops$'                   : 'devops',
    r'^branches/osg-(?P<osgver>\d+\.\d+)$' : 'versioned',
    r'^branches/condor'                    : 'condor'}
KOJI_RESTRICTED_TARGETS = {
    r'^osg-(el\d+)$'                       : 'main',
    r'^osg-upcoming-(el\d+)$'              : 'upcoming',
    r'^devops-(el\d+)$'                    : 'devops',
    r'^osg-(el\d+)-internal$'              : 'internal',
    r'^osg-(?P<osgver>\d+\.\d+)-(el\d+)$'  : 'versioned',
    r'^condor-(el\d+)$'                    : 'condor'}
GIT_RESTRICTED_BRANCHES = {
    r'^(\w*/)?master$'                     : 'main',
    r'^(\w*/)?upcoming$'                   : 'upcoming',
    r'^(\w*/)?internal$'                   : 'internal',
    r'^(\w*/)?devops$'                     : 'devops',
    r'^(\w*/)?osg-(?P<osgver>\d+\.\d+)$'   : 'versioned',
    r'^(\w*/)?condor-(el\d+)'              : 'condor'}

CSL_KOJI_DIR = "/p/vdt/workspace/koji-1.11.0"

OSG_REMOTE = 'https://github.com/opensciencegrid/Software-Redhat.git'
OSG_AUTH_REMOTE = 'git@github.com:opensciencegrid/Software-Redhat.git'
HCC_REMOTE = 'https://github.com/unlhcc/hcc-packaging.git'
HCC_AUTH_REMOTE = 'git@github.com:unlhcc/hcc-packaging.git'

KNOWN_GIT_REMOTES = [HCC_REMOTE,
                     HCC_AUTH_REMOTE,
                     OSG_REMOTE,
                     OSG_AUTH_REMOTE]
# Map the authenticated URL to an anonymous checkout URL.
GIT_REMOTE_MAPS = {HCC_AUTH_REMOTE: HCC_REMOTE,
                   OSG_AUTH_REMOTE: OSG_REMOTE}

DEFAULT_BUILDOPTS_COMMON = {
    'autoclean': True,
    'background': False,
    'cache_prefix': 'AUTO',
    'dry_run': False,
    'full_extract': False,
    'getfiles': False,
    'koji_backend': None,
    'kojilogin': None,
    'koji_wrapper': True,
    'mock_clean': True,
    'mock_config': None,
    'mock_config_from_koji': None,
    'no_wait': False,
    'regen_repos': False,
    'repo': 'osg',
    'scratch': False,
    'vcs': None,
    'target_arch': None,
    'working_directory': '.',
}

DVERS = ['el6', 'el7']

DEFAULT_BUILDOPTS_BY_DVER = {}
for dver in DVERS:
    DEFAULT_BUILDOPTS_BY_DVER[dver] = dict(
        distro_tag='osg.'+dver,
        koji_tag=None,
        koji_target=None,
        redhat_release=dver[2:]
    )

# If the dver on the current machine can't be detected for some reason, or
# isn't EL, use this.
FALLBACK_DVER = 'el7'
DEFAULT_DVERS = ['el7']
DEFAULT_DVERS_BY_REPO = {
    '3.3': ['el6', 'el7'],
    'osg-3.3': ['el6', 'el7'],
    '3.4': ['el6', 'el7'],
    'osg-3.4': ['el6', 'el7'],
    '3.5': ['el7'],
    'osg-3.5': ['el7'],
    'internal': ['el6', 'el7'],
    'condor': ['el6', 'el7'],
    'devops': ['el7'],
    'upcoming': ['el7'],
}
assert FALLBACK_DVER in DVERS
for d in DEFAULT_DVERS:
    assert d in DVERS
for ds in DEFAULT_DVERS_BY_REPO.values():
    for d in ds:
        assert d in DVERS

REPO_HINTS_STATIC = {
    'osg': {'target': 'osg-%(dver)s', 'tag': 'osg-%(dver)s'},
    'upcoming': {'target': 'osg-upcoming-%(dver)s', 'tag': 'osg-%(dver)s'},
    'internal': {'target': 'osg-%(dver)s-internal', 'tag': 'osg-%(dver)s'},
    'devops': {'target': 'devops-%(dver)s', 'tag': 'osg-%(dver)s'},
    'hcc': {'target': 'hcc-%(dver)s', 'tag': 'hcc-%(dver)s'},
    'condor': {'target': 'condor-%(dver)s', 'tag': 'condor-%(dver)s'},
}

BUGREPORT_EMAIL = "osg-software@opensciencegrid.org"

BACKGROUND_THRESHOLD = 5

<commit_msg>Add el8<commit_after>"""Global constants for osg-build"""
import os
import sys

WD_RESULTS = '_build_results'
WD_PREBUILD = '_final_srpm_contents'
WD_UNPACKED = '_upstream_srpm_contents'
WD_UNPACKED_TARBALL = '_upstream_tarball_contents'
WD_QUILT = '_quilt'
AFS_CACHE_PATH = '/p/vdt/public/html/upstream'
AFS_CACHE_PREFIX = 'file://' + AFS_CACHE_PATH
WEB_CACHE_PREFIX = 'http://vdt.cs.wisc.edu/upstream'

KOJI_USER_CONFIG_DIR = os.path.expanduser("~/.koji")
OSG_KOJI_USER_CONFIG_DIR = os.path.expanduser("~/.osg-koji")
KOJI_CLIENT_CERT = os.path.join(OSG_KOJI_USER_CONFIG_DIR, "client.crt")

KOJI_CONF = "osg-koji-site.conf"
OLD_KOJI_CONF = "osg-koji.conf"
DATA_DIR = "/usr/share/osg-build"

KOJI_HUB = "http://koji.opensciencegrid.org"
KOJI_WEB = "https://koji.opensciencegrid.org"

DATA_FILE_SEARCH_PATH = [os.path.abspath(os.path.dirname(__file__) + "/../data")]
if "OSG_LOCATION" in os.environ:
    DATA_FILE_SEARCH_PATH.append(os.environ["OSG_LOCATION"] + DATA_DIR)
DATA_FILE_SEARCH_PATH.append(DATA_DIR)

SVN_ROOT = "https://vdt.cs.wisc.edu/svn"
SVN_REDHAT_PATH = "/native/redhat"

SVN_RESTRICTED_BRANCHES = {
    r'^trunk$'                             : 'main',
    r'^branches/upcoming$'                 : 'upcoming',
    r'^branches/osg-internal$'             : 'internal',
    r'^branches/devops$'                   : 'devops',
    r'^branches/osg-(?P<osgver>\d+\.\d+)$' : 'versioned',
    r'^branches/condor'                    : 'condor'}
KOJI_RESTRICTED_TARGETS = {
    r'^osg-(el\d+)$'                       : 'main',
    r'^osg-upcoming-(el\d+)$'              : 'upcoming',
    r'^devops-(el\d+)$'                    : 'devops',
    r'^osg-(el\d+)-internal$'              : 'internal',
    r'^osg-(?P<osgver>\d+\.\d+)-(el\d+)$'  : 'versioned',
    r'^condor-(el\d+)$'                    : 'condor'}
GIT_RESTRICTED_BRANCHES = {
    r'^(\w*/)?master$'                     : 'main',
    r'^(\w*/)?upcoming$'                   : 'upcoming',
    r'^(\w*/)?internal$'                   : 'internal',
    r'^(\w*/)?devops$'                     : 'devops',
    r'^(\w*/)?osg-(?P<osgver>\d+\.\d+)$'   : 'versioned',
    r'^(\w*/)?condor-(el\d+)'              : 'condor'}

CSL_KOJI_DIR = "/p/vdt/workspace/koji-1.11.0"

OSG_REMOTE = 'https://github.com/opensciencegrid/Software-Redhat.git'
OSG_AUTH_REMOTE = 'git@github.com:opensciencegrid/Software-Redhat.git'
HCC_REMOTE = 'https://github.com/unlhcc/hcc-packaging.git'
HCC_AUTH_REMOTE = 'git@github.com:unlhcc/hcc-packaging.git'

KNOWN_GIT_REMOTES = [HCC_REMOTE,
                     HCC_AUTH_REMOTE,
                     OSG_REMOTE,
                     OSG_AUTH_REMOTE]
# Map the authenticated URL to an anonymous checkout URL.
GIT_REMOTE_MAPS = {HCC_AUTH_REMOTE: HCC_REMOTE,
                   OSG_AUTH_REMOTE: OSG_REMOTE}

DEFAULT_BUILDOPTS_COMMON = {
    'autoclean': True,
    'background': False,
    'cache_prefix': 'AUTO',
    'dry_run': False,
    'full_extract': False,
    'getfiles': False,
    'koji_backend': None,
    'kojilogin': None,
    'koji_wrapper': True,
    'mock_clean': True,
    'mock_config': None,
    'mock_config_from_koji': None,
    'no_wait': False,
    'regen_repos': False,
    'repo': 'osg',
    'scratch': False,
    'vcs': None,
    'target_arch': None,
    'working_directory': '.',
}

DVERS = ['el6', 'el7', 'el8']

DEFAULT_BUILDOPTS_BY_DVER = {}
for dver in DVERS:
    DEFAULT_BUILDOPTS_BY_DVER[dver] = dict(
        distro_tag='osg.'+dver,
        koji_tag=None,
        koji_target=None,
        redhat_release=dver[2:]
    )

# If the dver on the current machine can't be detected for some reason, or
# isn't EL, use this.
FALLBACK_DVER = 'el7'
DEFAULT_DVERS = ['el7']
DEFAULT_DVERS_BY_REPO = {
    '3.3': ['el6', 'el7'],
    'osg-3.3': ['el6', 'el7'],
    '3.4': ['el6', 'el7'],
    'osg-3.4': ['el6', 'el7'],
    '3.5': ['el7'],
    'osg-3.5': ['el7'],
    'internal': ['el6', 'el7'],
    'condor': ['el6', 'el7'],
    'devops': ['el7'],
    'upcoming': ['el7'],
}
assert FALLBACK_DVER in DVERS
for d in DEFAULT_DVERS:
    assert d in DVERS
for ds in DEFAULT_DVERS_BY_REPO.values():
    for d in ds:
        assert d in DVERS

REPO_HINTS_STATIC = {
    'osg': {'target': 'osg-%(dver)s', 'tag': 'osg-%(dver)s'},
    'upcoming': {'target': 'osg-upcoming-%(dver)s', 'tag': 'osg-%(dver)s'},
    'internal': {'target': 'osg-%(dver)s-internal', 'tag': 'osg-%(dver)s'},
    'devops': {'target': 'devops-%(dver)s', 'tag': 'osg-%(dver)s'},
    'hcc': {'target': 'hcc-%(dver)s', 'tag': 'hcc-%(dver)s'},
    'condor': {'target': 'condor-%(dver)s', 'tag': 'condor-%(dver)s'},
}

BUGREPORT_EMAIL = "osg-software@opensciencegrid.org"

BACKGROUND_THRESHOLD = 5

<|endoftext|>


<commit_before>#!/usr/bin/env python2.7
#
# UI for the MultiEncoder
#
# Author: Deque
# Co-author : Psycho_Coder
# URL: https://github.com/HCDevelopers/MultiEncoder
# For license information, see LICENSE

import hcdev

encrypters = hcdev.get_encryptors()
encoders = hcdev.get_encoders()
abbreviations = hcdev.get_abbrevs()

title = """
  __  __       _ _   _ ______                     _
 |  \/  |     | | | (_)  ____|                   | |
 | \  / |_   _| | |_ _| |__   _ __   ___ ___   __| | ___ _ __
 | |\/| | | | | | __| |  __| | '_ \ / __/ _ \ / _` |/ _ \ '__|
 | |  | | |_| | | |_| | |____| | | | (_| (_) | (_| |  __/ |
 |_|  |_|\__,_|_|\__|_|______|_| |_|\___\___/ \__,_|\___|_|

                Version 0.4 at Hackcommunity.com
            Authors: Ex094, noize, Psycho_Coder, Deque
"""

def get_algorithms():

    userargs = []
    while True:
        userin = raw_input("Encodings/encryptions (type '-i algo-name' to get a description): ")
        userargs = userin.split()
        if len(userargs) >= 2 and userargs[0] == "-i":
            print_description(userargs[1])
        else:
            break
    return clean_list(userargs)

def print_description(algorithm):

    enc = None
    alglist = clean_list([algorithm])
    if len(alglist) == 1:
        alg = alglist[0]
        if alg in encoders:
            enc = encoders[alg]
        elif alg in encrypters:
            enc = encrypters[alg]
        if hasattr(enc, "description"):
            print("\n-----------------------------------------------------------------")
            print(enc.description())
            print("-----------------------------------------------------------------")
        else:
            print("No description available for " + alg)
        print("\n")

def print_algorithms():
    print("Available encodings:\n"),
    for algo in encoders.keys():
        print(algo),
        if hasattr(encoders[algo], "description"):
            print "(-i)",
    print("\n\nAvailable encryptions:\n"),
    for algo in encrypters.keys():
        print(algo),
        if hasattr(encrypters[algo], "description"):
            print("(-i)"),
    print("\n")

def en_de_code(is_decode, text, algorithms):
    print

    for alg in algorithms:
        if alg in encoders:
            encoder = encoders[alg]
            function = encoder.decode if is_decode else encoder.encode
            text = function(text)
        if alg in encrypters:
            encrypter = encrypters[alg]
            key = ask_for_key(alg, encrypter)
            function = encrypter.decode if is_decode else encrypter.encode
            text = function(text, key)
        print(" > " + alg + " : " + text)
    print("\nResult: " + text)

def ask_for_key(algorithm, gen):
    if hasattr(gen, "generate_key"):
        print "Key for", algorithm, "(type '-gen' to generate):",
        key = raw_input()
        if key == "-gen":
            key = gen.generate_key()
            print("\nGenerated key: %s \n" % key)
    else:
        print("Key for " + algorithm + " : "),
        key = raw_input()
    return key

def clean_list(dirtylist):
    cleanlist = []
    for element in dirtylist:

        if element in abbreviations:
            element = abbreviations[element]

        if element in encoders or element in encrypters:
            cleanlist.append(element)
        # one element couldn't be found, so return empty list
        else:
            print("\ncouldn't find " + element)
            return []
    return cleanlist

def ask_is_decode():
    # ask user until appropriate answer was given
    while True:
        user_input = raw_input("Encode (e) or decode (d)?")
        if user_input == "d" or user_input == "D":
            return true
        if user_input == "e" or user_input == "E":
            return false
        print("\nCan not recognize your input " + user_input)

def main():
    print title
    print_algorithms()
    algorithms = get_algorithms()
    while algorithms == []:
        algorithms = get_algorithms()
    is_decode = ask_is_decode()
    text = raw_input("Text: ")
    en_de_code(is_decode, text, algorithms)

if __name__ == "__main__":
    main()
<commit_msg>UI changed a bit<commit_after>#!/usr/bin/env python2.7
#
# UI for the MultiEncoder
#
# Author: Deque
# Co-author : Psycho_Coder
# URL: https://github.com/HCDevelopers/MultiEncoder
# For license information, see LICENSE

import hcdev

encrypters = hcdev.get_encryptors()
encoders = hcdev.get_encoders()
abbreviations = hcdev.get_abbrevs()

title = """
  __  __       _ _   _ ______                     _
 |  \/  |     | | | (_)  ____|                   | |
 | \  / |_   _| | |_ _| |__   _ __   ___ ___   __| | ___ _ __
 | |\/| | | | | | __| |  __| | '_ \ / __/ _ \ / _` |/ _ \ '__|
 | |  | | |_| | | |_| | |____| | | | (_| (_) | (_| |  __/ |
 |_|  |_|\__,_|_|\__|_|______|_| |_|\___\___/ \__,_|\___|_|

                Version 0.4 at Hackcommunity.com
            Authors: Ex094, noize, Psycho_Coder, Deque
"""

def get_algorithms():

    userargs = []
    while True:
        userin = raw_input("Encodings/encryptions (type '-i algo-name' to get a description): ")
        userargs = userin.split()
        if len(userargs) >= 2 and userargs[0] == "-i":
            print_description(userargs[1])
        else:
            break
    return clean_list(userargs)

def print_description(algorithm):

    enc = None
    alglist = clean_list([algorithm])
    if len(alglist) == 1:
        alg = alglist[0]
        if alg in encoders:
            enc = encoders[alg]
        elif alg in encrypters:
            enc = encrypters[alg]
        if hasattr(enc, "description"):
            print("\n-----------------------------------------------------------------")
            print(enc.description())
            print("-----------------------------------------------------------------")
        else:
            print("No description available for " + alg)
        print("\n")

def print_algorithms():
    print("Available encodings:\n"),
    for algo in encoders.keys():
        print(algo),
        if hasattr(encoders[algo], "description"):
            print "(-i)",
    print("\n\nAvailable encryptions:\n"),
    for algo in encrypters.keys():
        print(algo),
        if hasattr(encrypters[algo], "description"):
            print("(-i)"),
    print("\n")

def en_de_code(is_decode, text, algorithms):
    print

    for alg in algorithms:
        if alg in encoders:
            encoder = encoders[alg]
            function = encoder.decode if is_decode else encoder.encode
            text = function(text)
        if alg in encrypters:
            encrypter = encrypters[alg]
            key = ask_for_key(alg, encrypter)
            function = encrypter.decode if is_decode else encrypter.encode
            text = function(text, key)
        print(" > " + alg + " : " + text)
    print("\nResult: " + text)

def ask_for_key(algorithm, gen):
    if hasattr(gen, "generate_key"):
        print "Key for", algorithm, "(type '-gen' to generate):",
        key = raw_input()
        if key == "-gen":
            key = gen.generate_key()
            print("\nGenerated key: %s \n" % key)
    else:
        print("Key for " + algorithm + " : "),
        key = raw_input()
    return key

def clean_list(dirtylist):
    cleanlist = []
    for element in dirtylist:

        if element in abbreviations:
            element = abbreviations[element]

        if element in encoders or element in encrypters:
            cleanlist.append(element)
        # one element couldn't be found, so return empty list
        else:
            print("\ncouldn't find " + element)
            return []
    return cleanlist

def ask_is_decode():
    # ask user until appropriate answer was given
    while True:
        user_input = raw_input("Encode (e) or decode (d)?")
        if user_input == "d" or user_input == "D":
            return True
        if user_input == "e" or user_input == "E":
            return False
        print("\nCan not recognize your input " + user_input)

def main():
    print title
    print_algorithms()
    algorithms = get_algorithms()
    while not algorithms:
        algorithms = get_algorithms()
    is_decode = ask_is_decode()
    text = raw_input("Text: ")
    en_de_code(is_decode, text, algorithms)

if __name__ == "__main__":
    main()
<|endoftext|>