<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

__author__ = 'leferrad'

from learninspy.core.model import NetworkParameters
from learninspy.core.autoencoder import AutoEncoder, StackedAutoencoder
from learninspy.core.optimization import OptimizerParameters
from learninspy.core.stops import criterion
from learninspy.utils.data import LocalLabeledDataSet
from learninspy.utils.fileio import get_logger

import os

logger = get_logger(name=__name__)

class TestAutoEncoder(object):
    def __init__(self, network_params=None, dropout_in=0.0):
        logger.info("Testeo de AutoEncoder con datos de Iris")
        # Datos
        logger.info("Cargando datos...")
        dataset = LocalLabeledDataSet()
        path = os.path.abspath(os.path.join(os.path.realpath(__file__),
                                            os.path.pardir,
                                            os.pardir,
                                            'examples/datasets/iris.csv'))
        dataset.load_file(path)
        self.train, self.valid, self.test = dataset.split_data([.5, .3, .2])
        self.train = self.train.collect()
        self.valid = self.valid.collect()
        self.test = self.test.collect()

        # Modelo
        if network_params is None:
            network_params = NetworkParameters(units_layers=[4, 8], activation=['ReLU', 'ReLU'],
                                               classification=False)
        self.model = AutoEncoder(network_params, dropout_in=dropout_in)

    def _fit(self, opt_params=None, stops=None, mini_batch=30, parallelism=2):
        if opt_params is None:
            opt_params = OptimizerParameters(algorithm='Adadelta')
        if stops is None:
            stops = [criterion['MaxIterations'](30),
                     criterion['AchieveTolerance'](0.95, key='hits')]
        logger.info("Entrenando modelo...")
        hits_valid = self.model.fit(self.train, self.valid, valid_iters=1, mini_batch=mini_batch,
                                    parallelism=parallelism, stops=stops,optimizer_params=opt_params,
                                    keep_best=True, reproducible=True)
        return hits_valid

    def test_fitting(self, opt_params=None, stops=None, mini_batch=30, parallelism=1):
        hits_valid = self._fit(opt_params=opt_params, stops=stops, mini_batch=mini_batch, parallelism=parallelism)
        logger.info("Asegurando salidas correctas...")
        assert hits_valid > 0.8
        hits_test = self.model.evaluate(self.test, predictions=False, measure='R2')
        assert hits_test > 0.8
        logger.info("OK")

class TestStackedAutoEncoder(object):
    def __init__(self, network_params=None, dropout=None):
        logger.info("Testeo de StackedAutoEncoder con datos de Iris")
        # Datos
        logger.info("Cargando datos...")
        dataset = LocalLabeledDataSet()
        path = os.path.abspath(os.path.join(os.path.realpath(__file__),
                                            os.path.pardir,
                                            os.pardir,
                                            'examples/datasets/iris.csv'))
        dataset.load_file(path)
        self.train, self.valid, self.test = dataset.split_data([.5, .3, .2])
        self.train = self.train.collect()
        self.valid = self.valid.collect()
        self.test = self.test.collect()

        # Modelo
        if network_params is None:
            network_params = NetworkParameters(units_layers=[4, 8, 3], activation=['ReLU', 'ReLU'],
                                               strength_l1=1e-7, strength_l2=1e-4,
                                               dropout_ratios=[0.5, 0.0], classification=True)
        self.model = StackedAutoencoder(network_params, dropout=dropout)

    def _fit(self, opt_params=None, stops=None, mini_batch=30, parallelism=2):
        if opt_params is None:
            opt_params = OptimizerParameters(algorithm='Adadelta')
        if stops is None:
            stops = [criterion['MaxIterations'](30),
                     criterion['AchieveTolerance'](0.95, key='hits')]
        logger.info("Entrenando modelo...")
        hits_valid = self.model.fit(self.train, self.valid, valid_iters=1, mini_batch=mini_batch,
                                    parallelism=parallelism, stops=stops,optimizer_params=opt_params,
                                    keep_best=True, reproducible=True)
        return hits_valid

    def _finetune(self, opt_params=None, stops=None, mini_batch=30, parallelism=2):
        if opt_params is None:
            opt_params = OptimizerParameters(algorithm='Adadelta')
        if stops is None:
            stops = [criterion['MaxIterations'](30),
                     criterion['AchieveTolerance'](0.95, key='hits')]
        logger.info("Entrenando modelo...")
        hits_valid = self.model.finetune(self.train, self.valid, valid_iters=1, mini_batch=mini_batch,
                                         parallelism=parallelism, stops=stops,optimizer_params=opt_params,
                                         keep_best=True, reproducible=True)
        return hits_valid

    def test_fitting(self, opt_params=None, stops=None, mini_batch=30, parallelism=1):
        hits_valid_pretrain = self._fit(opt_params=opt_params, stops=stops,
                                        mini_batch=mini_batch, parallelism=parallelism)
        hits_test_pretrain = self.model.evaluate(self.test, predictions=False, measure='F-measure')
        hits_valid_finetune = self._finetune(opt_params=opt_params, stops=stops,
                                             mini_batch=mini_batch, parallelism=parallelism)
        hits_test_finetune = self.model.evaluate(self.test, predictions=False, measure='F-measure')
        logger.info("Asegurando salidas correctas...")
        assert hits_valid_pretrain > 0.8
        assert hits_valid_finetune > 0.8
        assert hits_test_pretrain > 0.8
        assert hits_test_finetune > 0.8
        logger.info("OK")
<commit_msg>Cambios en tests de AEs<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

__author__ = 'leferrad'

from learninspy.core.model import NetworkParameters
from learninspy.core.autoencoder import AutoEncoder, StackedAutoencoder
from learninspy.core.optimization import OptimizerParameters
from learninspy.core.stops import criterion
from learninspy.utils.data import LocalLabeledDataSet
from learninspy.utils.fileio import get_logger

import os

logger = get_logger(name=__name__)

class TestAutoEncoder(object):
    def __init__(self, network_params=None, dropout_in=0.0):
        logger.info("Testeo de AutoEncoder con datos de Iris")
        # Datos
        logger.info("Cargando datos...")
        dataset = LocalLabeledDataSet()
        path = os.path.abspath(os.path.join(os.path.realpath(__file__),
                                            os.path.pardir,
                                            os.pardir,
                                            'examples/datasets/iris.csv'))
        dataset.load_file(path)
        self.train, self.valid, self.test = dataset.split_data([.5, .3, .2])
        self.train = self.train.collect()
        self.valid = self.valid.collect()
        self.test = self.test.collect()

        # Modelo
        if network_params is None:
            network_params = NetworkParameters(units_layers=[4, 8], activation=['ReLU', 'ReLU'],
                                               classification=False)
        self.model = AutoEncoder(network_params, dropout_in=dropout_in)

    def _fit(self, opt_params=None, stops=None, mini_batch=30, parallelism=2):
        if opt_params is None:
            opt_params = OptimizerParameters(algorithm='Adadelta')
        if stops is None:
            stops = [criterion['MaxIterations'](30),
                     criterion['AchieveTolerance'](0.95, key='hits')]
        logger.info("Entrenando modelo...")
        hits_valid = self.model.fit(self.train, self.valid, valid_iters=1, mini_batch=mini_batch,
                                    parallelism=parallelism, stops=stops,optimizer_params=opt_params,
                                    keep_best=True, reproducible=True)
        return hits_valid

    def test_fitting(self, opt_params=None, stops=None, mini_batch=30, parallelism=1):
        hits_valid = self._fit(opt_params=opt_params, stops=stops, mini_batch=mini_batch, parallelism=parallelism)
        logger.info("Asegurando salidas correctas...")
        assert hits_valid > 0.8
        hits_test = self.model.evaluate(self.test, predictions=False, measure='R2')
        assert hits_test > 0.8
        logger.info("OK")

class TestStackedAutoEncoder(object):
    def __init__(self, network_params=None, dropout=None):
        logger.info("Testeo de StackedAutoEncoder con datos de Iris")
        # Datos
        logger.info("Cargando datos...")
        dataset = LocalLabeledDataSet()
        path = os.path.abspath(os.path.join(os.path.realpath(__file__),
                                            os.path.pardir,
                                            os.pardir,
                                            'examples/datasets/iris.csv'))
        dataset.load_file(path)
        self.train, self.valid, self.test = dataset.split_data([.5, .3, .2])
        self.train = self.train.collect()
        self.valid = self.valid.collect()
        self.test = self.test.collect()

        # Modelo
        if network_params is None:
            network_params = NetworkParameters(units_layers=[4, 10, 3], activation=['ReLU', 'ReLU'],
                                               strength_l1=1e-5, strength_l2=3e-4,
                                               dropout_ratios=[0.0, 0.0], classification=True)
        self.model = StackedAutoencoder(network_params, dropout=dropout)

    def _fit(self, opt_params=None, stops=None, mini_batch=30, parallelism=2):
        if opt_params is None:
            options  = {'step-rate': 1.0, 'decay': 0.9, 'momentum': 0.0, 'offset': 1e-8}
            opt_params = OptimizerParameters(algorithm='Adadelta')
        if stops is None:
            stops = [criterion['MaxIterations'](30),
                     criterion['AchieveTolerance'](0.95, key='hits')]
        logger.info("Entrenando modelo...")
        hits_valid = self.model.fit(self.train, self.valid, valid_iters=1, mini_batch=mini_batch,
                                    parallelism=parallelism, stops=stops,optimizer_params=opt_params,
                                    keep_best=True, reproducible=True)
        return hits_valid

    def _finetune(self, opt_params=None, stops=None, mini_batch=30, parallelism=2):
        if opt_params is None:
            options  = {'step-rate': 1.0, 'decay': 0.9, 'momentum': 0.0, 'offset': 1e-8}
            opt_params = OptimizerParameters(algorithm='Adadelta', options=options)
        if stops is None:
            stops = [criterion['MaxIterations'](30),
                     criterion['AchieveTolerance'](0.95, key='hits')]
        logger.info("Entrenando modelo...")
        hits_valid = self.model.finetune(self.train, self.valid, valid_iters=1, mini_batch=mini_batch,
                                         parallelism=parallelism, stops=stops,optimizer_params=opt_params,
                                         keep_best=True, reproducible=True)
        return hits_valid

    def test_fitting(self, opt_params=None, stops=None, mini_batch=30, parallelism=1):
        hits_valid_pretrain = self._fit(opt_params=opt_params, stops=stops,
                                        mini_batch=mini_batch, parallelism=parallelism)
        hits_test_pretrain = self.model.evaluate(self.test, predictions=False, measure='F-measure')
        hits_valid_finetune = self._finetune(opt_params=opt_params, stops=stops,
                                             mini_batch=mini_batch, parallelism=parallelism)
        hits_test_finetune = self.model.evaluate(self.test, predictions=False, measure='F-measure')
        logger.info("Asegurando salidas correctas...")
        assert hits_valid_pretrain > 0.7
        assert hits_valid_finetune > 0.8
        assert hits_test_pretrain > 0.7
        assert hits_test_finetune > 0.8
        logger.info("OK")<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""

@author: jussi
"""

from __future__ import print_function
import sys
from PyQt4 import QtGui, QtCore, uic
from PyQt4.QtCore import pyqtSignal
import time
import struct
import FieldTrip
import numpy as np
import scipy.linalg
import os.path as op
import ast
import traceback
import socket
from config import Config
import elekta
from rt_server import start_rt_server, stop_rt_server, rt_server_pid

DEBUG = False

def debug_print(*args):
    if DEBUG:
        print(*args)

class HPImon(QtGui.QMainWindow):

    # new signals must be defined here
    new_data = pyqtSignal()

    def __init__(self):
        super(self.__class__, self).__init__()
        self.apptitle = 'hpimon'
        # load user interface made with designer
        uic.loadUi('hpimon.ui', self)
        self.setWindowTitle(self.apptitle)
        self.cfg = Config()
        self.timer = QtCore.QTimer()
        try:
            self.cfg.read()
        except ValueError:
            self.message_dialog('Cannot parse config file, creating %s '
                                'with default values. Please edit the file '
                                'according to your setup and restart.' %
                                self.cfg.configfile)
            self.cfg.write()
            sys.exit()

        """ Parse some options """
        linefreq_, cfreqs_ = elekta.read_collector_config()
        self.linefreq = self.cfg.LINE_FREQ or linefreq_
        self.cfreqs = ast.literal_eval(self.cfg.HPI_FREQS) or cfreqs_
        if not self.cfreqs:
            self.message_dialog('Cannot detect HPI frequencies and none are '
                                'specified in the config file. Aborting.')
            sys.exit()
        self.ncoils = len(self.cfreqs)
        self.SNR_COLORS = ast.literal_eval(self.cfg.SNR_COLORS)  # str to dict

        self.serverp = None
        if self.cfg.SERVER_AUTOSTART:
            if not op.isfile(self.cfg.SERVER_PATH):
                self.message_dialog('Cannot find server binary at %s, please '
                                    'check config.' % self.cfg.SERVER_PATH)
                sys.exit()
            server_bin = op.split(self.cfg.SERVER_PATH)[1]
            if rt_server_pid(server_bin):
                self.message_dialog('Realtime server already running. '
                                    'Please kill it first.')
                sys.exit()
            if self.cfg.HOST == 'localhost':
                debug_print('starting server')
                self.serverp = start_rt_server(self.cfg.SERVER_PATH,
                                               self.cfg.SERVER_OPTS.split())
                if not rt_server_pid(server_bin):
                    self.message_dialog('Could not start realtime server.')
                    sys.exit()
                time.sleep(1)  # give the server a second to get started

        self.init_widgets()
        self.ftclient = FieldTrip.Client()
        try:
            self.ftclient.connect(self.cfg.HOST, port=self.cfg.PORT)
        except socket.error:
            self.message_dialog('Cannot connect to the realtime server. '
                                'Possibly a networking problem or you have '
                                'specified a wrong TCP port.')
            stop_rt_server(self.serverp)
            sys.exit()

        """ Poll using timer until header info becomes available """
        self.statusbar.showMessage('Waiting for measurement to start...')
        self.timer.timeout.connect(self.start_if_header)
        self.timer.start(self.cfg.BUFFER_POLL_INTERVAL)

    def start_if_header(self):
        """ Start if header info has become available. """
        debug_print('polling for header info')
        if self.ftclient.getHeader():
            self.start()

    def start(self):
        """ We now have header info and can start running """
        self.timer.stop()
        self.pick_mag, self.pick_grad = self.get_ch_indices()
        self.pick_meg = np.sort(np.concatenate([self.pick_mag,
                                                self.pick_grad]))
        self.nchan = len(self.pick_meg)
        self.sfreq = self.get_header_info()['sfreq']
        self.init_glm()
        self.last_sample = self.buffer_last_sample()
        #self.last_sample = -1
        self.new_data.connect(self.update_snr_display)
        # from now on, use the timer for data buffer polling
        self.timer.timeout.disconnect(self.start_if_header)
        self.timer.timeout.connect(self.poll_buffer)
        self.timer.start(self.cfg.BUFFER_POLL_INTERVAL)
        self.statusbar.showMessage(self.msg_running())

    def init_widgets(self):
        # labels
        for wnum in range(5):
            lbname = 'label_' + str(wnum + 1)
            self.__dict__[lbname].setText(str(self.cfreqs[wnum]) + ' Hz')
        # progress bars
        for wnum in range(5):
            wname = 'progressBar_' + str(wnum + 1)
            sty = '.QProgressBar {'
            sty += self.cfg.BAR_STYLE
            sty += ' }'
            self.__dict__[wname].setStyleSheet(sty)
        # stylesheets for progress bars
        self.progbar_styles = dict()
        for snr in ['good', 'ok', 'bad']:
            sty = ('QProgressBar {%s} QProgressBar::chunk { background-color: '
                   '%s; %s }' % (self.cfg.BAR_STYLE, self.SNR_COLORS[snr],
                                 self.cfg.BAR_CHUNK_STYLE))
            self.progbar_styles[snr] = sty
        # buttons
        self.btnQuit.clicked.connect(self.close)
        self.btnStop.clicked.connect(self.toggle_timer)

    def toggle_timer(self):
        if self.timer.isActive():
            self.statusbar.showMessage(self.msg_stopped())
            self.btnStop.setText('Start monitoring')
            self.timer.stop()
        else:
            self.statusbar.showMessage(self.msg_running())
            self.btnStop.setText('Stop monitoring')
            self.timer.start()

    def msg_running(self):
        return ('Running [%s], poll every %d ms, window %d ms' %
                (self.cfg.HOST, self.cfg.BUFFER_POLL_INTERVAL,
                 self.cfg.WIN_LEN))

    def msg_stopped(self):
        raise ValueError
        return 'Stopped'

    def get_ch_indices(self):
        """ Return indices of magnetometers and gradiometers in the
        FieldTrip data matrix """
        grads, mags = [], []
        for ind, ch in enumerate(self.ftclient.getHeader().labels):
            if ch[:3] == 'MEG':
                if ch[-1] == '1':
                    mags.append(ind)
                elif ch[-1] in ['2', '3']:
                    grads.append(ind)
                else:
                    raise ValueError('Unexpected channel name: ' + ch)
        debug_print('Got %d magnetometers and %d gradiometers' %
                    (len(mags), len(grads)))
        return np.array(mags), np.array(grads)

    def get_header_info(self):
        """ Get misc info from FieldTrip header """
        return {'sfreq': self.ftclient.getHeader().fSample}

    def buffer_last_sample(self):
        """ Return number of last sample available from server. """
        return self.ftclient.getHeader().nSamples

    def poll_buffer(self):
        """ Emit a signal if new data is available in the buffer. """
        buflast = self.buffer_last_sample()
        debug_print('polling buffer, buffer last sample: %d, my last sample: %d' %
                    (buflast, self.last_sample))
        # buffer last sample can also decrease (reset) if streaming from file
        if buflast != self.last_sample:
            debug_print('poll: new data')
            self.new_data.emit()
            self.last_sample = buflast
        else:
            debug_print('poll: no new data')

    def fetch_buffer(self):
        start = self.last_sample - self.cfg.WIN_LEN + 1
        stop = self.last_sample
        debug_print('fetching buffer from %d to %d' % (start, stop))
        try:
            data = self.ftclient.getData([start, stop])
        except struct.error:  # something wrong with the buffer
            return None
        if data is None:
            debug_print('warning: server returned no data')
            return None
        else:
            return data[:, self.pick_meg]

    def init_glm(self):
        """ Build general linear model for amplitude estimation """
        # get some info from fiff
        self.linefreqs = (np.arange(self.cfg.NHARM+1)+1) * self.linefreq
        # time + dc and slope terms
        t = np.arange(self.cfg.WIN_LEN) / float(self.sfreq)
        self.model = np.empty((len(t),
                               2+2*(len(self.linefreqs)+len(self.cfreqs))))
        self.model[:, 0] = t
        self.model[:, 1] = np.ones(t.shape)
        # add sine and cosine term for each freq
        allfreqs = np.concatenate([self.linefreqs, self.cfreqs])
        self.model[:, 2::2] = np.cos(2 * np.pi * t[:, np.newaxis] * allfreqs)
        self.model[:, 3::2] = np.sin(2 * np.pi * t[:, np.newaxis] * allfreqs)
        self.inv_model = scipy.linalg.pinv(self.model)

    def compute_snr(self, data):
        #im = np.ascontiguousarray(self.inv_model)
        #da = np.ascontiguousarray(data)
        #coeffs = np.dot(im, da)
        coeffs = np.dot(self.inv_model, data)  # nterms * nchan
        coeffs_hpi = coeffs[2+2*len(self.linefreqs):]
        resid_vars = np.var(data - np.dot(self.model, coeffs), 0)
        # get total power by combining sine and cosine terms
        # sinusoidal of amplitude A has power of A**2/2
        hpi_pow = (coeffs_hpi[0::2, :]**2 + coeffs_hpi[1::2, :]**2)/2
        # average across channel types separately
        hpi_pow_grad_avg = hpi_pow[:, self.pick_grad].mean(1)
        # hpi_pow_mag_avg = hpi_pow[:, self.pick_mag].mean(1)
        # divide average HPI power by average variance
        snr_avg_grad = hpi_pow_grad_avg / \
            resid_vars[self.pick_grad].mean()
        # snr_avg_mag = hpi_pow_mag_avg / \
        #    resid_vars[self.pick_mag].mean()
        return 10 * np.log10(snr_avg_grad)

    def update_snr_display(self):
        buf = self.fetch_buffer()
        if buf is not None:
            snr = self.compute_snr(buf)
            for wnum in range(1, self.ncoils+1):
                wname = 'progressBar_' + str(wnum)
                this_snr = int(np.round(snr[wnum-1]))
                self.__dict__[wname].setValue(this_snr)
                if this_snr > self.cfg.SNR_OK:
                    sty = self.progbar_styles['good']
                elif this_snr > self.cfg.SNR_BAD:
                    sty = self.progbar_styles['ok']
                else:
                    sty = self.progbar_styles['bad']
                self.__dict__[wname].setStyleSheet(sty)

    def message_dialog(self, msg):
        """ Show message with an 'OK' button. """
        dlg = QtGui.QMessageBox()
        dlg.setWindowTitle(self.apptitle)
        dlg.setText(msg)
        dlg.addButton(QtGui.QPushButton('Ok'), QtGui.QMessageBox.YesRole)
        dlg.exec_()

    def closeEvent(self, event):
        """ Confirm and close application. """
        self.timer.stop()
        # disconnect from server
        if self.ftclient:
            self.ftclient.disconnect()
        stop_rt_server(self.serverp)
        event.accept()

def main():

    app = QtGui.QApplication(sys.argv)
    hpimon = HPImon()

    def my_excepthook(type, value, tback):
        """ Custom exception handler for fatal (unhandled) exceptions:
        report to user via GUI and terminate. """
        tb_full = u''.join(traceback.format_exception(type, value, tback))
        hpimon.message_dialog('Terminating due to unhandled exception. %s'
                              % tb_full)
        stop_rt_server(hpimon.serverp)
        sys.__excepthook__(type, value, tback)
        app.quit()

    sys.excepthook = my_excepthook
    hpimon.show()
    app.exec_()

if __name__ == '__main__':
    main()

<commit_msg>wip: support variable number of cfreqs<commit_after># -*- coding: utf-8 -*-
"""

@author: jussi
"""

from __future__ import print_function
import sys
from PyQt4 import QtGui, QtCore, uic
from PyQt4.QtCore import pyqtSignal
import time
import struct
import FieldTrip
import numpy as np
import scipy.linalg
import os.path as op
import ast
import traceback
import socket
from config import Config
import elekta
from rt_server import start_rt_server, stop_rt_server, rt_server_pid

DEBUG = False

def debug_print(*args):
    if DEBUG:
        print(*args)

class HPImon(QtGui.QMainWindow):

    # new signals must be defined here
    new_data = pyqtSignal()

    def __init__(self):
        super(self.__class__, self).__init__()
        self.apptitle = 'hpimon'
        # load user interface made with designer
        uic.loadUi('hpimon.ui', self)
        self.setWindowTitle(self.apptitle)
        self.cfg = Config()
        self.timer = QtCore.QTimer()
        try:
            self.cfg.read()
        except ValueError:
            self.message_dialog('Cannot parse config file, creating %s '
                                'with default values. Please edit the file '
                                'according to your setup and restart.' %
                                self.cfg.configfile)
            self.cfg.write()
            sys.exit()

        """ Parse some options """
        linefreq_, cfreqs_ = elekta.read_collector_config()
        self.linefreq = self.cfg.LINE_FREQ or linefreq_
        self.cfreqs = ast.literal_eval(self.cfg.HPI_FREQS) or cfreqs_
        if not self.cfreqs:
            self.message_dialog('Cannot detect HPI frequencies and none are '
                                'specified in the config file. Aborting.')
            sys.exit()
        self.ncoils = len(self.cfreqs)
        self.SNR_COLORS = ast.literal_eval(self.cfg.SNR_COLORS)  # str to dict

        self.serverp = None
        if self.cfg.SERVER_AUTOSTART:
            if not op.isfile(self.cfg.SERVER_PATH):
                self.message_dialog('Cannot find server binary at %s, please '
                                    'check config.' % self.cfg.SERVER_PATH)
                sys.exit()
            server_bin = op.split(self.cfg.SERVER_PATH)[1]
            if rt_server_pid(server_bin):
                self.message_dialog('Realtime server already running. '
                                    'Please kill it first.')
                sys.exit()
            if self.cfg.HOST == 'localhost':
                debug_print('starting server')
                self.serverp = start_rt_server(self.cfg.SERVER_PATH,
                                               self.cfg.SERVER_OPTS.split())
                if not rt_server_pid(server_bin):
                    self.message_dialog('Could not start realtime server.')
                    sys.exit()
                time.sleep(1)  # give the server a second to get started

        self.init_widgets()
        self.ftclient = FieldTrip.Client()
        try:
            self.ftclient.connect(self.cfg.HOST, port=self.cfg.PORT)
        except socket.error:
            self.message_dialog('Cannot connect to the realtime server. '
                                'Possibly a networking problem or you have '
                                'specified a wrong TCP port.')
            stop_rt_server(self.serverp)
            sys.exit()

        """ Poll using timer until header info becomes available """
        self.statusbar.showMessage('Waiting for measurement to start...')
        self.timer.timeout.connect(self.start_if_header)
        self.timer.start(self.cfg.BUFFER_POLL_INTERVAL)

    def start_if_header(self):
        """ Start if header info has become available. """
        debug_print('polling for header info')
        if self.ftclient.getHeader():
            self.start()

    def start(self):
        """ We now have header info and can start running """
        self.timer.stop()
        self.pick_mag, self.pick_grad = self.get_ch_indices()
        self.pick_meg = np.sort(np.concatenate([self.pick_mag,
                                                self.pick_grad]))
        self.nchan = len(self.pick_meg)
        self.sfreq = self.get_header_info()['sfreq']
        self.init_glm()
        self.last_sample = self.buffer_last_sample()
        #self.last_sample = -1
        self.new_data.connect(self.update_snr_display)
        # from now on, use the timer for data buffer polling
        self.timer.timeout.disconnect(self.start_if_header)
        self.timer.timeout.connect(self.poll_buffer)
        self.timer.start(self.cfg.BUFFER_POLL_INTERVAL)
        self.statusbar.showMessage(self.msg_running())

    def init_widgets(self):
        # labels
        for wnum in range(self.ncoils):
            lbname = 'label_' + str(wnum + 1)
            self.__dict__[lbname].setText(str(self.cfreqs[wnum]) + ' Hz')
        # progress bars
        for wnum in range(self.ncoils):
            wname = 'progressBar_' + str(wnum + 1)
            sty = '.QProgressBar {'
            sty += self.cfg.BAR_STYLE
            sty += ' }'
            self.__dict__[wname].setStyleSheet(sty)
        # stylesheets for progress bars
        self.progbar_styles = dict()
        for snr in ['good', 'ok', 'bad']:
            sty = ('QProgressBar {%s} QProgressBar::chunk { background-color: '
                   '%s; %s }' % (self.cfg.BAR_STYLE, self.SNR_COLORS[snr],
                                 self.cfg.BAR_CHUNK_STYLE))
            self.progbar_styles[snr] = sty
        # buttons
        self.btnQuit.clicked.connect(self.close)
        self.btnStop.clicked.connect(self.toggle_timer)

    def toggle_timer(self):
        if self.timer.isActive():
            self.statusbar.showMessage(self.msg_stopped())
            self.btnStop.setText('Start monitoring')
            self.timer.stop()
        else:
            self.statusbar.showMessage(self.msg_running())
            self.btnStop.setText('Stop monitoring')
            self.timer.start()

    def msg_running(self):
        return ('Running [%s], poll every %d ms, window %d ms' %
                (self.cfg.HOST, self.cfg.BUFFER_POLL_INTERVAL,
                 self.cfg.WIN_LEN))

    def msg_stopped(self):
        raise ValueError
        return 'Stopped'

    def get_ch_indices(self):
        """ Return indices of magnetometers and gradiometers in the
        FieldTrip data matrix """
        grads, mags = [], []
        for ind, ch in enumerate(self.ftclient.getHeader().labels):
            if ch[:3] == 'MEG':
                if ch[-1] == '1':
                    mags.append(ind)
                elif ch[-1] in ['2', '3']:
                    grads.append(ind)
                else:
                    raise ValueError('Unexpected channel name: ' + ch)
        debug_print('Got %d magnetometers and %d gradiometers' %
                    (len(mags), len(grads)))
        return np.array(mags), np.array(grads)

    def get_header_info(self):
        """ Get misc info from FieldTrip header """
        return {'sfreq': self.ftclient.getHeader().fSample}

    def buffer_last_sample(self):
        """ Return number of last sample available from server. """
        return self.ftclient.getHeader().nSamples

    def poll_buffer(self):
        """ Emit a signal if new data is available in the buffer. """
        buflast = self.buffer_last_sample()
        debug_print('polling buffer, buffer last sample: %d, my last sample: %d' %
                    (buflast, self.last_sample))
        # buffer last sample can also decrease (reset) if streaming from file
        if buflast != self.last_sample:
            debug_print('poll: new data')
            self.new_data.emit()
            self.last_sample = buflast
        else:
            debug_print('poll: no new data')

    def fetch_buffer(self):
        start = self.last_sample - self.cfg.WIN_LEN + 1
        stop = self.last_sample
        debug_print('fetching buffer from %d to %d' % (start, stop))
        try:
            data = self.ftclient.getData([start, stop])
        except struct.error:  # something wrong with the buffer
            return None
        if data is None:
            debug_print('warning: server returned no data')
            return None
        else:
            return data[:, self.pick_meg]

    def init_glm(self):
        """ Build general linear model for amplitude estimation """
        # get some info from fiff
        self.linefreqs = (np.arange(self.cfg.NHARM+1)+1) * self.linefreq
        # time + dc and slope terms
        t = np.arange(self.cfg.WIN_LEN) / float(self.sfreq)
        self.model = np.empty((len(t),
                               2+2*(len(self.linefreqs)+len(self.cfreqs))))
        self.model[:, 0] = t
        self.model[:, 1] = np.ones(t.shape)
        # add sine and cosine term for each freq
        allfreqs = np.concatenate([self.linefreqs, self.cfreqs])
        self.model[:, 2::2] = np.cos(2 * np.pi * t[:, np.newaxis] * allfreqs)
        self.model[:, 3::2] = np.sin(2 * np.pi * t[:, np.newaxis] * allfreqs)
        self.inv_model = scipy.linalg.pinv(self.model)

    def compute_snr(self, data):
        #im = np.ascontiguousarray(self.inv_model)
        #da = np.ascontiguousarray(data)
        #coeffs = np.dot(im, da)
        coeffs = np.dot(self.inv_model, data)  # nterms * nchan
        coeffs_hpi = coeffs[2+2*len(self.linefreqs):]
        resid_vars = np.var(data - np.dot(self.model, coeffs), 0)
        # get total power by combining sine and cosine terms
        # sinusoidal of amplitude A has power of A**2/2
        hpi_pow = (coeffs_hpi[0::2, :]**2 + coeffs_hpi[1::2, :]**2)/2
        # average across channel types separately
        hpi_pow_grad_avg = hpi_pow[:, self.pick_grad].mean(1)
        # hpi_pow_mag_avg = hpi_pow[:, self.pick_mag].mean(1)
        # divide average HPI power by average variance
        snr_avg_grad = hpi_pow_grad_avg / \
            resid_vars[self.pick_grad].mean()
        # snr_avg_mag = hpi_pow_mag_avg / \
        #    resid_vars[self.pick_mag].mean()
        return 10 * np.log10(snr_avg_grad)

    def update_snr_display(self):
        buf = self.fetch_buffer()
        if buf is not None:
            snr = self.compute_snr(buf)
            for wnum in range(1, self.ncoils+1):
                wname = 'progressBar_' + str(wnum)
                this_snr = int(np.round(snr[wnum-1]))
                self.__dict__[wname].setValue(this_snr)
                if this_snr > self.cfg.SNR_OK:
                    sty = self.progbar_styles['good']
                elif this_snr > self.cfg.SNR_BAD:
                    sty = self.progbar_styles['ok']
                else:
                    sty = self.progbar_styles['bad']
                self.__dict__[wname].setStyleSheet(sty)

    def message_dialog(self, msg):
        """ Show message with an 'OK' button. """
        dlg = QtGui.QMessageBox()
        dlg.setWindowTitle(self.apptitle)
        dlg.setText(msg)
        dlg.addButton(QtGui.QPushButton('Ok'), QtGui.QMessageBox.YesRole)
        dlg.exec_()

    def closeEvent(self, event):
        """ Confirm and close application. """
        self.timer.stop()
        # disconnect from server
        if self.ftclient:
            self.ftclient.disconnect()
        stop_rt_server(self.serverp)
        event.accept()

def main():

    app = QtGui.QApplication(sys.argv)
    hpimon = HPImon()

    def my_excepthook(type, value, tback):
        """ Custom exception handler for fatal (unhandled) exceptions:
        report to user via GUI and terminate. """
        tb_full = u''.join(traceback.format_exception(type, value, tback))
        hpimon.message_dialog('Terminating due to unhandled exception. %s'
                              % tb_full)
        stop_rt_server(hpimon.serverp)
        sys.__excepthook__(type, value, tback)
        app.quit()

    sys.excepthook = my_excepthook
    hpimon.show()
    app.exec_()

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before>"""Test the blob filter."""
from pathlib import Path
from typing import Sequence, Tuple
from unittest.mock import MagicMock

import pytest

from git.index.typ import BlobFilter, StageType
from git.objects import Blob
from git.types import PathLike

# fmt: off
@pytest.mark.parametrize('paths, stage_type, path, expected_result', [
    ((Path("foo"),), 0, Path("foo"), True),
    ((Path("foo"),), 0, Path("foo/bar"), True),
    ((Path("foo/bar"),), 0, Path("foo"), False),
    ((Path("foo"), Path("bar")), 0, Path("foo"), True),
])
# fmt: on
def test_blob_filter(paths: Sequence[PathLike], stage_type: StageType, path: PathLike, expected_result: bool) -> None:
    """Test the blob filter."""
    blob_filter = BlobFilter(paths)

    binsha = MagicMock(__len__=lambda self: 20)
    blob: Blob = Blob(repo=MagicMock(), binsha=binsha, path=path)
    stage_blob: Tuple[StageType, Blob] = (stage_type, blob)

    result = blob_filter(stage_blob)

    assert result == expected_result
<commit_msg>Remove stage type as parameter from blob filter test<commit_after>"""Test the blob filter."""
from pathlib import Path
from typing import Sequence, Tuple
from unittest.mock import MagicMock

import pytest

from git.index.typ import BlobFilter, StageType
from git.objects import Blob
from git.types import PathLike

# fmt: off
@pytest.mark.parametrize('paths, path, expected_result', [
    ((Path("foo"),), Path("foo"), True),
    ((Path("foo"),), Path("foo/bar"), True),
    ((Path("foo/bar"),), Path("foo"), False),
    ((Path("foo"), Path("bar")), Path("foo"), True),
])
# fmt: on
def test_blob_filter(paths: Sequence[PathLike], path: PathLike, expected_result: bool) -> None:
    """Test the blob filter."""
    blob_filter = BlobFilter(paths)

    binsha = MagicMock(__len__=lambda self: 20)
    stage_type: StageType = 0
    blob: Blob = Blob(repo=MagicMock(), binsha=binsha, path=path)
    stage_blob: Tuple[StageType, Blob] = (stage_type, blob)

    result = blob_filter(stage_blob)

    assert result == expected_result
<|endoftext|>


<commit_before>from model.contact import Contact

def test_delete_first_contact(app):
    if app.contact.count() == 0:
        app.contact.create(Contact(firstname="Jan", middlename="", lastname="Kowalski", nickname="janek",
                               title="Test title", company="Test company", address="test address",
                               home="+12 111 222 333", mobile="111222333", work="777888999", fax="",
                               email="jan.kowalski@test.pl", email2="jan.kowalski2@test.pl", byear="1985",
                               address2="test secondary address", phone2="", notes="test notes"))
    app.contact.delete_first_contact()

<commit_msg>My homework Lesson 4. Exercise no.11 Verification in the form of comparing lists<commit_after>from model.contact import Contact

def test_delete_first_contact(app):
    if app.contact.count() == 0:
        app.contact.create(Contact(firstname="Jan", middlename="", lastname="Kowalski", nickname="janek",
                               title="Test title", company="Test company", address="test address",
                               home="+12 111 222 333", mobile="111222333", work="777888999", fax="",
                               email="jan.kowalski@test.pl", email2="jan.kowalski2@test.pl", byear="1985",
                               address2="test secondary address", phone2="", notes="test notes"))
    old_contacts = app.contact.get_contact_list()
    app.contact.delete_first_contact()
    new_contacts = app.contact.get_contact_list()
    assert len(old_contacts) - 1 == len(new_contacts)
    old_contacts[0:1] = []
    assert old_contacts == new_contacts

<|endoftext|>


<commit_before>""" Tests for the HistoryNode module """

import unittest

from src import historynode
from test import helper

class TestHistoryNode(unittest.TestCase):
    """ Tests for the historynode module, containing the HistoryNode class """

    def test_print_board(self):
        """Check that print_board works"""
        with helper.captured_output() as out:
            hn_obj = historynode.HistoryNode()
            hn_obj.print_board()
            actual_print = out.getvalue().strip()
            expected_print = ("Player 1: None\n"
                              "Player 2: None\n"
                              "Result: None\n"
                              "Game Type: None\n"
                              "Fox Search: None\n"
                              "Goose Search: None\n"
                              "Half Move: None\n"
                              "      -1 -1 -1      \n"
                              "      -1 -1 -1      \n"
                              "-1 -1 -1 -1 -1 -1 -1\n"
                              "-1 -1 -1 -1 -1 -1 -1\n"
                              "-1 -1 -1 -1 -1 -1 -1\n"
                              "      -1 -1 -1      \n"
                              "      -1 -1 -1")
            self.assertEqual(actual_print, expected_print)

    def test_constructor(self):
        """ Check that HistoryNode object is initialized correctly """
        hn_obj = historynode.HistoryNode()
        hn_obj.constructor()
        self.assertEqual(hn_obj.leafP, False)
        self.assertEqual(hn_obj.rootP, True)
        self.assertEqual(hn_obj.result, 0)
        self.assertEqual(hn_obj.gameType, 1)
        self.assertEqual(hn_obj.foxSearch, 1)
        self.assertEqual(hn_obj.gooseSearch, 1)
        self.assertEqual(hn_obj.halfMove, 1)
        self.assertEqual(hn_obj.gameState[1][1], -1)
        self.assertEqual(hn_obj.gameState[6][0], -1)
<commit_msg>Change default array values from updated GameNode class<commit_after>""" Tests for the HistoryNode module """

import unittest

from src import historynode
from test import helper

class TestHistoryNode(unittest.TestCase):
    """ Tests for the historynode module, containing the HistoryNode class """

    def test_print_board(self):
        """Check that print_board works"""
        with helper.captured_output() as out:
            hn_obj = historynode.HistoryNode()
            hn_obj.print_board()
            actual_print = out.getvalue().strip()
            expected_print = ("Player 1: None\n"
                              "Player 2: None\n"
                              "Result: None\n"
                              "Game Type: None\n"
                              "Fox Search: None\n"
                              "Goose Search: None\n"
                              "Half Move: None\n"
                              "      0 0 0      \n"
                              "      0 0 0      \n"
                              "0 0 0 0 0 0 0\n"
                              "0 0 0 0 0 0 0\n"
                              "0 0 0 0 0 0 0\n"
                              "      0 0 0      \n"
                              "      0 0 0")
            self.assertEqual(actual_print, expected_print)

    def test_constructor(self):
        """ Check that HistoryNode object is initialized correctly """
        hn_obj = historynode.HistoryNode()
        hn_obj.constructor()
        self.assertEqual(hn_obj.leafP, False)
        self.assertEqual(hn_obj.rootP, True)
        self.assertEqual(hn_obj.result, 0)
        self.assertEqual(hn_obj.gameType, 1)
        self.assertEqual(hn_obj.foxSearch, 1)
        self.assertEqual(hn_obj.gooseSearch, 1)
        self.assertEqual(hn_obj.halfMove, 1)
        self.assertEqual(hn_obj.gameState[1][1], -1)
        self.assertEqual(hn_obj.gameState[6][0], -1)
<|endoftext|>


<commit_before>#!/usr/bin/python3
# NOTE: May require the system Python 3 rather than using 3.9
import os.path
import sys
import json
import speech_recognition as sr

"""
TODO:
* Have an ongoing process that holds the microphone device open
* Signal it. Worst case, listen on a Unix socket, and have something open it.
* Record notes, then go back to the loop. This will prevent multiple concurrent notesings though.
* May need a timeout, eg max 30 seconds of notes (to prevent it getting stuck)
* Alternatively, thread them all off. Can they share the microphone??
"""

# Get rid of the ALSA warnings by preloading it with stderr muted
def silence_pyaudio():
	devnull = os.open(os.devnull, os.O_WRONLY)
	old_stderr = os.dup(2)
	sys.stderr.flush()
	os.dup2(devnull, 2)
	os.close(devnull)
	try:
		import pyaudio; pyaudio.PyAudio()
	finally:
		os.dup2(old_stderr, 2)
		os.close(old_stderr)

recog = None
def take_notes(*, desc, new_match=False, **extra):
	NOTES_DIR = os.path.expanduser(os.environ.get("NOTES_DIR", "~/tmp/notes"))
	os.makedirs(NOTES_DIR, exist_ok=True)

	def safe_int(n):
		"""Sort key for probably-numeric strings

		Sorts unparseable strings first in lexicographical order, then
		everything that intifies in numerical order.
		"""
		try: return (1, int(n))
		except (ValueError, TypeError): return (0, n)

	blocks = sorted(os.listdir(NOTES_DIR), key=safe_int)

	try: int(blocks[-1])
	except (IndexError, ValueError): new_match = 2
	if new_match:
		next = int(blocks[-1] if new_match != 2 else 0) + 1
		blocks.append(str(next))
		os.mkdir(NOTES_DIR + "/" + blocks[-1])
	block = NOTES_DIR + "/" + blocks[-1] # Use the latest block (which we may have just created)

	notes = sorted(fn for fn in os.listdir(block) if fn[0] in "0123456789")
	note_id = int(notes[-1].split("-")[0]) + 1 if notes else 1

	global recog
	if recog is None:
		silence_pyaudio()
		recog = sr.Recognizer()

	# Can I increase the gain at all?
	with sr.Microphone() as source:
		print("Listening for notes...")
		audio = recog.listen(source)#, snowboy_configuration=(
		#	"/home/rosuav/voice-tinkering/snowboy/examples/Python3",
		#	["/home/rosuav/voice-tinkering/snowboy/resources/models/snowboy.umdl"]
		#))

	# Discard crazily-long entries. They seem to happen if the recognizer doesn't
	# get a proper silence to start with or something, and it just records forever.
	if len(audio.frame_data) / audio.sample_width / audio.sample_rate > 60.0:
		# More than sixty seconds? Throw it away.
		sys.exit(0)

	print("Got notes.")
	log = open(NOTES_DIR + "/notes.log", "a")
	if new_match:
		print("-" * 65, file=log)
		print("http://localhost:27013/static/notes.html#" + blocks[-1], file=log)
		import webbrowser; webbrowser.open("http://localhost:27013/static/notes.html#" + blocks[-1])

	fn = "%02d - " % note_id + desc
	print("[%s]" % block, fn, file=log, flush=True)
	with open(block + "/%s.flac" % fn, "wb") as f: f.write(audio.get_flac_data())

	d = None
	try: d = recog.recognize_sphinx(audio, show_all=True)
	except sr.UnknownValueError: pass
	except sr.RequestError as e: print("Sphinx:", e, file=log, flush=True)

	options = [b.hypstr for b in d.nbest()] if d else []
	seen = {}
	for txt in options[:5]:
		print("Sphinx: %s" % txt, file=log, flush=True)
		seen[txt] = 1

	# Maybe TODO: Set an API key with key="...."
	try: google = recog.recognize_google(audio)
	except sr.UnknownValueError: google = ""
	except sr.RequestError as e: google = repr(e)

	print("Google:", google, file=log, flush=True)

	try:
		with open(block + "/metadata.json") as f: meta = json.load(f)
	except (FileNotFoundError, json.decoder.JSONDecodeError): meta = {}
	if "recordings" not in meta: meta["recordings"] = []
	meta["recordings"].append({
		"id": note_id,
		"desc": desc,
		"filename": "/%s.flac" % fn,
		"sphinx": options[:5],
		"google": google,
	})
	for key in "round", "spec", "score", "time", "bombtime":
		if key in extra: meta["recordings"][-1][key] = extra[key]
	with open(block + "/metadata.json", "w") as f:
		json.dump(meta, f, sort_keys=True, indent=2)

	if "--gsi" in sys.argv:
		# Signal the GSI server to load new metadata, if appropriate
		requests.post("http://localhost:27013/metadata/" + blocks[-1], json=meta)

if "--gsi" in sys.argv:
	# Call on the GSI server to find out if we're in a CS:GO match, and
	# if so, what we should use as our description
	import requests
	gsi_data = requests.get("http://localhost:27013/status.json").json()
	if gsi_data["playing"]:
		# When running under GSI control (ie NOT explicitly called
		# upon by the terminal), take notes only when in a match.
		take_notes(**gsi_data)
else:
	desc = " ".join(arg for arg in sys.argv[1:] if not arg.startswith("-"))
	take_notes(desc=desc, new_match="--new-block" in sys.argv)
<commit_msg>notes: Abandon snowboy in favour of direct triggering<commit_after>#!/usr/bin/python3
# NOTE: May require the system Python 3 rather than using 3.9
import os.path
import sys
import json
import speech_recognition as sr

"""
TODO:
* Have an ongoing process that holds the microphone device open
* Signal it. Worst case, listen on a Unix socket, and have something open it.
* Record notes, then go back to the loop. This will prevent multiple concurrent notesings though.
* May need a timeout, eg max 30 seconds of notes (to prevent it getting stuck)
* Alternatively, thread them all off. Can they share the microphone??
"""

# Get rid of the ALSA warnings by preloading it with stderr muted
def silence_pyaudio():
	devnull = os.open(os.devnull, os.O_WRONLY)
	old_stderr = os.dup(2)
	sys.stderr.flush()
	os.dup2(devnull, 2)
	os.close(devnull)
	try:
		import pyaudio; pyaudio.PyAudio()
	finally:
		os.dup2(old_stderr, 2)
		os.close(old_stderr)

recog = None
def take_notes(*, desc, new_match=False, **extra):
	NOTES_DIR = os.path.expanduser(os.environ.get("NOTES_DIR", "~/tmp/notes"))
	os.makedirs(NOTES_DIR, exist_ok=True)

	def safe_int(n):
		"""Sort key for probably-numeric strings

		Sorts unparseable strings first in lexicographical order, then
		everything that intifies in numerical order.
		"""
		try: return (1, int(n))
		except (ValueError, TypeError): return (0, n)

	blocks = sorted(os.listdir(NOTES_DIR), key=safe_int)

	try: int(blocks[-1])
	except (IndexError, ValueError): new_match = 2
	if new_match:
		next = int(blocks[-1] if new_match != 2 else 0) + 1
		blocks.append(str(next))
		os.mkdir(NOTES_DIR + "/" + blocks[-1])
	block = NOTES_DIR + "/" + blocks[-1] # Use the latest block (which we may have just created)

	notes = sorted(fn for fn in os.listdir(block) if fn[0] in "0123456789")
	note_id = int(notes[-1].split("-")[0]) + 1 if notes else 1

	global recog
	if recog is None:
		silence_pyaudio()
		recog = sr.Recognizer()

	# Can I increase the gain at all?
	with sr.Microphone() as source:
		print("Listening for notes...")
		audio = recog.listen(source)

	# Discard crazily-long entries. They seem to happen if the recognizer doesn't
	# get a proper silence to start with or something, and it just records forever.
	if len(audio.frame_data) / audio.sample_width / audio.sample_rate > 60.0:
		# More than sixty seconds? Throw it away.
		sys.exit(0)

	print("Got notes.")
	log = open(NOTES_DIR + "/notes.log", "a")
	if new_match:
		print("-" * 65, file=log)
		print("http://localhost:27013/static/notes.html#" + blocks[-1], file=log)
		import webbrowser; webbrowser.open("http://localhost:27013/static/notes.html#" + blocks[-1])

	fn = "%02d - " % note_id + desc
	print("[%s]" % block, fn, file=log, flush=True)
	with open(block + "/%s.flac" % fn, "wb") as f: f.write(audio.get_flac_data())

	d = None
	try: d = recog.recognize_sphinx(audio, show_all=True)
	except sr.UnknownValueError: pass
	except sr.RequestError as e: print("Sphinx:", e, file=log, flush=True)

	options = [b.hypstr for b in d.nbest()] if d else []
	seen = {}
	for txt in options[:5]:
		print("Sphinx: %s" % txt, file=log, flush=True)
		seen[txt] = 1

	# Maybe TODO: Set an API key with key="...."
	try: google = recog.recognize_google(audio)
	except sr.UnknownValueError: google = ""
	except sr.RequestError as e: google = repr(e)

	print("Google:", google, file=log, flush=True)

	try:
		with open(block + "/metadata.json") as f: meta = json.load(f)
	except (FileNotFoundError, json.decoder.JSONDecodeError): meta = {}
	if "recordings" not in meta: meta["recordings"] = []
	meta["recordings"].append({
		"id": note_id,
		"desc": desc,
		"filename": "/%s.flac" % fn,
		"sphinx": options[:5],
		"google": google,
	})
	for key in "round", "spec", "score", "time", "bombtime":
		if key in extra: meta["recordings"][-1][key] = extra[key]
	with open(block + "/metadata.json", "w") as f:
		json.dump(meta, f, sort_keys=True, indent=2)

	if "--gsi" in sys.argv:
		# Signal the GSI server to load new metadata, if appropriate
		requests.post("http://localhost:27013/metadata/" + blocks[-1], json=meta)

if "--gsi" in sys.argv:
	# Call on the GSI server to find out if we're in a CS:GO match, and
	# if so, what we should use as our description
	import requests
	gsi_data = requests.get("http://localhost:27013/status.json").json()
	if gsi_data["playing"]:
		# When running under GSI control (ie NOT explicitly called
		# upon by the terminal), take notes only when in a match.
		take_notes(**gsi_data)
else:
	desc = " ".join(arg for arg in sys.argv[1:] if not arg.startswith("-"))
	take_notes(desc=desc, new_match="--new-block" in sys.argv)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright 2016,  Digital Reasoning
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from __future__ import print_function, unicode_literals

import logging
import getpass
import os

import six
import yaml
from django.utils.crypto import get_random_string
from jinja2 import Template

logger = logging.getLogger(__name__)

# Get the stackdio base dir.  Mostly only useful when not installing with pip.
BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

SECRET_CHARS = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)'

class StackdioConfigException(Exception):
    pass

class StackdioConfig(dict):
    CONFIG_LOCATIONS = (
        os.environ.get('STACKDIO_CONFIG_FILE', ''),
        '/etc/stackdio/server.yaml',
        os.path.expanduser('~/.stackdio/server.yaml'),
        'config/stackdio.yaml',
    )

    REQUIRED_FIELDS = {
        'cloud_providers': list,
        'user': six.string_types,
        'database_url': six.string_types,
        'storage_dir': six.string_types,
        'log_dir': six.string_types,
        'django_secret_key': six.string_types,
        'create_ssh_users': bool,
        'salt_bootstrap_args': six.binary_type,
    }

    DEFAULT_CONTEXT = {
        'user': getpass.getuser(),
        'random_secret_key': get_random_string(50, SECRET_CHARS),
    }

    def __init__(self):
        super(StackdioConfig, self).__init__()
        self.cfg_file = None
        self._load_stackdio_config()

    def _load_stackdio_config(self):
        for cfg_file in self.CONFIG_LOCATIONS:
            cfg_file = os.path.join(BASE_DIR, cfg_file)
            if os.path.isfile(cfg_file):
                self.cfg_file = cfg_file
                break

        if self.cfg_file is None:
            raise StackdioConfigException(
                'Missing stackdio configuration file. '
                'To create the file, you may use `stackdio init`'
            )

        logger.info('Loading configuration from {}'.format(self.cfg_file))
        with open(self.cfg_file) as f:
            template = Template(f.read())
            stackdio_config = yaml.safe_load(template.render(**self.DEFAULT_CONTEXT))

        if not stackdio_config:
            raise StackdioConfigException(
                'stackdio configuration file appears to be empty or not valid yaml.'
            )

        errors = []
        for k, t in self.REQUIRED_FIELDS.items():
            if k not in stackdio_config:
                errors.append('Missing parameter `{0}`'.format(k))
            if not isinstance(stackdio_config[k], t):
                errors.append('Config parameter `{0}` must be of type `{1}`, '
                              'got `{2}` instead'.format(k, t, type(stackdio_config[k])))

        if errors:
            msg = 'stackdio configuration errors:\n'
            for err in errors:
                msg += '  - {0}\n'.format(err)
            raise StackdioConfigException(msg)

        self.update(stackdio_config)

        self.storage_dir = os.path.join(BASE_DIR, self.storage_dir)

        # Try to create the storage dir if it's not there
        if not os.path.isdir(self.storage_dir):
            os.makedirs(self.storage_dir)

        # Try to create the log dir if it's not there
        if not os.path.isdir(self.log_dir):
            os.makedirs(self.log_dir)

        # additional helper attributes
        self.salt_root = os.path.join(self.storage_dir, 'salt')
        self.salt_config_root = os.path.join(self.salt_root, 'config')
        self.salt_master_config = os.path.join(self.salt_config_root, 'master')
        self.salt_cloud_config = os.path.join(self.salt_config_root, 'cloud')
        self.salt_core_states = os.path.join(self.salt_root, 'core_states')
        self.salt_providers_dir = os.path.join(self.salt_config_root, 'cloud.providers.d')
        self.salt_profiles_dir = os.path.join(self.salt_config_root, 'cloud.profiles.d')

        if '{salt_version}' not in self.salt_bootstrap_args:
            raise StackdioConfigException('salt_bootstrap_args must contain `{salt_version}`')

        # defaults
        if not self.salt_master_log_level:  # pylint: disable=access-member-before-definition
            self.salt_master_log_level = 'info'

    def __getattr__(self, k):
        return self.get(k)

    def __setattr__(self, k, v):
        self[k] = v
<commit_msg>Accidentally used binary_type instead of string_type...<commit_after># -*- coding: utf-8 -*-

# Copyright 2016,  Digital Reasoning
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from __future__ import print_function, unicode_literals

import logging
import getpass
import os

import six
import yaml
from django.utils.crypto import get_random_string
from jinja2 import Template

logger = logging.getLogger(__name__)

# Get the stackdio base dir.  Mostly only useful when not installing with pip.
BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

SECRET_CHARS = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)'

class StackdioConfigException(Exception):
    pass

class StackdioConfig(dict):
    CONFIG_LOCATIONS = (
        os.environ.get('STACKDIO_CONFIG_FILE', ''),
        '/etc/stackdio/server.yaml',
        os.path.expanduser('~/.stackdio/server.yaml'),
        'config/stackdio.yaml',
    )

    REQUIRED_FIELDS = {
        'cloud_providers': list,
        'user': six.string_types,
        'database_url': six.string_types,
        'storage_dir': six.string_types,
        'log_dir': six.string_types,
        'django_secret_key': six.string_types,
        'create_ssh_users': bool,
        'salt_bootstrap_args': six.string_types,
    }

    DEFAULT_CONTEXT = {
        'user': getpass.getuser(),
        'random_secret_key': get_random_string(50, SECRET_CHARS),
    }

    def __init__(self):
        super(StackdioConfig, self).__init__()
        self.cfg_file = None
        self._load_stackdio_config()

    def _load_stackdio_config(self):
        for cfg_file in self.CONFIG_LOCATIONS:
            cfg_file = os.path.join(BASE_DIR, cfg_file)
            if os.path.isfile(cfg_file):
                self.cfg_file = cfg_file
                break

        if self.cfg_file is None:
            raise StackdioConfigException(
                'Missing stackdio configuration file. '
                'To create the file, you may use `stackdio init`'
            )

        logger.info('Loading configuration from {}'.format(self.cfg_file))
        with open(self.cfg_file) as f:
            template = Template(f.read())
            stackdio_config = yaml.safe_load(template.render(**self.DEFAULT_CONTEXT))

        if not stackdio_config:
            raise StackdioConfigException(
                'stackdio configuration file appears to be empty or not valid yaml.'
            )

        errors = []
        for k, t in self.REQUIRED_FIELDS.items():
            if k not in stackdio_config:
                errors.append('Missing parameter `{0}`'.format(k))
            if not isinstance(stackdio_config[k], t):
                errors.append('Config parameter `{0}` must be of type `{1}`, '
                              'got `{2}` instead'.format(k, t, type(stackdio_config[k])))

        if errors:
            msg = 'stackdio configuration errors:\n'
            for err in errors:
                msg += '  - {0}\n'.format(err)
            raise StackdioConfigException(msg)

        self.update(stackdio_config)

        self.storage_dir = os.path.join(BASE_DIR, self.storage_dir)

        # Try to create the storage dir if it's not there
        if not os.path.isdir(self.storage_dir):
            os.makedirs(self.storage_dir)

        # Try to create the log dir if it's not there
        if not os.path.isdir(self.log_dir):
            os.makedirs(self.log_dir)

        # additional helper attributes
        self.salt_root = os.path.join(self.storage_dir, 'salt')
        self.salt_config_root = os.path.join(self.salt_root, 'config')
        self.salt_master_config = os.path.join(self.salt_config_root, 'master')
        self.salt_cloud_config = os.path.join(self.salt_config_root, 'cloud')
        self.salt_core_states = os.path.join(self.salt_root, 'core_states')
        self.salt_providers_dir = os.path.join(self.salt_config_root, 'cloud.providers.d')
        self.salt_profiles_dir = os.path.join(self.salt_config_root, 'cloud.profiles.d')

        if '{salt_version}' not in self.salt_bootstrap_args:
            raise StackdioConfigException('salt_bootstrap_args must contain `{salt_version}`')

        # defaults
        if not self.salt_master_log_level:  # pylint: disable=access-member-before-definition
            self.salt_master_log_level = 'info'

    def __getattr__(self, k):
        return self.get(k)

    def __setattr__(self, k, v):
        self[k] = v
<|endoftext|>


<commit_before>from datetime import datetime
from pytz import timezone
from troposphere import Ref, Template, Parameter, GetAZs, Output, Join, GetAtt
from troposphere.autoscaling import LaunchConfiguration, AutoScalingGroup, Tag
from troposphere.ec2 import Instance, SecurityGroup, SecurityGroupRule
from troposphere.elasticloadbalancing import LoadBalancer, AccessLoggingPolicy
from troposphere.rds import DBInstance, DBParameterGroup, DBSecurityGroup, DBSecurityGroupIngress, RDSSecurityGroup
from troposphere.s3 import Bucket, PublicRead
from troposphere.route53 import RecordSetType

t = Template()
t.add_description("Create FireCARES Webserver Load Balancer and Auto-Scaling group")

base_ami = "ami-7646e460"

now = datetime.utcnow().replace(tzinfo=timezone('UTC')).isoformat()

key_name = t.add_parameter(Parameter(
    "KeyName",
    Description="Name of an existing EC2 KeyPair to enable SSH access to the instances",
    Type="AWS::EC2::KeyPair::KeyName",
    ConstraintDescription="Must be the name of an existing EC2 KeyPair."
))

ami = t.add_parameter(Parameter(
    "baseAmi",
    Description="Name of the AMI to use",
    Type="String",
    ConstraintDescription="Must be the name of an existing AMI.",
    Default=base_ami
))

web_capacity = t.add_parameter(Parameter(
    "WebServerCapacity",
    Default="2",
    Description="The initial number of WebServer instances",
    Type="Number",
    ConstraintDescription="must be between 1 and 5 EC2 instances.",
    MinValue="1",
    MaxValue="5",
))

commit = t.add_parameter(Parameter(
    "CommitHash",
    Description="Commit hash used for building the web VM",
    Type="String"
))

web_instance_class = t.add_parameter(Parameter(
    "WebInstanceClass",
    Default="t2.medium",
    Description="WebServer EC2 instance type",
    Type="String",
    ConstraintDescription="must be a valid EC2 instance type.",
    AllowedValues=[
        "t1.micro",
        "t2.nano",
        "t2.micro",
        "t2.small",
        "t2.medium",
        "t2.large",
        "m1.small",
        "m1.medium",
        "m1.large",
        "m1.xlarge",
        "m2.xlarge",
        "m2.2xlarge",
        "m2.4xlarge",
        "m3.medium",
        "m3.large",
        "m3.xlarge",
        "m3.2xlarge",
        "m4.large",
        "m4.xlarge",
        "m4.2xlarge",
        "m4.4xlarge",
        "m4.10xlarge",
        "c1.medium",
        "c1.xlarge",
        "c3.large",
        "c3.xlarge",
        "c3.2xlarge",
        "c3.4xlarge",
        "c3.8xlarge",
        "c4.large",
        "c4.xlarge",
        "c4.2xlarge",
        "c4.4xlarge",
        "c4.8xlarge",
        "g2.2xlarge",
        "g2.8xlarge",
        "r3.large",
        "r3.xlarge",
        "r3.2xlarge",
        "r3.4xlarge",
        "r3.8xlarge",
        "i2.xlarge",
        "i2.2xlarge",
        "i2.4xlarge",
        "i2.8xlarge",
        "d2.xlarge",
        "d2.2xlarge",
        "d2.4xlarge",
        "d2.8xlarge",
        "hi1.4xlarge",
        "hs1.8xlarge",
        "cr1.8xlarge",
        "cc2.8xlarge",
        "cg1.4xlarge"
      ]
))

environment = t.add_parameter(Parameter(
    "Environment",
    Description="Stack environment (e.g. prod, dev, int)",
    Type="String",
    MinLength="1",
    MaxLength="12",
    Default="dev",
))

load_balancer = t.add_resource(LoadBalancer(
    "LoadBalancer",
    CrossZone=True,
    AvailabilityZones=GetAZs(""),
    LoadBalancerName=Join('-', ['fc', Ref(environment), Ref(commit)]),
    AppCookieStickinessPolicy=[
      {
        "PolicyName": "AppCookieBasedPolicy",
        "CookieName": "sticky"
      }
    ],
    Listeners=[
     {
        "LoadBalancerPort": "80",
        "InstancePort": "80",
        "Protocol": "HTTP"
      },
     {
        "LoadBalancerPort": "443",
        "InstancePort": "80",
        "Protocol": "HTTPS",
        "SSLCertificateId": "arn:aws:iam::164077527722:server-certificate/firecares",
        "PolicyNames": [
            "AppCookieBasedPolicy"
        ]
      }
    ]
))

web_sg = t.add_resource(SecurityGroup(
    "WebServers",
    GroupDescription=Join(' - ', ["FireCARES webserver group", Ref(environment), Ref(commit)]),
    SecurityGroupIngress=[
        SecurityGroupRule("ELBAccess",
                          IpProtocol="tcp",
                          FromPort="80",
                          ToPort="80",
                          SourceSecurityGroupOwnerId=GetAtt(load_balancer, "SourceSecurityGroup.OwnerAlias"),
                          SourceSecurityGroupName=GetAtt(load_balancer, "SourceSecurityGroup.GroupName")
                          ),
        SecurityGroupRule("JenkinsAccess", IpProtocol="tcp", FromPort="22", ToPort="22", CidrIp="54.173.150.226/32"),
        SecurityGroupRule("TylerAccess", IpProtocol="tcp", FromPort="22", ToPort="22", CidrIp="69.255.184.149/32"),
        SecurityGroupRule("JoeAccess", IpProtocol="tcp", FromPort="22", ToPort="22", CidrIp="65.254.97.100/32"),
        SecurityGroupRule("JoeAccess2", IpProtocol="tcp", FromPort="22", ToPort="22", CidrIp="108.66.75.162/32"),
        SecurityGroupRule("JoeAccess3", IpProtocol="tcp", FromPort="22", ToPort="22", CidrIp="71.86.4.190/32"),
        SecurityGroupRule("JoeAccess4", IpProtocol="tcp", FromPort="22", ToPort="22", CidrIp="75.133.14.178/32")
        ],
    ))

launch_configuration = t.add_resource(LaunchConfiguration(
    "WebServerLaunchConfiguration",
    ImageId=Ref(ami),
    InstanceType=Ref(web_instance_class),
    KeyName=Ref(key_name),
    SecurityGroups=[Ref(web_sg)]
    # SecurityGroups=['sg-6774931c', 'sg-3674934d', 'sg-5f749324'],
    # ClassicLinkVPCId='vpc-bb3cd6dc'
))

autoscaling_group = t.add_resource(AutoScalingGroup(
    "WebserverAutoScale",
    AvailabilityZones=['us-east-1b', 'us-east-1c'],
    DesiredCapacity=Ref(web_capacity),
    MinSize="1",
    MaxSize="5",
    Tags=[
        Tag("environment", Ref(environment), True),
        Tag("Name", Join('-', ['web-server', Ref(environment), Ref(commit)]), True),
        Tag("Group", Join('-', ['web-server', Ref(environment)]), True)
    ],
    LoadBalancerNames=[Ref(load_balancer)],
    HealthCheckType="EC2",
    LaunchConfigurationName=Ref(launch_configuration)
))

t.add_output([
    Output(
        "stackURL",
        Description="Stack url",
        Value=Join("", [GetAtt(load_balancer, 'DNSName')]),
    )
])

t.add_output([
    Output(
        "WebServerSecurityGroup",
        Description="Web server security group.",
        Value=Join("", [GetAtt(web_sg, 'GroupId')]),
    )
])

t.add_output([
    Output(
        "AMI",
        Description="Web server ami image group.",
        Value=Ref(ami),
    )
])

if __name__ == '__main__':
    print t.to_json()
<commit_msg>Small vs medium instances<commit_after>from datetime import datetime
from pytz import timezone
from troposphere import Ref, Template, Parameter, GetAZs, Output, Join, GetAtt
from troposphere.autoscaling import LaunchConfiguration, AutoScalingGroup, Tag
from troposphere.ec2 import Instance, SecurityGroup, SecurityGroupRule
from troposphere.elasticloadbalancing import LoadBalancer, AccessLoggingPolicy
from troposphere.rds import DBInstance, DBParameterGroup, DBSecurityGroup, DBSecurityGroupIngress, RDSSecurityGroup
from troposphere.s3 import Bucket, PublicRead
from troposphere.route53 import RecordSetType

t = Template()
t.add_description("Create FireCARES Webserver Load Balancer and Auto-Scaling group")

base_ami = "ami-7646e460"

now = datetime.utcnow().replace(tzinfo=timezone('UTC')).isoformat()

key_name = t.add_parameter(Parameter(
    "KeyName",
    Description="Name of an existing EC2 KeyPair to enable SSH access to the instances",
    Type="AWS::EC2::KeyPair::KeyName",
    ConstraintDescription="Must be the name of an existing EC2 KeyPair."
))

ami = t.add_parameter(Parameter(
    "baseAmi",
    Description="Name of the AMI to use",
    Type="String",
    ConstraintDescription="Must be the name of an existing AMI.",
    Default=base_ami
))

web_capacity = t.add_parameter(Parameter(
    "WebServerCapacity",
    Default="2",
    Description="The initial number of WebServer instances",
    Type="Number",
    ConstraintDescription="must be between 1 and 5 EC2 instances.",
    MinValue="1",
    MaxValue="5",
))

commit = t.add_parameter(Parameter(
    "CommitHash",
    Description="Commit hash used for building the web VM",
    Type="String"
))

web_instance_class = t.add_parameter(Parameter(
    "WebInstanceClass",
    Default="t2.small",
    Description="WebServer EC2 instance type",
    Type="String",
    ConstraintDescription="must be a valid EC2 instance type.",
    AllowedValues=[
        "t1.micro",
        "t2.nano",
        "t2.micro",
        "t2.small",
        "t2.medium",
        "t2.large",
        "m1.small",
        "m1.medium",
        "m1.large",
        "m1.xlarge",
        "m2.xlarge",
        "m2.2xlarge",
        "m2.4xlarge",
        "m3.medium",
        "m3.large",
        "m3.xlarge",
        "m3.2xlarge",
        "m4.large",
        "m4.xlarge",
        "m4.2xlarge",
        "m4.4xlarge",
        "m4.10xlarge",
        "c1.medium",
        "c1.xlarge",
        "c3.large",
        "c3.xlarge",
        "c3.2xlarge",
        "c3.4xlarge",
        "c3.8xlarge",
        "c4.large",
        "c4.xlarge",
        "c4.2xlarge",
        "c4.4xlarge",
        "c4.8xlarge",
        "g2.2xlarge",
        "g2.8xlarge",
        "r3.large",
        "r3.xlarge",
        "r3.2xlarge",
        "r3.4xlarge",
        "r3.8xlarge",
        "i2.xlarge",
        "i2.2xlarge",
        "i2.4xlarge",
        "i2.8xlarge",
        "d2.xlarge",
        "d2.2xlarge",
        "d2.4xlarge",
        "d2.8xlarge",
        "hi1.4xlarge",
        "hs1.8xlarge",
        "cr1.8xlarge",
        "cc2.8xlarge",
        "cg1.4xlarge"
      ]
))

environment = t.add_parameter(Parameter(
    "Environment",
    Description="Stack environment (e.g. prod, dev, int)",
    Type="String",
    MinLength="1",
    MaxLength="12",
    Default="dev",
))

load_balancer = t.add_resource(LoadBalancer(
    "LoadBalancer",
    CrossZone=True,
    AvailabilityZones=GetAZs(""),
    LoadBalancerName=Join('-', ['fc', Ref(environment), Ref(commit)]),
    AppCookieStickinessPolicy=[
      {
        "PolicyName": "AppCookieBasedPolicy",
        "CookieName": "sticky"
      }
    ],
    Listeners=[
     {
        "LoadBalancerPort": "80",
        "InstancePort": "80",
        "Protocol": "HTTP"
      },
     {
        "LoadBalancerPort": "443",
        "InstancePort": "80",
        "Protocol": "HTTPS",
        "SSLCertificateId": "arn:aws:iam::164077527722:server-certificate/firecares",
        "PolicyNames": [
            "AppCookieBasedPolicy"
        ]
      }
    ]
))

web_sg = t.add_resource(SecurityGroup(
    "WebServers",
    GroupDescription=Join(' - ', ["FireCARES webserver group", Ref(environment), Ref(commit)]),
    SecurityGroupIngress=[
        SecurityGroupRule("ELBAccess",
                          IpProtocol="tcp",
                          FromPort="80",
                          ToPort="80",
                          SourceSecurityGroupOwnerId=GetAtt(load_balancer, "SourceSecurityGroup.OwnerAlias"),
                          SourceSecurityGroupName=GetAtt(load_balancer, "SourceSecurityGroup.GroupName")
                          ),
        SecurityGroupRule("JenkinsAccess", IpProtocol="tcp", FromPort="22", ToPort="22", CidrIp="54.173.150.226/32"),
        SecurityGroupRule("TylerAccess", IpProtocol="tcp", FromPort="22", ToPort="22", CidrIp="69.255.184.149/32"),
        SecurityGroupRule("JoeAccess", IpProtocol="tcp", FromPort="22", ToPort="22", CidrIp="65.254.97.100/32"),
        SecurityGroupRule("JoeAccess2", IpProtocol="tcp", FromPort="22", ToPort="22", CidrIp="108.66.75.162/32"),
        SecurityGroupRule("JoeAccess3", IpProtocol="tcp", FromPort="22", ToPort="22", CidrIp="71.86.4.190/32"),
        SecurityGroupRule("JoeAccess4", IpProtocol="tcp", FromPort="22", ToPort="22", CidrIp="75.133.14.178/32")
        ],
    ))

launch_configuration = t.add_resource(LaunchConfiguration(
    "WebServerLaunchConfiguration",
    ImageId=Ref(ami),
    InstanceType=Ref(web_instance_class),
    KeyName=Ref(key_name),
    SecurityGroups=[Ref(web_sg)]
    # SecurityGroups=['sg-6774931c', 'sg-3674934d', 'sg-5f749324'],
    # ClassicLinkVPCId='vpc-bb3cd6dc'
))

autoscaling_group = t.add_resource(AutoScalingGroup(
    "WebserverAutoScale",
    AvailabilityZones=['us-east-1b', 'us-east-1c'],
    DesiredCapacity=Ref(web_capacity),
    MinSize="1",
    MaxSize="5",
    Tags=[
        Tag("environment", Ref(environment), True),
        Tag("Name", Join('-', ['web-server', Ref(environment), Ref(commit)]), True),
        Tag("Group", Join('-', ['web-server', Ref(environment)]), True)
    ],
    LoadBalancerNames=[Ref(load_balancer)],
    HealthCheckType="EC2",
    LaunchConfigurationName=Ref(launch_configuration)
))

t.add_output([
    Output(
        "stackURL",
        Description="Stack url",
        Value=Join("", [GetAtt(load_balancer, 'DNSName')]),
    )
])

t.add_output([
    Output(
        "WebServerSecurityGroup",
        Description="Web server security group.",
        Value=Join("", [GetAtt(web_sg, 'GroupId')]),
    )
])

t.add_output([
    Output(
        "AMI",
        Description="Web server ami image group.",
        Value=Ref(ami),
    )
])

if __name__ == '__main__':
    print t.to_json()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Neurosynth & Allen Brain Atlas: nsaba
Authors: Simon Haxby, Scott Susi, Torben Noto

Last Updated: 7/26/2015
"""
import numpy as np
import pandas as pd
import os
import itertools
from scipy import spatial

class Nsaba(object):
    aba = {
        'exp_df': None,
        'probe_df': None,
        'si_df': None,
        'mni_coords': None
    }

    ns = {
        'database_df': None,
        'features_df': None,
    }

    @classmethod
    def aba_load(cls, aba_path=".", csv_names=None):
        """Initialization of 'aba' dictionary"""
        if not csv_names:
            csv_names = [
                'MicroarrayExpression.csv',
                'SampleAnnot.csv',
                'Probes.csv']

        if len(csv_names) != 3:
            raise IndexError("'csv_names' must a list of 3 'str' variables")

        print 'This may take a minute or two ...'

        csv_path = os.path.join(aba_path, csv_names[1])
        cls.aba['si_df'] = pd.read_csv(csv_path)
        print '%s loaded.' % csv_names[1]

        csv_path = os.path.join(aba_path, csv_names[0])
        cls.aba['exp_df'] = pd.read_csv(csv_path, header=None)
        cls.aba['exp_df'].columns = list(
            itertools.chain.from_iterable(
                [['probe_id'], range(cls.aba['si_df'].shape[0])]))

        print '%s loaded.' % csv_names[0]

        csv_path = os.path.join(aba_path, csv_names[2])
        cls.aba['probe_df'] = pd.read_csv(csv_path)
        print '%s loaded.' % csv_names[2]

        cls.aba['mni_coords'] = cls.aba['si_df'].loc[:, 'mni_x':'mni_z'].as_matrix().astype(float)
        print "Nsaba.aba['mni_coords'] initialized."

    @classmethod
    def ns_load(cls, ns_path=".", ns_files=None):
        """Initialization of 'ns' dictionary"""
        if not ns_files:
            ns_files = ['database.txt', 'features.txt']

        df = pd.read_table(os.path.join(ns_path, ns_files[0]))
        cls.ns['database_df'] = df.loc[df.space == 'MNI', ['id', 'x', 'y', 'z']]
        print "%s loaded." % ns_files[0]

        cls.ns['features_df'] = pd.read_table(os.path.join(ns_path, ns_files[1]))
        print "%s loaded." % ns_files[1]

    def __init__(self):

        self.ge = {}
        self.term = {}
        self.__ns_weight_f = lambda r: 1. / r ** 2

    def check_entrez_struct(self, entrez_ids):
        """ Checks if 'entrez_ids' parameter is an non-str iterable"""
        try:
            iter(entrez_ids)
        except TypeError:
            print "Invalid parameter form; please contain entrez ids as 'str' types in iterable container"
            return 1
        else:
            if isinstance(entrez_ids, str):
                print "Invalid parameter form; please contain entrez ids as 'str' types in iterable container"
                return 1
            else:
                return 0

    def get_aba_ge(self, entrez_ids):
        """ Retrieves an stores gene expression coefficients in ABA dictionary based on a
        a passed list of Entrez IDs"""
        if self.__check_static_members() == 1:
            return 1
        if self.check_entrez_struct(entrez_ids) == 1:
            return 1

        for entrez_id in entrez_ids:
            probe_ids = self.aba['probe_df'].loc[self.aba['probe_df']['entrez_id']
                                                 == entrez_id]['probe_id'].tolist()

            if len(probe_ids) == 0:
                print 'Entrez ID: %s not registered with ABA database' % entrez_id
                continue

            ge_df = self.aba['exp_df'].loc[self.aba['exp_df']['probe_id'].isin(probe_ids)]
            ge_mat = ge_df.as_matrix().astype(float)[:, 1:].T
            self.ge[entrez_id] = np.mean(ge_mat, axis=1)

    def is_term(self, term):
        """ Checks if this term is in the neurosynth database """
        if term in self.ns['features_df'].columns:
            return True
        else:
            return False

    def is_id(self, ID):
        """ Checks if ID is registered """
        if (self.ns['features_df']['pmid'] == ID).any():
            return True
        else:
            return False

    def coord_to_ids(self, coord):
        """ Uses the study dictionary above to find study ids from x,y,z coordinates """
        # Use Later?

        return

    def id_to_terms(self, ID):
        """ Finds all of the term heat values of a given ID """
        # Use Later?

        return

    def __term_to_coords(self, term, thresh=0):
        """ Finds coordinates associated with a given term.
        Returns NS coordinate tree and ID/coordinate/activation DataFrame"""
        term_ids_act = self.ns['features_df'].loc[self.ns['features_df'][term] > thresh, ['pmid', term]]
        term_ids = term_ids_act['pmid'].tolist()
        term_coords = self.ns['database_df'].loc[self.ns['database_df']['id'].isin(term_ids)]
        try:
            ns_coord_tree = spatial.KDTree(term_coords.loc[:, 'x':'z'].as_matrix().astype(float))
        except ValueError:
            print "No studies with term: '%s' and threshold: %.2f found" % (term, thresh)
            return 1, 1
        else:
            term_ids_act.rename(columns={'pmid': 'id'}, inplace=True)
            return ns_coord_tree, term_coords.merge(term_ids_act)

    def __sphere(self, xyz, ns_coord_tree, max_rad=5):
        """ Returns 3D Array containing coordinates in each layer of the sphere """
        sphere_bucket = []
        set_bucket = []

        # Needs work; generalize
        for i, r in enumerate(range(max_rad, 0, -1)):
            pts = ns_coord_tree.query_ball_point(xyz, r)
            set_bucket.append(set(map(tuple, ns_coord_tree.data[pts])))

        for i in range(0, 3):
            sphere_bucket.append(list(set_bucket[i].difference(set_bucket[i + 1])))
        sphere_bucket.append(list(set_bucket[3]))
        rev_iter = reversed(sphere_bucket)

        return [layer for layer in rev_iter]

    def __knn_search(self, xyz, ns_coord_tree, max_rad=5, k=20):
        """ KNN search of NS coordinates about ABA coordinates """
        r, inds = ns_coord_tree.query(xyz, k)
        return ns_coord_tree.data[inds[r < max_rad]], r[r < max_rad]

    def __get_act_values(self, bucket, weight, term, ns_coord_act_df):
        """ Returns weighted NS activation """
        bucket_act_vec = []
        for coords in bucket:
            coord = ns_coord_act_df[(ns_coord_act_df['x'] == coords[0])
                                    & (ns_coord_act_df['y'] == coords[1])
                                    & (ns_coord_act_df['z'] == coords[2])][term]
            bucket_act_vec.append(coord.mean())

        return np.array(bucket_act_vec)*weight

    def __knn_method(self, term, ns_coord_act_df, ns_coord_tree, search_radii, k):
        """ KNN method """
        for irow, xyz in enumerate(self.aba['mni_coords']):
            coords, radii = self.__knn_search(xyz, ns_coord_tree, search_radii, k)
            weight = self.__ns_weight_f(radii)
            weighted_means = self.__get_act_values(coords, weight, term, ns_coord_act_df)
            if len(weighted_means) == 0:
                self.term[term]['aba_void_indices'].append(irow)
            else:
                act_coeff = np.sum(weighted_means) / np.sum(weight)
                self.term[term]['ns_act_vector'].append(act_coeff)

    def __sphere_method(self, term, ns_coord_act_df, ns_coord_tree, search_radii):
        """ Sphere buckets method"""
        for irow, xyz in enumerate(self.aba['mni_coords']):
            sphere_bucket = self.__sphere(xyz, ns_coord_tree, search_radii)
            sphere_vals = [0, 0]
            for w, bucket in enumerate(sphere_bucket):
                weight = self.__ns_weight_f(w + 1)
                bucket_mean = np.mean(self.__get_act_values(bucket, weight, term, ns_coord_act_df))
                if np.isnan(bucket_mean):
                    sphere_vals[0] += 0
                    sphere_vals[1] += 0
                else:
                    sphere_vals[0] += bucket_mean
                    sphere_vals[1] += weight
            if sphere_vals[1] == 0:
                self.term[term]['aba_void_indices'].append(irow)
            else:
                act_coeff = sphere_vals[0] / sphere_vals[1]
                self.term[term]['ns_act_vector'].append(act_coeff)

    #@profile
    def get_ns_act(self, term, thresh=0, method='knn', search_radii=5, k=None):
        """ Generates NS activation vector about ABA MNI coordinates  """
        if self.__check_static_members() == 1:
            return 1
        if not self.is_term(term):
            print "'%s' is not a registered term." % term
            return 1

        ns_coord_tree, ns_coord_act_df = self.__term_to_coords(term, thresh)
        if type(ns_coord_tree) == int:
            return 1

        self.term[term] = {}
        self.term[term]['ns_act_vector'] = []
        self.term[term]['aba_void_indices'] = []

        if method == 'knn':
            if k is None:
                k = 20
            self.__knn_method(term, ns_coord_act_df, ns_coord_tree, search_radii, k)
        elif method == 'sphere':
            if k is not None:
                raise ValueError("'k' parameter cannot be used with 'sphere' method.")
            self.__sphere_method(term, ns_coord_act_df, ns_coord_tree, search_radii)
        else:
            raise TypeError("'%s' is not a valid parameter value for 'method' parameter, use either 'knn' or 'sphere"
                            % method)

    def make_ge_ns_mat(self, ns_term, entrez_ids):
        if self.__check_static_members() == 1:
            return 1
        if self.check_entrez_struct(entrez_ids) == 1:
            return 1

        if ns_term in self.term and all([key in self.ge for key in entrez_ids]):
            ge_ns_mat = []
            for entrez_id in entrez_ids:
                aba_indices = np.array([i for i in range(len(self.aba['mni_coords']))
                                        if i not in self.term[ns_term]['aba_void_indices']])
                ge_ns_mat.append(self.ge[entrez_id][aba_indices])
            ge_ns_mat.append(self.term[ns_term]['ns_act_vector'])
            return np.vstack(ge_ns_mat).T
        else:
            print "Either term['%s'] or one or more Entrez ID keys does not exist; please check arguments" \
                  % ns_term

    def set_ns_weight_f(self, f):
        try:
            print "Test: f(e) = %.2f" % f(np.e)
            self.__ns_weight_f = f
        except TypeError:
            print "'f' is improper, ensure 'f' receives only one parameter and returns a numeric type"

    def __check_static_members(self):
        for val in self.aba.itervalues():
            if val is None:
                print "Unassigned Nsaba 'aba' static variable: see Nsaba.aba_load(path)"
                return 1
        for val in self.ns.itervalues():
            if val is None:
                print "Unassigned Nsaba 'ns' static variable: see Nsaba.ns_load(path)"
                return 1
        return 0
<commit_msg>coord_tree more generalized<commit_after># -*- coding: utf-8 -*-
"""
Neurosynth & Allen Brain Atlas: nsaba
Authors: Simon Haxby, Scott Susi, Torben Noto

Last Updated: 7/26/2015
"""
import numpy as np
import pandas as pd
import os
import itertools
from scipy import spatial

class Nsaba(object):
    aba = {
        'exp_df': None,
        'probe_df': None,
        'si_df': None,
        'mni_coords': None
    }

    ns = {
        'database_df': None,
        'features_df': None,
    }

    @classmethod
    def aba_load(cls, aba_path=".", csv_names=None):
        """Initialization of 'aba' dictionary"""
        if not csv_names:
            csv_names = [
                'MicroarrayExpression.csv',
                'SampleAnnot.csv',
                'Probes.csv']

        if len(csv_names) != 3:
            raise IndexError("'csv_names' must a list of 3 'str' variables")

        print 'This may take a minute or two ...'

        csv_path = os.path.join(aba_path, csv_names[1])
        cls.aba['si_df'] = pd.read_csv(csv_path)
        print '%s loaded.' % csv_names[1]

        csv_path = os.path.join(aba_path, csv_names[0])
        cls.aba['exp_df'] = pd.read_csv(csv_path, header=None)
        cls.aba['exp_df'].columns = list(
            itertools.chain.from_iterable(
                [['probe_id'], range(cls.aba['si_df'].shape[0])]))

        print '%s loaded.' % csv_names[0]

        csv_path = os.path.join(aba_path, csv_names[2])
        cls.aba['probe_df'] = pd.read_csv(csv_path)
        print '%s loaded.' % csv_names[2]

        cls.aba['mni_coords'] = cls.aba['si_df'].loc[:, 'mni_x':'mni_z'].as_matrix().astype(float)
        print "Nsaba.aba['mni_coords'] initialized."

    @classmethod
    def ns_load(cls, ns_path=".", ns_files=None):
        """Initialization of 'ns' dictionary"""
        if not ns_files:
            ns_files = ['database.txt', 'features.txt']

        df = pd.read_table(os.path.join(ns_path, ns_files[0]))
        cls.ns['database_df'] = df.loc[df.space == 'MNI', ['id', 'x', 'y', 'z']]
        print "%s loaded." % ns_files[0]

        cls.ns['features_df'] = pd.read_table(os.path.join(ns_path, ns_files[1]))
        print "%s loaded." % ns_files[1]

    def __init__(self):

        self.ge = {}
        self.term = {}
        self.__ns_weight_f = lambda r: 1. / r ** 2

    def check_entrez_struct(self, entrez_ids):
        """ Checks if 'entrez_ids' parameter is an non-str iterable"""
        try:
            iter(entrez_ids)
        except TypeError:
            print "Invalid parameter form; please contain entrez ids as 'str' types in iterable container"
            return 1
        else:
            if isinstance(entrez_ids, str):
                print "Invalid parameter form; please contain entrez ids as 'str' types in iterable container"
                return 1
            else:
                return 0

    def get_aba_ge(self, entrez_ids):
        """ Retrieves an stores gene expression coefficients in ABA dictionary based on a
        a passed list of Entrez IDs"""
        if self.__check_static_members() == 1:
            return 1
        if self.check_entrez_struct(entrez_ids) == 1:
            return 1

        for entrez_id in entrez_ids:
            probe_ids = self.aba['probe_df'].loc[self.aba['probe_df']['entrez_id']
                                                 == entrez_id]['probe_id'].tolist()

            if len(probe_ids) == 0:
                print 'Entrez ID: %s not registered with ABA database' % entrez_id
                continue

            ge_df = self.aba['exp_df'].loc[self.aba['exp_df']['probe_id'].isin(probe_ids)]
            ge_mat = ge_df.as_matrix().astype(float)[:, 1:].T
            self.ge[entrez_id] = np.mean(ge_mat, axis=1)

    def is_term(self, term):
        """ Checks if this term is in the neurosynth database """
        if term in self.ns['features_df'].columns:
            return True
        else:
            return False

    def is_id(self, ID):
        """ Checks if ID is registered """
        if (self.ns['features_df']['pmid'] == ID).any():
            return True
        else:
            return False

    def coord_to_ids(self, coord):
        """ Uses the study dictionary above to find study ids from x,y,z coordinates """
        # Use Later?

        return

    def id_to_terms(self, ID):
        """ Finds all of the term heat values of a given ID """
        # Use Later?

        return

    def __term_to_coords(self, term, thresh=0):
        """ Finds coordinates associated with a given term.
        Returns NS coordinate tree and ID/coordinate/activation DataFrame"""
        term_ids_act = self.ns['features_df'].loc[self.ns['features_df'][term] > thresh, ['pmid', term]]
        term_ids = term_ids_act['pmid'].tolist()
        term_coords = self.ns['database_df'].loc[self.ns['database_df']['id'].isin(term_ids)]
        try:
            ns_coord_tree = spatial.KDTree(term_coords.loc[:, 'x':'z'].as_matrix().astype(float))
        except ValueError:
            print "No studies with term: '%s' and threshold: %.2f found" % (term, thresh)
            return 1, 1
        else:
            term_ids_act.rename(columns={'pmid': 'id'}, inplace=True)
            return ns_coord_tree, term_coords.merge(term_ids_act)

    def __sphere(self, xyz, ns_coord_tree, max_rad=5):
        """ Returns 3D Array containing coordinates in each layer of the sphere """
        sphere_bucket = []
        set_bucket = []

        # Needs work; generalize
        for i, r in enumerate(range(max_rad, 0, -1)):
            pts = ns_coord_tree.query_ball_point(xyz, r)
            set_bucket.append(set(map(tuple, ns_coord_tree.data[pts])))

        for i in range(0, 3):
            sphere_bucket.append(list(set_bucket[i].difference(set_bucket[i + 1])))
        sphere_bucket.append(list(set_bucket[3]))
        rev_iter = reversed(sphere_bucket)

        return [layer for layer in rev_iter]

    def __knn_search(self, xyz, coord_tree, max_rad=5, k=20):
        """ KNN search of NS coordinates about ABA coordinates """
        r, inds = coord_tree.query(xyz, k)
        return coord_tree.data[inds[r < max_rad]], r[r < max_rad]

    def __get_act_values(self, bucket, weight, term, ns_coord_act_df):
        """ Returns weighted NS activation """
        bucket_act_vec = []
        for coords in bucket:
            coord = ns_coord_act_df[(ns_coord_act_df['x'] == coords[0])
                                    & (ns_coord_act_df['y'] == coords[1])
                                    & (ns_coord_act_df['z'] == coords[2])][term]
            bucket_act_vec.append(coord.mean())

        return np.array(bucket_act_vec)*weight

    def __knn_method(self, term, ns_coord_act_df, ns_coord_tree, search_radii, k):
        """ KNN method """
        for irow, xyz in enumerate(self.aba['mni_coords']):
            coords, radii = self.__knn_search(xyz, ns_coord_tree, search_radii, k)
            weight = self.__ns_weight_f(radii)
            weighted_means = self.__get_act_values(coords, weight, term, ns_coord_act_df)
            if len(weighted_means) == 0:
                self.term[term]['aba_void_indices'].append(irow)
            else:
                act_coeff = np.sum(weighted_means) / np.sum(weight)
                self.term[term]['ns_act_vector'].append(act_coeff)

    def __sphere_method(self, term, ns_coord_act_df, ns_coord_tree, search_radii):
        """ Sphere buckets method"""
        for irow, xyz in enumerate(self.aba['mni_coords']):
            sphere_bucket = self.__sphere(xyz, ns_coord_tree, search_radii)
            sphere_vals = [0, 0]
            for w, bucket in enumerate(sphere_bucket):
                weight = self.__ns_weight_f(w + 1)
                bucket_mean = np.mean(self.__get_act_values(bucket, weight, term, ns_coord_act_df))
                if np.isnan(bucket_mean):
                    sphere_vals[0] += 0
                    sphere_vals[1] += 0
                else:
                    sphere_vals[0] += bucket_mean
                    sphere_vals[1] += weight
            if sphere_vals[1] == 0:
                self.term[term]['aba_void_indices'].append(irow)
            else:
                act_coeff = sphere_vals[0] / sphere_vals[1]
                self.term[term]['ns_act_vector'].append(act_coeff)

    #@profile
    def get_ns_act(self, term, thresh=0, method='knn', search_radii=5, k=None):
        """ Generates NS activation vector about ABA MNI coordinates  """
        if self.__check_static_members() == 1:
            return 1
        if not self.is_term(term):
            print "'%s' is not a registered term." % term
            return 1

        ns_coord_tree, ns_coord_act_df = self.__term_to_coords(term, thresh)
        if type(ns_coord_tree) == int:
            return 1

        self.term[term] = {}
        self.term[term]['ns_act_vector'] = []
        self.term[term]['aba_void_indices'] = []

        if method == 'knn':
            if k is None:
                k = 20
            self.__knn_method(term, ns_coord_act_df, ns_coord_tree, search_radii, k)
        elif method == 'sphere':
            if k is not None:
                raise ValueError("'k' parameter cannot be used with 'sphere' method.")
            self.__sphere_method(term, ns_coord_act_df, ns_coord_tree, search_radii)
        else:
            raise TypeError("'%s' is not a valid parameter value for 'method' parameter, use either 'knn' or 'sphere"
                            % method)

    def make_ge_ns_mat(self, ns_term, entrez_ids):
        if self.__check_static_members() == 1:
            return 1
        if self.check_entrez_struct(entrez_ids) == 1:
            return 1

        if ns_term in self.term and all([key in self.ge for key in entrez_ids]):
            ge_ns_mat = []
            for entrez_id in entrez_ids:
                aba_indices = np.array([i for i in range(len(self.aba['mni_coords']))
                                        if i not in self.term[ns_term]['aba_void_indices']])
                ge_ns_mat.append(self.ge[entrez_id][aba_indices])
            ge_ns_mat.append(self.term[ns_term]['ns_act_vector'])
            return np.vstack(ge_ns_mat).T
        else:
            print "Either term['%s'] or one or more Entrez ID keys does not exist; please check arguments" \
                  % ns_term

    def set_ns_weight_f(self, f):
        try:
            print "Test: f(e) = %.2f" % f(np.e)
            self.__ns_weight_f = f
        except TypeError:
            print "'f' is improper, ensure 'f' receives only one parameter and returns a numeric type"

    def __check_static_members(self):
        for val in self.aba.itervalues():
            if val is None:
                print "Unassigned Nsaba 'aba' static variable: see Nsaba.aba_load(path)"
                return 1
        for val in self.ns.itervalues():
            if val is None:
                print "Unassigned Nsaba 'ns' static variable: see Nsaba.ns_load(path)"
                return 1
        return 0
<|endoftext|>