<commit_before>from __future__ import print_function
import numpy as np
from theano import *
import theano.tensor as T
import lasagne
import lasagne.layers

class SharedBatch(object):
	def __init__(self, size, view_size, action_hist_size):
		self.size = size
		self.has_action_hist = action_hist_size.w > 0
		self.v = shared(np.zeros((self.size, 1, view_size.w, view_size.h), dtype=theano.config.floatX))
		if self.has_action_hist:
			self.ah = shared(
				np.zeros((self.size, 1, action_hist_size.w, action_hist_size.h), dtype=theano.config.floatX))
		self.a = shared(np.zeros((self.size, 1), dtype=np.int32), broadcastable=(False, True))
		self.v2 = shared(np.zeros((self.size, 1, view_size.w, view_size.h), dtype=theano.config.floatX))
		if self.has_action_hist:
			self.ah2 = shared(
				np.zeros((self.size, 1, action_hist_size.w, action_hist_size.h), dtype=theano.config.floatX))
		self.r = shared(np.zeros((self.size, 1), dtype=theano.config.floatX), broadcastable=(False, True))
		self.t = shared(np.zeros((self.size, 1), dtype=np.int32), broadcastable=(False, True))

	def set(self, v, ah, a, v2, ah2, r, t):
		self.v.set_value(v)
		if self.has_action_hist:
			self.ah.set_value(ah)
		self.a.set_value(a)
		self.v2.set_value(v2)
		if self.has_action_hist:
			self.ah2.set_value(ah2)
		self.r.set_value(r)
		self.t.set_value(t)

	def givens(self, views, action_hists, actions, next_states, next_action_hists, rewards, terminals):
		return {
			views: self.v,
			action_hists: self.ah,
			actions: self.a,
			next_states: self.v2,
			next_action_hists: self.ah2,
			rewards: self.r,
			terminals: self.t
		} if self.has_action_hist else {
			views: self.v,
			actions: self.a,
			next_states: self.v2,
			rewards: self.r,
			terminals: self.t
		}

class SharedState(object):
	def __init__(self, view_size, action_hist_size):
		self.view_size = view_size
		self.action_hist_size = action_hist_size
		self.has_action_hist = self.action_hist_size.w > 0
		self.v = theano.shared(np.zeros((view_size.w, view_size.h), dtype=theano.config.floatX))
		if self.has_action_hist:
			self.ah = theano.shared(np.zeros((action_hist_size.w, action_hist_size.h), dtype=theano.config.floatX))

	def set(self, v, ah):
		self.v.set_value(v)
		if self.has_action_hist:
			self.ah.set_value(ah)

	def givens(self, views, action_hists):
		return {
			views: self.v.reshape((1, 1, self.view_size.w, self.view_size.h)),
			action_hists: self.ah.reshape((1, 1, self.action_hist_size.w, self.action_hist_size.h))
		} if self.has_action_hist else {
			views: self.v.reshape((1, 1, self.view_size.w, self.view_size.h))
		}

class LasagneMlpModel(object):
	"""clone_interval: 0 or negative disables cloning"""

	def __init__(self, logger, batch_size, discount, actions, view_size, action_hist_size,
				 network_builder, optimizer, clone_interval=0):
		self.logger = logger
		self.batch_size = batch_size
		self.discount = discount
		self.actions = actions
		self.view_size = view_size
		self.action_hist_size = action_hist_size
		self.network_builder = network_builder
		self.optimizer = optimizer
		self.clone_interval = clone_interval
		self.shared_batch = SharedBatch(batch_size, view_size, action_hist_size)
		self.shared_state = SharedState(view_size, action_hist_size)
		self.net_out = None
		self.net_out_next = None
		self.train_fn = None
		self.predict_fn = None
		self.build_model()
		self.steps_clone = 0

	def build_network(self, network_builder, view_size, action_hist_size):
		net_in_views = lasagne.layers.InputLayer(name='views', shape=(None, 1, view_size.w, view_size.h))
		net_in_actions = lasagne.layers.InputLayer(name='action_hists',
												   shape=(None, 1, action_hist_size.w, action_hist_size.h)) \
			if action_hist_size.w > 0 else None
		net_out = network_builder(net_in_views, net_in_actions)
		return net_in_views, net_in_actions, net_out

	def build_model(self):
		views = T.tensor4('views')
		action_hists = T.tensor4('action_hists')
		actions = T.icol('actions')
		next_views = T.tensor4('next_views')
		next_action_hists = T.tensor4('next_action_hists')
		rewards = T.col('rewards')
		terminals = T.icol('terminals')

		# initialize network(s) for computing q-values
		net_in_view, net_in_action_hist, self.net_out = self.build_network(self.network_builder, self.view_size,
																		   self.action_hist_size)
		inputs = {net_in_view: views, net_in_action_hist: action_hists} if self.action_hist_size.w > 0 else {
			net_in_view: views}
		q_vals = lasagne.layers.get_output(self.net_out, inputs)
		if self.clone_interval > 0:
			net_in_view_next, net_in_action_hist_next, self.net_out_next = self.build_network(self.network_builder,
																							  self.view_size,
																							  self.action_hist_size)
			next_inputs = {net_in_view_next: next_views, net_in_action_hist_next: next_action_hists} \
				if self.action_hist_size.w > 0 else {net_in_view_next: next_views}
			next_q_vals = lasagne.layers.get_output(self.net_out_next, next_inputs)
			self._clone()
		else:
			next_inputs = {net_in_view: next_views, net_in_action_hist: next_action_hists} \
				if self.action_hist_size.w > 0 else {net_in_view: next_views}
			next_q_vals = lasagne.layers.get_output(self.net_out, next_inputs)
		# define loss computation
		actionmask = T.eq(T.arange(4).reshape((1, -1)), actions.reshape((-1, 1))).astype(theano.config.floatX)
		terminals_float = terminals.astype(theano.config.floatX)
		target = rewards + \
				 (T.ones_like(terminals_float) - terminals_float) * \
				 self.discount * T.max(next_q_vals, axis=1, keepdims=True)
		output = (q_vals * actionmask).sum(axis=1).reshape((-1, 1))
		diff = target - output
		loss = diff ** 2  # TODO error clipping
		loss = T.mean(loss)  # batch accumulator sum or mean

		# define network update for training
		params = lasagne.layers.helper.get_all_params(self.net_out)
		updates = self.optimizer(loss, params)
		train_givens = self.shared_batch.givens(views, action_hists, actions, next_views, next_action_hists, rewards,
												terminals)
		self.train_fn = theano.function([], [loss], updates=updates, givens=train_givens)

		# define output prediction
		predict_givens = self.shared_state.givens(views, action_hists)
		self.predict_fn = theano.function([], q_vals[0], givens=predict_givens)

	def _clone(self):
		param_values = lasagne.layers.get_all_param_values(self.net_out)
		lasagne.layers.set_all_param_values(self.net_out_next, param_values)

	def predict_qs(self, view, action_hist):
		self.shared_state.set(view, action_hist)
		return self.predict_fn()

	def update_qs(self, views, action_hists, actions, next_views, next_action_hists, rewards, terminals):
		self.shared_batch.set(views, action_hists, actions, next_views, next_action_hists, rewards, terminals)
		loss = self.train_fn()
		self.logger.log_parameter("loss", loss)
		self.steps_clone += 1
		if self.steps_clone == self.clone_interval:
			self._clone()
			self.steps_clone = 0
		# self.log_weights_diffs()
		self.log_weights()
		return loss

	def save(self, file_path):
		self.logger.log_message("Save model to {0}".format(file_path))
		np.savez(file_path, *lasagne.layers.get_all_param_values(self.net_out))

	def load(self, file_path):
		self.logger.log_message("Load model from {0}".format(file_path))
		with np.load(file_path) as f:
			param_values = [f['arr_%d' % i] for i in range(len(f.files))]
		lasagne.layers.set_all_param_values(self.net_out, param_values)

	def log_weights_diff(self):
		""" check if weights of model and model_cloned are actually different one step before cloning"""
		if self.steps_clone == self.clone_interval - 1:
			param_values_next = lasagne.layers.get_all_param_values(self.net_out_next)
			param_values_org = lasagne.layers.get_all_param_values(self.net_out)
			diffs = np.zeros(len(param_values_org), dtype=theano.config.floatX)
			for i in range(len(param_values_org)):
				diffs[i] = np.sum(param_values_next[i] - param_values_org[i])
			diff = np.sum(diffs)
			self.logger.log_parameter("weights_diff", diff)
			assert diff != 0

	def log_weights(self):
		layers = lasagne.layers.get_all_param_values(self.net_out)
		# calculate min, max, mean, std per layer
		for layer in range(len(layers)):
			weights = layers[layer]
			self.logger.log_parameter("weights",
									  [layer, np.min(weights), np.max(weights), np.mean(weights), np.std(weights)],
									  headers=["layer", "min", "max", "mean", "std"])
<commit_msg>Implement delta clipping and rmsprop like in deepmind paper, in LasagneMlpModel<commit_after>from __future__ import print_function
import numpy as np
from theano import *
import theano.tensor as T
import lasagne
import lasagne.layers

class SharedBatch(object):
	def __init__(self, size, view_size, action_hist_size):
		self.size = size
		self.has_action_hist = action_hist_size.w > 0
		self.v = shared(np.zeros((self.size, 1, view_size.w, view_size.h), dtype=theano.config.floatX))
		if self.has_action_hist:
			self.ah = shared(
				np.zeros((self.size, 1, action_hist_size.w, action_hist_size.h), dtype=theano.config.floatX))
		self.a = shared(np.zeros((self.size, 1), dtype=np.int32), broadcastable=(False, True))
		self.v2 = shared(np.zeros((self.size, 1, view_size.w, view_size.h), dtype=theano.config.floatX))
		if self.has_action_hist:
			self.ah2 = shared(
				np.zeros((self.size, 1, action_hist_size.w, action_hist_size.h), dtype=theano.config.floatX))
		self.r = shared(np.zeros((self.size, 1), dtype=theano.config.floatX), broadcastable=(False, True))
		self.t = shared(np.zeros((self.size, 1), dtype=np.int32), broadcastable=(False, True))

	def set(self, v, ah, a, v2, ah2, r, t):
		self.v.set_value(v)
		if self.has_action_hist:
			self.ah.set_value(ah)
		self.a.set_value(a)
		self.v2.set_value(v2)
		if self.has_action_hist:
			self.ah2.set_value(ah2)
		self.r.set_value(r)
		self.t.set_value(t)

	def givens(self, views, action_hists, actions, next_states, next_action_hists, rewards, terminals):
		return {
			views: self.v,
			action_hists: self.ah,
			actions: self.a,
			next_states: self.v2,
			next_action_hists: self.ah2,
			rewards: self.r,
			terminals: self.t
		} if self.has_action_hist else {
			views: self.v,
			actions: self.a,
			next_states: self.v2,
			rewards: self.r,
			terminals: self.t
		}

class SharedState(object):
	def __init__(self, view_size, action_hist_size):
		self.view_size = view_size
		self.action_hist_size = action_hist_size
		self.has_action_hist = self.action_hist_size.w > 0
		self.v = theano.shared(np.zeros((view_size.w, view_size.h), dtype=theano.config.floatX))
		if self.has_action_hist:
			self.ah = theano.shared(np.zeros((action_hist_size.w, action_hist_size.h), dtype=theano.config.floatX))

	def set(self, v, ah):
		self.v.set_value(v)
		if self.has_action_hist:
			self.ah.set_value(ah)

	def givens(self, views, action_hists):
		return {
			views: self.v.reshape((1, 1, self.view_size.w, self.view_size.h)),
			action_hists: self.ah.reshape((1, 1, self.action_hist_size.w, self.action_hist_size.h))
		} if self.has_action_hist else {
			views: self.v.reshape((1, 1, self.view_size.w, self.view_size.h))
		}

class LasagneMlpModel(object):
	"""clone_interval: 0 or negative disables cloning"""

	def __init__(self, logger, batch_size, discount, actions, view_size, action_hist_size,
				 network_builder, optimizer, clone_interval=0, clip_delta=0):
		self.logger = logger
		self.batch_size = batch_size
		self.discount = discount
		self.actions = actions
		self.view_size = view_size
		self.action_hist_size = action_hist_size
		self.network_builder = network_builder
		self.optimizer = optimizer
		self.clone_interval = clone_interval
		self.clip_delta = clip_delta
		self.shared_batch = SharedBatch(batch_size, view_size, action_hist_size)
		self.shared_state = SharedState(view_size, action_hist_size)
		self.net_out = None
		self.net_out_next = None
		self.train_fn = None
		self.predict_fn = None
		self.build_model()
		self.steps_clone = 0

	def build_network(self, network_builder, view_size, action_hist_size):
		net_in_views = lasagne.layers.InputLayer(name='views', shape=(None, 1, view_size.w, view_size.h))
		net_in_actions = lasagne.layers.InputLayer(name='action_hists',
												   shape=(None, 1, action_hist_size.w, action_hist_size.h)) \
			if action_hist_size.w > 0 else None
		net_out = network_builder(net_in_views, net_in_actions)
		return net_in_views, net_in_actions, net_out

	def build_model(self):
		views = T.tensor4('views')
		action_hists = T.tensor4('action_hists')
		actions = T.icol('actions')
		next_views = T.tensor4('next_views')
		next_action_hists = T.tensor4('next_action_hists')
		rewards = T.col('rewards')
		terminals = T.icol('terminals')

		# initialize network(s) for computing q-values
		net_in_view, net_in_action_hist, self.net_out = self.build_network(self.network_builder, self.view_size,
																		   self.action_hist_size)
		inputs = {net_in_view: views, net_in_action_hist: action_hists} if self.action_hist_size.w > 0 else {
			net_in_view: views}
		q_vals = lasagne.layers.get_output(self.net_out, inputs)
		if self.clone_interval > 0:
			net_in_view_next, net_in_action_hist_next, self.net_out_next = self.build_network(self.network_builder,
																							  self.view_size,
																							  self.action_hist_size)
			next_inputs = {net_in_view_next: next_views, net_in_action_hist_next: next_action_hists} \
				if self.action_hist_size.w > 0 else {net_in_view_next: next_views}
			next_q_vals = lasagne.layers.get_output(self.net_out_next, next_inputs)
			self._clone()
		else:
			next_inputs = {net_in_view: next_views, net_in_action_hist: next_action_hists} \
				if self.action_hist_size.w > 0 else {net_in_view: next_views}
			next_q_vals = lasagne.layers.get_output(self.net_out, next_inputs)
		# define loss computation
		actionmask = T.eq(T.arange(4).reshape((1, -1)), actions.reshape((-1, 1))).astype(theano.config.floatX)
		terminals_float = terminals.astype(theano.config.floatX)
		target = rewards + \
				 (T.ones_like(terminals_float) - terminals_float) * \
				 self.discount * T.max(next_q_vals, axis=1, keepdims=True)
		output = (q_vals * actionmask).sum(axis=1).reshape((-1, 1))
		diff = target - output
		if self.clip_delta > 0:
			# see https://github.com/spragunr/deep_q_rl/blob/master/deep_q_rl/q_network.py
			quadratic_part = T.minimum(abs(diff), self.clip_delta)
			linear_part = abs(diff) - quadratic_part
			loss = 0.5 * quadratic_part ** 2 + self.clip_delta * linear_part
		else:
			loss = diff ** 2  # TODO error clipping
		loss = T.mean(loss)  # batch accumulator sum or mean

		# define network update for training
		params = lasagne.layers.helper.get_all_params(self.net_out)
		updates = self.optimizer(loss, params)
		train_givens = self.shared_batch.givens(views, action_hists, actions, next_views, next_action_hists, rewards,
												terminals)
		self.train_fn = theano.function([], [loss], updates=updates, givens=train_givens)

		# define output prediction
		predict_givens = self.shared_state.givens(views, action_hists)
		self.predict_fn = theano.function([], q_vals[0], givens=predict_givens)

	def _clone(self):
		param_values = lasagne.layers.get_all_param_values(self.net_out)
		lasagne.layers.set_all_param_values(self.net_out_next, param_values)

	def predict_qs(self, view, action_hist):
		self.shared_state.set(view, action_hist)
		return self.predict_fn()

	def update_qs(self, views, action_hists, actions, next_views, next_action_hists, rewards, terminals):
		self.shared_batch.set(views, action_hists, actions, next_views, next_action_hists, rewards, terminals)
		loss = self.train_fn()
		self.logger.log_parameter("loss", loss)
		self.steps_clone += 1
		if self.steps_clone == self.clone_interval:
			self._clone()
			self.steps_clone = 0
		# self.log_weights_diffs()
		self.log_weights()
		return loss

	def save(self, file_path):
		self.logger.log_message("Save model to {0}".format(file_path))
		np.savez(file_path, *lasagne.layers.get_all_param_values(self.net_out))

	def load(self, file_path):
		self.logger.log_message("Load model from {0}".format(file_path))
		with np.load(file_path) as f:
			param_values = [f['arr_%d' % i] for i in range(len(f.files))]
		lasagne.layers.set_all_param_values(self.net_out, param_values)

	def log_weights_diff(self):
		""" check if weights of model and model_cloned are actually different one step before cloning"""
		if self.steps_clone == self.clone_interval - 1:
			param_values_next = lasagne.layers.get_all_param_values(self.net_out_next)
			param_values_org = lasagne.layers.get_all_param_values(self.net_out)
			diffs = np.zeros(len(param_values_org), dtype=theano.config.floatX)
			for i in range(len(param_values_org)):
				diffs[i] = np.sum(param_values_next[i] - param_values_org[i])
			diff = np.sum(diffs)
			self.logger.log_parameter("weights_diff", diff)
			assert diff != 0

	def log_weights(self):
		layers = lasagne.layers.get_all_param_values(self.net_out)
		# calculate min, max, mean, std per layer
		for layer in range(len(layers)):
			weights = layers[layer]
			self.logger.log_parameter("weights",
									  [layer, np.min(weights), np.max(weights), np.mean(weights), np.std(weights)],
									  headers=["layer", "min", "max", "mean", "std"])

def deepmind_rmsprop(loss_or_grads, params, learning_rate, rho, epsilon):
	from lasagne.updates import get_or_compute_grads
	from collections import OrderedDict

	grads = get_or_compute_grads(loss_or_grads, params)
	updates = OrderedDict()
	for param, grad in zip(params, grads):
		value = param.get_value(borrow=True)
		acc_grad = theano.shared(np.zeros(value.shape, dtype=value.dtype),
								 broadcastable=param.broadcastable)
		acc_grad_new = rho * acc_grad + (1 - rho) * grad
		acc_rms = theano.shared(np.zeros(value.shape, dtype=value.dtype),
								broadcastable=param.broadcastable)
		acc_rms_new = rho * acc_rms + (1 - rho) * grad ** 2
		updates[acc_grad] = acc_grad_new
		updates[acc_rms] = acc_rms_new
		updates[param] = (param - learning_rate * (grad /
						   T.sqrt(acc_rms_new - acc_grad_new ** 2 + epsilon)))
	return updates
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
    tests.unit.context_test
    ~~~~~~~~~~~~~~~~~~~~
'''
# Import python libs
from __future__ import absolute_import
import tornado.stack_context
import tornado.gen
from tornado.testing import AsyncTestCase, gen_test
import threading
import time

# Import Salt Testing libs
from salttesting import TestCase
from salt.ext.six.moves import range
from salttesting.helpers import ensure_in_syspath
ensure_in_syspath('../../')

# Import Salt libs
from salt.utils.context import ContextDict, NamespacedDictWrapper

class ContextDictTests(AsyncTestCase):
    # how many threads/coroutines to run at a time
    num_concurrent_tasks = 5

    def setUp(self):
        super(ContextDictTests, self).setUp()
        self.cd = ContextDict()
        # set a global value
        self.cd['foo'] = 'global'

    def test_threads(self):
        '''Verify that ContextDict overrides properly within threads
        '''
        rets = []

        def tgt(x, s):
            inner_ret = []
            over = self.cd.clone()

            inner_ret.append(self.cd.get('foo'))
            with over:
                inner_ret.append(over.get('foo'))
                over['foo'] = x
                inner_ret.append(over.get('foo'))
                time.sleep(s)
                inner_ret.append(over.get('foo'))
                rets.append(inner_ret)

        threads = []
        for x in range(0, self.num_concurrent_tasks):
            s = self.num_concurrent_tasks - x
            t = threading.Thread(target=tgt, args=(x, s))
            t.start()
            threads.append(t)

        for t in threads:
            t.join()

        for r in rets:
            self.assertEqual(r[0], r[1])
            self.assertEqual(r[2], r[3])

    @gen_test
    def test_coroutines(self):
        '''Verify that ContextDict overrides properly within coroutines
        '''
        @tornado.gen.coroutine
        def secondary_coroutine(over):
            raise tornado.gen.Return(over.get('foo'))

        @tornado.gen.coroutine
        def tgt(x, s, over):
            inner_ret = []
            # first grab the global
            inner_ret.append(self.cd.get('foo'))
            # grab the child's global (should match)
            inner_ret.append(over.get('foo'))
            # override the global
            over['foo'] = x
            inner_ret.append(over.get('foo'))
            # sleep for some time to let other coroutines do this section of code
            yield tornado.gen.sleep(s)
            # get the value of the global again.
            inner_ret.append(over.get('foo'))
            # Call another coroutine to verify that we keep our context
            r = yield secondary_coroutine(over)
            inner_ret.append(r)
            raise tornado.gen.Return(inner_ret)

        futures = []

        for x in range(0, self.num_concurrent_tasks):
            s = self.num_concurrent_tasks - x
            over = self.cd.clone()

            f = tornado.stack_context.run_with_stack_context(
                tornado.stack_context.StackContext(lambda: over),  # pylint: disable=W0640
                lambda: tgt(x, s/5.0, over),  # pylint: disable=W0640
            )
            futures.append(f)

        wait_iterator = tornado.gen.WaitIterator(*futures)
        while not wait_iterator.done():
            r = yield wait_iterator.next()
            self.assertEqual(r[0], r[1])  # verify that the global value remails
            self.assertEqual(r[2], r[3])  # verify that the override sticks locally
            self.assertEqual(r[3], r[4])  # verify that the override sticks across coroutines

    def test_basic(self):
        '''Test that the contextDict is a dict
        '''
        # ensure we get the global value
        self.assertEqual(
            dict(self.cd),
            {'foo': 'global'},
        )

    def test_override(self):
        over = self.cd.clone()
        over['bar'] = 'global'
        self.assertEqual(
            dict(over),
            {'foo': 'global', 'bar': 'global'},
        )
        self.assertEqual(
            dict(self.cd),
            {'foo': 'global'},
        )
        with over:
            self.assertEqual(
                dict(over),
                {'foo': 'global', 'bar': 'global'},
            )
            self.assertEqual(
                dict(self.cd),
                {'foo': 'global', 'bar': 'global'},
            )
            over['bar'] = 'baz'
            self.assertEqual(
                dict(over),
                {'foo': 'global', 'bar': 'baz'},
            )
            self.assertEqual(
                dict(self.cd),
                {'foo': 'global', 'bar': 'baz'},
            )
        self.assertEqual(
            dict(over),
            {'foo': 'global', 'bar': 'baz'},
        )
        self.assertEqual(
            dict(self.cd),
            {'foo': 'global'},
        )

    def test_multiple_contexts(self):
        cds = []
        for x in range(0, 10):
            cds.append(self.cd.clone(bar=x))
        for x, cd in enumerate(cds):
            self.assertNotIn('bar', self.cd)
            with cd:
                self.assertEqual(
                    dict(self.cd),
                    {'bar': x, 'foo': 'global'},
                )
        self.assertNotIn('bar', self.cd)

class NamespacedDictWrapperTests(TestCase):
    PREFIX = 'prefix'

    def setUp(self):
        self._dict = {}

    def test_single_key(self):
        self._dict['prefix'] = {'foo': 'bar'}
        w = NamespacedDictWrapper(self._dict, 'prefix')
        self.assertEqual(w['foo'], 'bar')

    def test_multiple_key(self):
        self._dict['prefix'] = {'foo': {'bar': 'baz'}}
        w = NamespacedDictWrapper(self._dict, ('prefix', 'foo'))
        self.assertEqual(w['bar'], 'baz')
<commit_msg>Python3 compat for context_test<commit_after># -*- coding: utf-8 -*-
'''
    tests.unit.context_test
    ~~~~~~~~~~~~~~~~~~~~
'''
# Import python libs
from __future__ import absolute_import
import tornado.stack_context
import tornado.gen
from tornado.testing import AsyncTestCase, gen_test
import threading
import time

# Import Salt Testing libs
from salttesting import TestCase
from salt.ext.six.moves import range
from salttesting.helpers import ensure_in_syspath
ensure_in_syspath('../../')

# Import Salt libs
from salt.utils.context import ContextDict, NamespacedDictWrapper

class ContextDictTests(AsyncTestCase):
    # how many threads/coroutines to run at a time
    num_concurrent_tasks = 5

    def setUp(self):
        super(ContextDictTests, self).setUp()
        self.cd = ContextDict()
        # set a global value
        self.cd['foo'] = 'global'

    def test_threads(self):
        '''Verify that ContextDict overrides properly within threads
        '''
        rets = []

        def tgt(x, s):
            inner_ret = []
            over = self.cd.clone()

            inner_ret.append(self.cd.get('foo'))
            with over:
                inner_ret.append(over.get('foo'))
                over['foo'] = x
                inner_ret.append(over.get('foo'))
                time.sleep(s)
                inner_ret.append(over.get('foo'))
                rets.append(inner_ret)

        threads = []
        for x in range(0, self.num_concurrent_tasks):
            s = self.num_concurrent_tasks - x
            t = threading.Thread(target=tgt, args=(x, s))
            t.start()
            threads.append(t)

        for t in threads:
            t.join()

        for r in rets:
            self.assertEqual(r[0], r[1])
            self.assertEqual(r[2], r[3])

    @gen_test
    def test_coroutines(self):
        '''Verify that ContextDict overrides properly within coroutines
        '''
        @tornado.gen.coroutine
        def secondary_coroutine(over):
            raise tornado.gen.Return(over.get('foo'))

        @tornado.gen.coroutine
        def tgt(x, s, over):
            inner_ret = []
            # first grab the global
            inner_ret.append(self.cd.get('foo'))
            # grab the child's global (should match)
            inner_ret.append(over.get('foo'))
            # override the global
            over['foo'] = x
            inner_ret.append(over.get('foo'))
            # sleep for some time to let other coroutines do this section of code
            yield tornado.gen.sleep(s)
            # get the value of the global again.
            inner_ret.append(over.get('foo'))
            # Call another coroutine to verify that we keep our context
            r = yield secondary_coroutine(over)
            inner_ret.append(r)
            raise tornado.gen.Return(inner_ret)

        futures = []

        for x in range(0, self.num_concurrent_tasks):
            s = self.num_concurrent_tasks - x
            over = self.cd.clone()

            f = tornado.stack_context.run_with_stack_context(
                tornado.stack_context.StackContext(lambda: over),  # pylint: disable=W0640
                lambda: tgt(x, s/5.0, over),  # pylint: disable=W0640
            )
            futures.append(f)

        wait_iterator = tornado.gen.WaitIterator(*futures)
        while not wait_iterator.done():
            r = yield next(wait_iterator)
            self.assertEqual(r[0], r[1])  # verify that the global value remails
            self.assertEqual(r[2], r[3])  # verify that the override sticks locally
            self.assertEqual(r[3], r[4])  # verify that the override sticks across coroutines

    def test_basic(self):
        '''Test that the contextDict is a dict
        '''
        # ensure we get the global value
        self.assertEqual(
            dict(self.cd),
            {'foo': 'global'},
        )

    def test_override(self):
        over = self.cd.clone()
        over['bar'] = 'global'
        self.assertEqual(
            dict(over),
            {'foo': 'global', 'bar': 'global'},
        )
        self.assertEqual(
            dict(self.cd),
            {'foo': 'global'},
        )
        with over:
            self.assertEqual(
                dict(over),
                {'foo': 'global', 'bar': 'global'},
            )
            self.assertEqual(
                dict(self.cd),
                {'foo': 'global', 'bar': 'global'},
            )
            over['bar'] = 'baz'
            self.assertEqual(
                dict(over),
                {'foo': 'global', 'bar': 'baz'},
            )
            self.assertEqual(
                dict(self.cd),
                {'foo': 'global', 'bar': 'baz'},
            )
        self.assertEqual(
            dict(over),
            {'foo': 'global', 'bar': 'baz'},
        )
        self.assertEqual(
            dict(self.cd),
            {'foo': 'global'},
        )

    def test_multiple_contexts(self):
        cds = []
        for x in range(0, 10):
            cds.append(self.cd.clone(bar=x))
        for x, cd in enumerate(cds):
            self.assertNotIn('bar', self.cd)
            with cd:
                self.assertEqual(
                    dict(self.cd),
                    {'bar': x, 'foo': 'global'},
                )
        self.assertNotIn('bar', self.cd)

class NamespacedDictWrapperTests(TestCase):
    PREFIX = 'prefix'

    def setUp(self):
        self._dict = {}

    def test_single_key(self):
        self._dict['prefix'] = {'foo': 'bar'}
        w = NamespacedDictWrapper(self._dict, 'prefix')
        self.assertEqual(w['foo'], 'bar')

    def test_multiple_key(self):
        self._dict['prefix'] = {'foo': {'bar': 'baz'}}
        w = NamespacedDictWrapper(self._dict, ('prefix', 'foo'))
        self.assertEqual(w['bar'], 'baz')
<|endoftext|>


<commit_before>#!/usr/bin/env python

import numbers

import jsonobject
import model

class Mission(jsonobject.JSONSerializableObject):

    id = jsonobject.ReadonlyJSONProperty('id', 0, value_type=int)
    """ID."""
    name = jsonobject.ReadonlyJSONProperty('name', u'', value_type=unicode)
    """Name."""
    description = jsonobject.ReadonlyJSONProperty('description', u'',
                                                  value_type=unicode)
    """Description."""
    difficulty = jsonobject.ReadonlyJSONProperty('difficulty', 0,
                                                 value_type=int)
    """Difficulty."""
    DIFFICULTY_E = 1
    DIFFICULTY_D = 2
    DIFFICULTY_C = 3
    DIFFICULTY_B = 4
    DIFFICULTY_A = 5
    DIFFICULTY_S = 6
    DIFFICULTY_SS = 7
    DIFFICULTY_SSS = 8
    maparea = jsonobject.ReadonlyJSONProperty('maparea', 0, value_type=int)
    """Map area."""
    MAPAREA_BASE = 1
    MAPAREA_SOUTHWESTERN_ISLANDS = 2
    MAPAREA_NORTH = 3
    MAPAREA_WEST = 4
    MAPAREA_SOUTH = 5
    state = jsonobject.ReadonlyJSONProperty('state', 0, value_type=int)
    """State."""
    STATE_NEW = 0
    STATE_ACTIVE = 1
    STATE_COMPLETE = 2
    time = jsonobject.ReadonlyJSONProperty('time', 0, value_type=int)
    """Required time to complete in minutes."""
    ammo_consumption = jsonobject.ReadonlyJSONProperty(
        'ammo_consumption', 0, value_type=numbers.Number)
    """Ammo consumption relative to the fleet capacity. Ranges from 0 to 1."""
    fuel_consumption = jsonobject.ReadonlyJSONProperty(
        'fuel_consumption', 0, value_type=numbers.Number)
    """Fuel consumption relative to the fleet capacity. Ranges from 0 to 1."""
    bonus_items = jsonobject.ReadonlyJSONProperty('bonus_items', None)
    """TODO: Bonus items?"""
    active = jsonobject.ReadonlyJSONProperty('active', 0, value_type=int)
    """TODO: Active?"""

class MissionList(model.KCAAObject):

    missions = jsonobject.JSONProperty('missions', [], value_type=list,
                                       element_type=Mission)
    """Mission instances."""

    def update(self, api_name, response):
        super(MissionList, self).update(api_name, response)
        missions = []
        for data in response['api_data']:
            mission_data = jsonobject.parse(data)
            missions.append(Mission(
                id=mission_data.api_id,
                name=mission_data.api_name,
                description=mission_data.api_details,
                difficulty=mission_data.api_difficulty,
                maparea=mission_data.api_maparea_id,
                state=mission_data.api_state,
                time=mission_data.api_time,
                ammo_consumption=mission_data.api_use_bull,
                fuel_consumption=mission_data.api_use_fuel,
                active=mission_data.api_active))
        missions.sort(lambda x, y: x.id - y.id)
        self.missions = model.merge_list(self.missions, missions)
<commit_msg>Remove Mission.active, an unknown field.<commit_after>#!/usr/bin/env python

import numbers

import jsonobject
import model

class Mission(jsonobject.JSONSerializableObject):

    id = jsonobject.ReadonlyJSONProperty('id', 0, value_type=int)
    """ID."""
    name = jsonobject.ReadonlyJSONProperty('name', u'', value_type=unicode)
    """Name."""
    description = jsonobject.ReadonlyJSONProperty('description', u'',
                                                  value_type=unicode)
    """Description."""
    difficulty = jsonobject.ReadonlyJSONProperty('difficulty', 0,
                                                 value_type=int)
    """Difficulty."""
    DIFFICULTY_E = 1
    DIFFICULTY_D = 2
    DIFFICULTY_C = 3
    DIFFICULTY_B = 4
    DIFFICULTY_A = 5
    DIFFICULTY_S = 6
    DIFFICULTY_SS = 7
    DIFFICULTY_SSS = 8
    maparea = jsonobject.ReadonlyJSONProperty('maparea', 0, value_type=int)
    """Map area."""
    MAPAREA_BASE = 1
    MAPAREA_SOUTHWESTERN_ISLANDS = 2
    MAPAREA_NORTH = 3
    MAPAREA_WEST = 4
    MAPAREA_SOUTH = 5
    state = jsonobject.ReadonlyJSONProperty('state', 0, value_type=int)
    """State."""
    STATE_NEW = 0
    STATE_ACTIVE = 1
    STATE_COMPLETE = 2
    time = jsonobject.ReadonlyJSONProperty('time', 0, value_type=int)
    """Required time to complete in minutes."""
    ammo_consumption = jsonobject.ReadonlyJSONProperty(
        'ammo_consumption', 0, value_type=numbers.Number)
    """Ammo consumption relative to the fleet capacity. Ranges from 0 to 1."""
    fuel_consumption = jsonobject.ReadonlyJSONProperty(
        'fuel_consumption', 0, value_type=numbers.Number)
    """Fuel consumption relative to the fleet capacity. Ranges from 0 to 1."""
    bonus_items = jsonobject.ReadonlyJSONProperty('bonus_items', None)
    """TODO: Bonus items?"""

class MissionList(model.KCAAObject):

    missions = jsonobject.JSONProperty('missions', [], value_type=list,
                                       element_type=Mission)
    """Mission instances."""

    def update(self, api_name, response):
        super(MissionList, self).update(api_name, response)
        missions = []
        for data in response['api_data']:
            mission_data = jsonobject.parse(data)
            missions.append(Mission(
                id=mission_data.api_id,
                name=mission_data.api_name,
                description=mission_data.api_details,
                difficulty=mission_data.api_difficulty,
                maparea=mission_data.api_maparea_id,
                state=mission_data.api_state,
                time=mission_data.api_time,
                ammo_consumption=mission_data.api_use_bull,
                fuel_consumption=mission_data.api_use_fuel))
        missions.sort(lambda x, y: x.id - y.id)
        self.missions = model.merge_list(self.missions, missions)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
===============================================================================
Atom class with extended feature set (:mod:`sknano.core.atoms._extended_atom`)
===============================================================================

An "eXtended" `Atom` class for structure analysis.

.. currentmodule:: sknano.core.atoms._extended_atom

"""
from __future__ import absolute_import, division, print_function
__docformat__ = 'restructuredtext en'

import numpy as np

from sknano.core.math import Vector
from ._atom import Atom

__all__ = ['XAtom']

class XAtom(Atom):
    """An eXtended `Atom` class for structure analysis.

    Parameters
    ----------
    element : {str, int}, optional
        A string representation of the element symbol or an integer specifying
        an element atomic number.
    x, y, z : float, optional
        :math:`x, y, z` components of `XAtom` position vector relative to
        origin.
    atomID : int, optional
        atom ID
    moleculeID : int, optional
        molecule ID
    atomtype : int, optional
        atom type
    q : {int, float}, optional
        Net charge of `XAtom`.
    vx, vy, vz : float, optional
        :math:`v_x, v_y, v_z` components of `XAtom` velocity.
    nx, ny, nz : int, optional
        :math:`n_x, n_y, n_z` image flags
    CN : int, optional
        `XAtom` coordination number.
    NN : sequence, optional
        List of nearest-neighbor `XAtom` objects instances

    """

    def __init__(self, element=None, atomID=0, moleculeID=0, atomtype=1,
                 q=0., m=None, mass=None, x=None, y=None, z=None,
                 vx=None, vy=None, vz=None, nx=None, ny=None, nz=None,
                 CN=None, NN=None, **kwargs):
        if m is None and mass is not None:
            m = mass

        super(XAtom, self).__init__(element=element, m=m, x=x, y=y, z=z)

        self._v = Vector([vx, vy, vz])

        self._n = np.zeros(3, dtype=int)
        for i, ni in enumerate((nx, ny, nz)):
            if ni is not None:
                self._n[i] = ni

        self._atomID = int(atomID)
        self._moleculeID = int(moleculeID)
        self._atomtype = int(atomtype)
        self._q = q
        self._CN = CN
        self._NN = NN

        self._attributes.extend(
            ['q','v', 'atomID', 'moleculeID', 'atomtype', 'CN', 'NN'])

    def __repr__(self):
        """Return string representation of `XAtom`."""
        reprstr = "Atom(element={element!r}, atomID={atomID!r}, " + \
            "moleculeID={moleculeID!r}, atomtype={atomtype!r}, " + \
            "q={q!r}, m={m!r}, x={x:.6f}, y={y:.6f}, z={z:.6f})"
        parameters = dict(element=self.element, atomID=self.atomID,
                          moleculeID=self.moleculeID, atomtype=self.atomtype,
                          q=self.q, m=self.m, x=self.x, y=self.y, z=self.z)

        return reprstr.format(**parameters)

    @property
    def CN(self):
        """Return `XAtom` coordination number."""
        return self._CN

    @CN.setter
    def CN(self, value=int):
        """Set `XAtom` coordination number."""
        self._CN = int(value)

    @property
    def NN(self):
        """Return list of nearest-neighbor `XAtom` objects."""
        return self._NN

    @NN.setter
    def NN(self, value=list):
        """Set list of nearest-neighbor `XAtom` objects."""
        self._NN = value

    @property
    def q(self):
        """Charge :math:`q` as multiple of elementary charge :math:`e`.

        """
        return self._q

    @q.setter
    def q(self, value=float):
        """Set `XAtom` charge :math:`q`.

        Parameters
        ----------
        value : {int, float}
            net charge on atom as a multiple of the elementary charge
            :math:`e`.

        """
        self._q = value

    @property
    def atomID(self):
        """:attr:`~XAtom.atomID`."""
        return self._atomID

    @atomID.setter
    def atomID(self, value=int):
        """Set atom ID of atom.

        Parameters
        ----------
        value : int
            atom ID

        """
        self._atomID = int(value)

    @property
    def moleculeID(self):
        """:attr:`~XAtom.moleculeID`."""
        return self._moleculeID

    @moleculeID.setter
    def moleculeID(self, value=int):
        """Set :attr:`~XAtom.moleculeID` attribute.

        Parameters
        ----------
        value : int
            molecule ID

        """
        self._moleculeID = int(value)

    @property
    def atomtype(self):
        """:attr:`~XAtom.atomtype`."""
        return self._atomtype

    @atomtype.setter
    def atomtype(self, value=int):
        """Set :attr:`~XAtom.atomtype` attribute.

        Parameters
        ----------
        value : int
            atom type

        """
        self._atomtype = int(value)

    @property
    def nx(self):
        """:math:`n_x` image flag.

        Returns
        -------
        int
            :math:`n_x` image flag.

        """
        return self._n[0]

    @nx.setter
    def nx(self, value=int):
        """Set :math:`n_x` image flag.

        Parameters
        ----------
        value : float
            :math:`n_x` image flag.

        """
        self._n[0] = int(value)

    @property
    def ny(self):
        """:math:`n_y` image flag.

        Returns
        -------
        int
            :math:`n_y` image flag.

        """
        return self._n[1]

    @ny.setter
    def ny(self, value=int):
        """Set :math:`n_y` image flag.

        Parameters
        ----------
        value : float
            :math:`n_y` image flag.

        """
        self._n[1] = int(value)

    @property
    def nz(self):
        """:math:`n_z` image flag.

        Returns
        -------
        int
            :math:`n_z` image flag.

        """
        return self._n[2]

    @nz.setter
    def nz(self, value=int):
        """Set `XAtom` :math:`n_z` image flag.

        Parameters
        ----------
        value : float
            :math:`n_z` image flag.

        """
        self._n[2] = int(value)

    @property
    def n(self):
        """:math:`n_x, n_y, n_z` image flags of `XAtom`.

        Returns
        -------
        ndarray
            3-element ndarray of [:math:`n_x`, :math:`n_y`, :math:`n_z`]
            image flags of `XAtom`.

        """
        return self._n

    @n.setter
    def n(self, value=np.ndarray):
        """Set :math:`n_x, n_y, n_z` image flags of `XAtom`.

        Parameters
        ----------
        value : array_like
            3-element ndarray of [:math:`n_x`, :math:`n_y`, :math:`n_z`]
            image flags of `XAtom`.

        """
        self._n[:] = value

    @property
    def vx(self):
        """:math:`x` component of `XAtom` velocity vector"""
        return self._v.x

    @vx.setter
    def vx(self, value=float):
        """Set :math:`v_x`.

        Set :math:`v_x`, the :math:`x` component of `XAtom` velocity
        vector.

        Parameters
        ----------
        value : float
            :math:`v_x` component of velocity

        """
        self._v.x = value

    @property
    def vy(self):
        """:math:`x` component of `XAtom` velocity vector"""
        return self._v.y

    @vy.setter
    def vy(self, value=float):
        """Set :math:`v_y`.

        Set :math:`v_y`, the :math:`y` component of `XAtom` velocity
        vector.

        Parameters
        ----------
        value : float
            :math:`v_y` component of velocity

        """
        self._v.y = value

    @property
    def vz(self):
        """:math:`z` component of `XAtom` velocity vector"""
        return self._v.z

    @vz.setter
    def vz(self, value=float):
        """Set :math:`v_z`.

        Set :math:`v_z`, the :math:`z` component of `XAtom` velocity
        vector.

        Parameters
        ----------
        value : float
            :math:`v_z` component of velocity

        """
        self._v.z = value

    @property
    def v(self):
        """:math:`v_x, v_y, v_z` array of velocity components.

        Returns
        -------
        ndarray
            3-element ndarray of [:math:`v_x`, :math:`v_y`, :math:`v_z`]
            velocity components of `XAtom`.

        """
        return self._v

    @v.setter
    def v(self, value=np.ndarray):
        """Set :math:`x, y, z` components of `XAtom` velocity.

        Parameters
        ----------
        value : array_like
            3-element ndarray of [:math:`v_x`, :math:`v_y`, :math:`v_z`]
            velocity components of `XAtom`.

        """
        self._v[:] = value

    def compute_pyramidalization_angle(self):
        pass
<commit_msg>adding total_ordering wrapper for object comparisons<commit_after># -*- coding: utf-8 -*-
"""
===============================================================================
Atom class with extended feature set (:mod:`sknano.core.atoms._extended_atom`)
===============================================================================

An "eXtended" `Atom` class for structure analysis.

.. currentmodule:: sknano.core.atoms._extended_atom

"""
from __future__ import absolute_import, division, print_function
__docformat__ = 'restructuredtext en'

from functools import total_ordering
import numpy as np

from sknano.core.math import Vector
from ._atom import Atom

__all__ = ['XAtom']

@total_ordering
class XAtom(Atom):
    """An eXtended `Atom` class for structure analysis.

    Parameters
    ----------
    element : {str, int}, optional
        A string representation of the element symbol or an integer specifying
        an element atomic number.
    x, y, z : float, optional
        :math:`x, y, z` components of `XAtom` position vector relative to
        origin.
    atomID : int, optional
        atom ID
    moleculeID : int, optional
        molecule ID
    atomtype : int, optional
        atom type
    q : {int, float}, optional
        Net charge of `XAtom`.
    vx, vy, vz : float, optional
        :math:`v_x, v_y, v_z` components of `XAtom` velocity.
    nx, ny, nz : int, optional
        :math:`n_x, n_y, n_z` image flags
    CN : int, optional
        `XAtom` coordination number.
    NN : sequence, optional
        List of nearest-neighbor `XAtom` objects instances

    """

    def __init__(self, element=None, atomID=0, moleculeID=0, atomtype=1,
                 q=0., m=None, mass=None, x=None, y=None, z=None,
                 vx=None, vy=None, vz=None, nx=None, ny=None, nz=None,
                 CN=None, NN=None, **kwargs):
        if m is None and mass is not None:
            m = mass

        super(XAtom, self).__init__(element=element, m=m, x=x, y=y, z=z)

        self._v = Vector([vx, vy, vz])

        self._n = np.zeros(3, dtype=int)
        for i, ni in enumerate((nx, ny, nz)):
            if ni is not None:
                self._n[i] = ni

        self._atomID = int(atomID)
        self._moleculeID = int(moleculeID)
        self._atomtype = int(atomtype)
        self._q = q
        self._CN = CN
        self._NN = NN

        self._attributes.extend(
            ['q','v', 'atomID', 'moleculeID', 'atomtype', 'CN', 'NN'])

    def __repr__(self):
        """Return string representation of `XAtom`."""
        reprstr = "Atom(element={element!r}, atomID={atomID!r}, " + \
            "moleculeID={moleculeID!r}, atomtype={atomtype!r}, " + \
            "q={q!r}, m={m!r}, x={x:.6f}, y={y:.6f}, z={z:.6f})"
        parameters = dict(element=self.element, atomID=self.atomID,
                          moleculeID=self.moleculeID, atomtype=self.atomtype,
                          q=self.q, m=self.m, x=self.x, y=self.y, z=self.z)

        return reprstr.format(**parameters)

    def __eq__(self, other):
        return self is other

    def __lt__(self, other):
        return self.atomID < other.atomID

    @property
    def CN(self):
        """Return `XAtom` coordination number."""
        return self._CN

    @CN.setter
    def CN(self, value=int):
        """Set `XAtom` coordination number."""
        self._CN = int(value)

    @property
    def NN(self):
        """Return list of nearest-neighbor `XAtom` objects."""
        return self._NN

    @NN.setter
    def NN(self, value=list):
        """Set list of nearest-neighbor `XAtom` objects."""
        self._NN = value

    @property
    def q(self):
        """Charge :math:`q` as multiple of elementary charge :math:`e`.

        """
        return self._q

    @q.setter
    def q(self, value=float):
        """Set `XAtom` charge :math:`q`.

        Parameters
        ----------
        value : {int, float}
            net charge on atom as a multiple of the elementary charge
            :math:`e`.

        """
        self._q = value

    @property
    def atomID(self):
        """:attr:`~XAtom.atomID`."""
        return self._atomID

    @atomID.setter
    def atomID(self, value=int):
        """Set atom ID of atom.

        Parameters
        ----------
        value : int
            atom ID

        """
        self._atomID = int(value)

    @property
    def moleculeID(self):
        """:attr:`~XAtom.moleculeID`."""
        return self._moleculeID

    @moleculeID.setter
    def moleculeID(self, value=int):
        """Set :attr:`~XAtom.moleculeID` attribute.

        Parameters
        ----------
        value : int
            molecule ID

        """
        self._moleculeID = int(value)

    @property
    def atomtype(self):
        """:attr:`~XAtom.atomtype`."""
        return self._atomtype

    @atomtype.setter
    def atomtype(self, value=int):
        """Set :attr:`~XAtom.atomtype` attribute.

        Parameters
        ----------
        value : int
            atom type

        """
        self._atomtype = int(value)

    @property
    def nx(self):
        """:math:`n_x` image flag.

        Returns
        -------
        int
            :math:`n_x` image flag.

        """
        return self._n[0]

    @nx.setter
    def nx(self, value=int):
        """Set :math:`n_x` image flag.

        Parameters
        ----------
        value : float
            :math:`n_x` image flag.

        """
        self._n[0] = int(value)

    @property
    def ny(self):
        """:math:`n_y` image flag.

        Returns
        -------
        int
            :math:`n_y` image flag.

        """
        return self._n[1]

    @ny.setter
    def ny(self, value=int):
        """Set :math:`n_y` image flag.

        Parameters
        ----------
        value : float
            :math:`n_y` image flag.

        """
        self._n[1] = int(value)

    @property
    def nz(self):
        """:math:`n_z` image flag.

        Returns
        -------
        int
            :math:`n_z` image flag.

        """
        return self._n[2]

    @nz.setter
    def nz(self, value=int):
        """Set `XAtom` :math:`n_z` image flag.

        Parameters
        ----------
        value : float
            :math:`n_z` image flag.

        """
        self._n[2] = int(value)

    @property
    def n(self):
        """:math:`n_x, n_y, n_z` image flags of `XAtom`.

        Returns
        -------
        ndarray
            3-element ndarray of [:math:`n_x`, :math:`n_y`, :math:`n_z`]
            image flags of `XAtom`.

        """
        return self._n

    @n.setter
    def n(self, value=np.ndarray):
        """Set :math:`n_x, n_y, n_z` image flags of `XAtom`.

        Parameters
        ----------
        value : array_like
            3-element ndarray of [:math:`n_x`, :math:`n_y`, :math:`n_z`]
            image flags of `XAtom`.

        """
        self._n[:] = value

    @property
    def vx(self):
        """:math:`x` component of `XAtom` velocity vector"""
        return self._v.x

    @vx.setter
    def vx(self, value=float):
        """Set :math:`v_x`.

        Set :math:`v_x`, the :math:`x` component of `XAtom` velocity
        vector.

        Parameters
        ----------
        value : float
            :math:`v_x` component of velocity

        """
        self._v.x = value

    @property
    def vy(self):
        """:math:`x` component of `XAtom` velocity vector"""
        return self._v.y

    @vy.setter
    def vy(self, value=float):
        """Set :math:`v_y`.

        Set :math:`v_y`, the :math:`y` component of `XAtom` velocity
        vector.

        Parameters
        ----------
        value : float
            :math:`v_y` component of velocity

        """
        self._v.y = value

    @property
    def vz(self):
        """:math:`z` component of `XAtom` velocity vector"""
        return self._v.z

    @vz.setter
    def vz(self, value=float):
        """Set :math:`v_z`.

        Set :math:`v_z`, the :math:`z` component of `XAtom` velocity
        vector.

        Parameters
        ----------
        value : float
            :math:`v_z` component of velocity

        """
        self._v.z = value

    @property
    def v(self):
        """:math:`v_x, v_y, v_z` array of velocity components.

        Returns
        -------
        ndarray
            3-element ndarray of [:math:`v_x`, :math:`v_y`, :math:`v_z`]
            velocity components of `XAtom`.

        """
        return self._v

    @v.setter
    def v(self, value=np.ndarray):
        """Set :math:`x, y, z` components of `XAtom` velocity.

        Parameters
        ----------
        value : array_like
            3-element ndarray of [:math:`v_x`, :math:`v_y`, :math:`v_z`]
            velocity components of `XAtom`.

        """
        self._v[:] = value

    def compute_pyramidalization_angle(self):
        pass
<|endoftext|>


<commit_before>import os, logging
from flask import Flask
from flask_wtf.csrf import CsrfProtect

app = Flask(__name__)

CsrfProtect(app)

app.config.from_object(os.environ.get('SETTINGS'))

if not app.debug:
    app.logger.addHandler(logging.StreamHandler())
    app.logger.setLevel(logging.INFO)

app.logger.info("\nConfiguration\n%s\n" % app.config)

# govuk_template asset path
@app.context_processor
def asset_path_context_processor():
    return {'asset_path': '/static/govuk_template/'}

<commit_msg>Yeah but probably after config is read is a better idea<commit_after>import os, logging
from flask import Flask
from flask_wtf.csrf import CsrfProtect

app = Flask(__name__)
app.config.from_object(os.environ.get('SETTINGS'))

CsrfProtect(app)

if not app.debug:
    app.logger.addHandler(logging.StreamHandler())
    app.logger.setLevel(logging.INFO)

app.logger.info("\nConfiguration\n%s\n" % app.config)

# govuk_template asset path
@app.context_processor
def asset_path_context_processor():
    return {'asset_path': '/static/govuk_template/'}

<|endoftext|>


<commit_before>#!/usr/bin/python -u

f = open("pwn0meter.txt","r")
w = open("pwn0meter.html","w")
w.write("<TITLE>Pwn0meter</TITLE>
Pwn0meter

\n")

killers = {}
deaders = {}

l = f.readline()
while l != "":
	l = l.strip()
	if l == "":
		l = f.readline()
		continue
	( time, deader, killer ) = l.split("\t")
	if not killer in killers.keys():
		killers[killer] = []
	if not deader in deaders.keys():
		deaders[deader] = []
	killers[killer].append(deader)
	deaders[deader].append(killer)
	l = f.readline()
f.close()

print killers

sorted = killers.keys()
def sortFunc(s1, s2):
	kills1 = len(killers[s1]) - killers[s1].count(s1)
	kills2 = len(killers[s2]) - killers[s2].count(s2)
	if kills1 < kills2: return 1
	if kills1 > kills2: return -1
	deaths1 = len(deaders[s1])
	deaths2 = len(deaders[s2])
	if deaths1 < deaths2: return -1
	if deaths1 > deaths2: return 1
	return 0

sorted.sort(cmp=sortFunc)

for k in sorted:
	kills = len(killers[k])
	deaths = len(deaders[k])
	suicides = killers[k].count(k)
	kills -= suicides
	deaths -= suicides
	w.write("[B]%s[/B]: %i kills %i deaths %i suicides, killed:" % ( k, kills, deaths, suicides ))
	# Ugly killer sorting
	killedMax = {}
	for f in killers[k]:
		if not killers[k].count(f) in killedMax.keys():
			killedMax[killers[k].count(f)] = []
		if killedMax[killers[k].count(f)].count(f) == 0:
			killedMax[killers[k].count(f)].append(f)
	killedMax1 = killedMax.keys()
	killedMax1.sort(reverse=True)
	count = 0
	for f in killedMax1:
		for f1 in killedMax[f]:
			if f1 == k: # Don't write suicides
				continue
			count += 1
			if count >= 5:
				break
			if count != 1:
				w.write(",")
			w.write(" %s - %i" % ( f1, f ) )
	w.write("
\n")

w.write("\n")
w.close()
<commit_msg>some fixes to pwn0meter.py<commit_after>#!/usr/bin/python -u

import sys

f = open("pwn0meter.txt","r")
w = open("pwn0meter.html","w")
#w = sys.stdout
w.write("<TITLE>Pwn0meter</TITLE>
Pwn0meter

\n")

killers = {}
deaders = {}

for l in f.readlines():
	l = l.strip()
	if l == "":
		continue
	try:
		( time, deader, killer ) = l.split("\t")
	except:
		continue
	if killer.find("[CPU]") >= 0:
		continue
	if killer.find("The Third") >= 0:
		continue
	if not killer in killers.keys():
		killers[killer] = []
	if not deader in deaders.keys():
		deaders[deader] = []
	killers[killer].append(deader)
	deaders[deader].append(killer)
f.close()

#print killers

sorted = killers.keys()
def sortFunc(s1, s2):
	kills1 = len(killers[s1]) - killers[s1].count(s1)
	kills2 = len(killers[s2]) - killers[s2].count(s2)
	if kills1 < kills2: return 1
	if kills1 > kills2: return -1
	try:
		deaths1 = len(deaders[s1])
	except:
		deaths1 = 0
	try:
		deaths2 = len(deaders[s2])
	except:
		deaths2 = 0
	if deaths1 < deaths2: return -1
	if deaths1 > deaths2: return 1
	return 0

sorted.sort(cmp=sortFunc)

for k in sorted:
	kills = len(killers[k])
	try:
		deaths = len(deaders[k])
	except:
		deatsh = 0
	suicides = killers[k].count(k)
	kills -= suicides
	deaths -= suicides
	w.write("[B]%s[/B]: %i kills %i deaths %i suicides, killed:" % ( k, kills, deaths, suicides ))
	# Ugly killer sorting
	killedMax = {}
	for f in killers[k]:
		if not killers[k].count(f) in killedMax.keys():
			killedMax[killers[k].count(f)] = []
		if killedMax[killers[k].count(f)].count(f) == 0:
			killedMax[killers[k].count(f)].append(f)
	killedMax1 = killedMax.keys()
	killedMax1.sort(reverse=True)
	count = 0
	for f in killedMax1:
		for f1 in killedMax[f]:
			if f1 == k: # Don't write suicides
				continue
			count += 1
			if count >= 5:
				break
			if count != 1:
				w.write(",")
			w.write(" %s - %i" % ( f1, f ) )
	w.write("
\n")

w.write("\n")
w.close()
<|endoftext|>


<commit_before>
# -*- coding: utf-8 -*-

from __future__ import unicode_literals
from __future__ import division
from __future__ import absolute_import
from __future__ import print_function

import os
import logging
import subprocess
import multiprocessing
import itertools

from scipy.optimize import leastsq
from skimage import feature

import numpy as np
import pandas as pd

from ..utils import print_progress

log = logging.getLogger(__name__)

__all__ = []

DEFAULT_PARAMETERS = {'w_s': 0.7,
                      'peak_radius': 0.2,
                      'threshold': 27.,
                      'max_peaks': 1e4
                      }

def peak_detector(data_iterator,
                  metadata,
                  parallel=True,
                  show_progress=False,
                  parameters={}):
    """Gaussian peak detection described in Segré et al. Nature Methods, (2008).

    Parameters
    ----------
    data_iterator : python iterator
        To iterate over data.
    metadata : dict
        Metadata to scale detected peaks and parameters.
    parallel : bool
        Used several processes at once.
    show_progress : bool (default: False)
        Print progress bar during detection.
    parameters : dict
        Contains gaussian detection algorithm parameters:
            - w_s: int, optional
                Width (in um) of the sliding window over which the hypothesis ratio
                is computed :math:`w_s` in the article. It should be wide enough
                to contain some background to allow a proper noise evaluation.
            - peak_radius: float, optional
                Typical radius (in um) of the peaks to detect. It must be higher than one
                (as peaks less that a pixel wide would yield bogus results
            - threshold: float, optional
                Criterium for a positive detection (i.e. the null hypothesis is false).
                Corresponds to the :math:`\chi^2` parameter in the Constant False
                Alarm Rate section of the article supplementary text (p. 12).
                A higher `threshold` corresponds to a more stringent test.
                According to the authors, this parameters needs to be adjusted
                once for a given data set.
            - max_peaks : int, optional
                Deflation loop will stop if detected peaks is higher than max_peaks.

    Returns
    -------
    trajs : :class:`pd.DataFrame`
    """

    _parameters = DEFAULT_PARAMETERS.copy()
    _parameters.update(parameters)
    parameters = _parameters

    # Scale parameters in pixels
    parameters['w_s'] /= metadata['PhysicalSizeX']
    parameters['w_s'] = np.round(parameters['w_s'])
    parameters['peak_radius'] /= metadata['PhysicalSizeX']

    # Find number of stacks to process
    # Only iteration over T and Z are assumed
    n_stack = metadata['SizeT'] * metadata['SizeZ']

    if parallel:

        # Snippet to allow multiprocessing while importing
        # module such as numpy (only needed on linux)
        if os.name == 'posix':
            subprocess.call("taskset -p 0xff %d" % os.getpid(),
                            shell=True)#, stdout=subprocess.DEVNULL) ## Py2.7 compat

        def init_worker():
            import signal
            signal.signal(signal.SIGINT, signal.SIG_IGN)

        ncore = multiprocessing.cpu_count() + 1
        log.info('Parallel mode enabled: %i cores will be used to process %i stacks' %
                 (ncore, n_stack))
        pool = multiprocessing.Pool(processes=ncore, initializer=init_worker)

    # Build arguments list
    arguments = zip(data_iterator,
                    itertools.repeat(parameters),
                    range(n_stack))

    try:
        # Launch peak_detection
        if parallel:
            results = pool.imap_unordered(find_gaussian_peaks, arguments)
        else:
            results = map(find_gaussian_peaks, arguments)

        all_peaks = []

        # Get unordered results and log progress
        for i, (pos, peaks) in enumerate(results):

            n_peaks = len(peaks)
            percent_progression = (i + 1) / n_stack * 100

            if show_progress:
                message = ("%i/%i - %i peaks detected on stack n°%i" %
                           ((i + 1), n_stack, n_peaks, pos))
                print_progress(percent_progression, message)

            all_peaks.append((pos, peaks))

        if show_progress:
            print_progress(-1)

    except KeyboardInterrupt:
        if parallel:
            pool.terminate()
            pool.join()
        raise Exception('Detection has been canceled by user')

    if parallel:
        pool.close()
        pool.terminate()

    # Sort peaks and remove index used to sort
    all_peaks.sort(key=lambda x: x[0])
    all_peaks = [x[1] for x in all_peaks]

    # Store peaks in pd.DataFrame
    index = []
    peaks_df = []

    for n, peaks in enumerate(all_peaks):
        if peaks.any():
            for peak in peaks:
                peaks_df.append(peak)
                index.append((n,))

    if not peaks_df:
        return pd.DataFrame([])

    peaks_df = pd.DataFrame(peaks_df, columns=['x', 'y', 'w', 'I'], dtype='float')
    peaks_df.index = pd.MultiIndex.from_tuples(index, names=['t_stamp'])

    t_stamp = peaks_df.index.get_level_values('t_stamp').values
    peaks_df['t'] = t_stamp // metadata['SizeZ']
    peaks_df['z'] = t_stamp % metadata['SizeZ']

    peaks_df['label'] = np.arange(len(peaks_df))

    peaks_df = peaks_df.reset_index(level=['t_stamp', 'label'])
    peaks_df['t_stamp'] = peaks_df['t'].copy()
    peaks_df.set_index(['t_stamp', 'label'], inplace=True)

    peaks_df['x'] *= metadata["PhysicalSizeX"]
    peaks_df['y'] *= metadata["PhysicalSizeY"]
    peaks_df['z'] *= metadata["PhysicalSizeZ"]
    peaks_df['w'] *= metadata["PhysicalSizeX"]
    peaks_df['t'] *= metadata["TimeIncrement"]

    return peaks_df

def find_gaussian_peaks(args):  # pragma: no cover
    """
    Buffer function for _find_gaussian_peaks
    """
    frame, detection_parameters, i = args
    return (i, _find_gaussian_peaks(frame, **detection_parameters))

def _find_gaussian_peaks(image, w_s=15, peak_radius=1.5,
                         threshold=27., max_peaks=1e4):  # pragma: no cover
    """
    This function implements the Gaussian peak detection described
    in Segré et al. Nature Methods **5**, 8 (2008). It is based on a
    likelyhood ratio test on the presence or absence of a Gaussian
    peak on a patch moving over the input 2D image and a successive
    sub-pixel localization of the peaks by a least square fit.  This
    detection is followed recursively by a _deflation_ of the image
    from the detected peaks and a new detection, until no more peaks
    are found

    Parameters
    ----------

    image: a 2D array
        the input image
    w_s: int, optional
        the width of the sliding window over which the hypothesis ratio
        is computed :math:`w_s` in the article. It should be wide enough
        to contain some background to allow a proper noise evaluation.
    peak_radius: float, optional
        typical radius of the peaks to detect. It must be higher than one
        (as peaks less that a pixel wide would yield bogus results
    thershold: float, optional
        Criterium for a positive detection (i.e. the null hypothesis is false).
        Corresponds to the :mat:`\chi^2` parameter in the Constant False
        Alarm Rate section of the article supplementary text (p. 12).
        A higher `threshold` corresponds to a more stringent test.
        According to the authors, this parameters needs to be adjusted
        once for a given data set.

    Returns
    -------

    peaks: ndarray
        peaks is a Nx4 array, where N is the number of detected peaks in the
        image. Each line gives the x position, y position, width,
        and (background corrected) intensity of a detected peak (in that order).

    """
    peaks_coords = glrt_detection(image, peak_radius,
                                  w_s, threshold)
    peaks = gauss_estimation(image, peaks_coords, w_s)
    d_image = image_deflation(image, peaks, w_s)
    peaks_coords = glrt_detection(d_image, peak_radius,
                                  w_s, threshold)
    while len(peaks_coords) > 0 and len(peaks) < max_peaks:
        new_peaks = gauss_estimation(d_image, peaks_coords, w_s)
        # in case the 2D gauss fit fails
        if len(new_peaks) < 1:
            break
        peaks.extend(new_peaks[:])
        d_image = image_deflation(d_image, new_peaks, w_s)
        peaks_coords = glrt_detection(d_image, peak_radius,
                                      w_s, threshold)
    peaks = np.array(peaks)
    return peaks

def image_deflation(image, peaks, w_s):  # pragma: no cover
    """
    Substracts the detected Gaussian peaks from the input image and
    returns the deflated image.
    """
    d_image = image.copy()
    for peak in peaks:
        xc, yc, width, I = peak
        xc_rel = w_s // 2 + xc - np.floor(xc)
        yc_rel = w_s // 2 + yc - np.floor(yc)
        low_x = int(xc - w_s // 2)
        low_y = int(yc - w_s // 2)

        params = xc_rel, yc_rel, width, I, 0
        deflated_peak = gauss_continuous(params, w_s)
        d_image[low_x:low_x + w_s,
                low_y:low_y + w_s] -= deflated_peak.reshape((w_s, w_s))
    return d_image

def gauss_estimation(image, peaks_coords, w_s):  # pragma: no cover
    """
    Least square fit of a 2D Gauss peaks (with radial symmetry)
    on regions of width `w_s` centered on each element
    of `peaks_coords`.

    Parameters:
    ----------
    image : 2D array
        a greyscale 2D input image.
    peaks_coords: iterable of pairs of int.
       The peaks_coords should contain `(x, y)` pairs
       corresponding to the approximate peak center,
       in pixels.
    """
    peaks = []
    for coords in peaks_coords:
        low_x, low_y = coords - w_s // 2
        try:
            patch = image[low_x: low_x + w_s,
                          low_y: low_y + w_s]
            params, success = gauss_estimate(patch, w_s)
            xc, yc, width, I, bg = params
            if success and I > 0 and width < w_s:
                peaks.append([xc + low_x, yc + low_y, width, I])
        except IndexError:
            log.error('peak too close from the edge\n'
                      'use a smaller window\n'
                      'peak @ (%i, %i) discarded' % (coords[0], coords[1]))
            continue
    return peaks

def glrt_detection(image, r0, w_s, threshold):  # pragma: no cover
    """
    Implements the Generalized Likelyhood Ratio Test, by
    computing equation 4 in Segré et al. Supplementary Note (p. 12)
    in a window sliding other the image.

    Parameters:
    ----------
    image: array
        the 2D input image
    r0: float
        the detected Gaussian peak 1/e radius
    w_s: int
        Size of the sliding window over which the test is
        computed ( :math:`w_s` in the article).
    threshold: float
        Criterium for a positive detection (i.e. the null hypothesis is false).
        Corresponds to the :mat:`\chi^2` parameter in the Constant False
        Alarm Rate section of the article supplementary text (p. 12).
        A higher `threshold` corresponds to a more stringent test.
        According to the authors, this parameters needs to be adjusted
        once for a given data set.

    Returns:
    --------

    peaks_coords: array
        An Nx2 array containing (x, y) pairs of the detected peaks
        in integer pixel coordinates.
    """

    if isinstance(image, np.ma.core.MaskedArray):
        mask = image.mask
        image = image.data
    elif isinstance(image, np.ndarray):
        mask = None
    else:
        raise Exception("Image has to be np.ndarray or np.ma.core.MaskedArray")

    w, h = image.shape
    g_patch = gauss_patch(r0, w_s)
    g_patch -= g_patch.mean()
    g_squaresum = np.sum(g_patch ** 2)

    hmap = []
    for i, j in np.ndindex((w - w_s, h - w_s)):
        tmp = image[int(i): int(i + w_s), int(j): int(j + w_s)]
        h = hypothesis_map(tmp,
                           g_patch,
                           g_squaresum)
        hmap.append(h)
    hmap = np.array(hmap)

    try:
        hmap = -2 * hmap.reshape((w - w_s, h - w_s))
        peaks_coords = feature.peak_local_max(hmap, 3,
                                              threshold_abs=threshold)
        peaks_coords += w_s / 2
        if isinstance(mask, np.ndarray):
            peaks_coords = list(filter(lambda x: not mask[x[0], x[1]], peaks_coords))

        return peaks_coords

    except ValueError:
        return np.array([])

def hypothesis_map(patch, g_patch, g_squaresum):  # pragma: no cover
    """
    Computes the ratio for a given patch position.
    """
    w_s = patch.shape[0]
    # mean = patch.mean()
    multiplicative = g_patch * patch

    intensity = multiplicative.sum()
    normalisation = w_s * patch.std()
    ratio = (w_s ** 2 / 2.) * np.log(1 - (intensity
                                          / normalisation) ** 2
                                     / g_squaresum)
    return ratio

def gauss_estimate(patch, w_s):  # pragma: no cover
    """
    Least square 2D gauss fit
    """
    params0 = [w_s / 2., w_s / 2., 3.,
               np.float(patch.max() - patch.min()), np.float(patch.min())]
    errfunc = lambda p: patch.flatten() - gauss_continuous(p, w_s)
    return leastsq(errfunc, params0, xtol=0.01)

def gauss_continuous(params, w_s):  # pragma: no cover
    """2D gauss function with a float center position"""
    xc, yc, width, I, bg = params
    xc = np.float(xc)
    yc = np.float(yc)
    x = np.exp(- (np.arange(0, w_s) - xc) ** 2 / width ** 2)
    y = np.exp(- (np.arange(0, w_s) - yc) ** 2 / width ** 2)
    g_patch = I * np.outer(x, y) + bg
    return g_patch.flatten()

def gauss_patch(r0, w_s):  # pragma: no cover
    """
    Computes an w_s by w_s image with a
    power normalized  Gaussian peak with radial symmetry
    at its center.
    """
    x = y = np.exp(- (np.arange(w_s) - w_s // 2) ** 2 / r0 ** 2)
    A = 1. / (np.sqrt(np.pi) * r0)
    g_patch = A * np.outer(x, y)
    return g_patch

def gauss_discrete(r0, i, j, w_s):  # pragma: no cover
    """
    2D gauss function with a discrete center position
    """
    i -= w_s // 2
    j -= w_s // 2
    A = 1. / (np.sqrt(np.pi) * r0)
    return A * np.exp(-(i ** 2 + j ** 2) / r0 ** 2)
<commit_msg>Better log in peak detector<commit_after>
# -*- coding: utf-8 -*-

from __future__ import unicode_literals
from __future__ import division
from __future__ import absolute_import
from __future__ import print_function

import os
import logging
import subprocess
import multiprocessing
import itertools

from scipy.optimize import leastsq
from skimage import feature

import numpy as np
import pandas as pd

from ..utils import print_progress

log = logging.getLogger(__name__)

__all__ = []

DEFAULT_PARAMETERS = {'w_s': 0.7,
                      'peak_radius': 0.2,
                      'threshold': 27.,
                      'max_peaks': 1e4
                      }

def peak_detector(data_iterator,
                  metadata,
                  parallel=True,
                  show_progress=False,
                  parameters={}):
    """Gaussian peak detection described in Segré et al. Nature Methods, (2008).

    Parameters
    ----------
    data_iterator : python iterator
        To iterate over data.
    metadata : dict
        Metadata to scale detected peaks and parameters.
    parallel : bool
        Used several processes at once.
    show_progress : bool (default: False)
        Print progress bar during detection.
    parameters : dict
        Contains gaussian detection algorithm parameters:
            - w_s: int, optional
                Width (in um) of the sliding window over which the hypothesis ratio
                is computed :math:`w_s` in the article. It should be wide enough
                to contain some background to allow a proper noise evaluation.
            - peak_radius: float, optional
                Typical radius (in um) of the peaks to detect. It must be higher than one
                (as peaks less that a pixel wide would yield bogus results
            - threshold: float, optional
                Criterium for a positive detection (i.e. the null hypothesis is false).
                Corresponds to the :math:`\chi^2` parameter in the Constant False
                Alarm Rate section of the article supplementary text (p. 12).
                A higher `threshold` corresponds to a more stringent test.
                According to the authors, this parameters needs to be adjusted
                once for a given data set.
            - max_peaks : int, optional
                Deflation loop will stop if detected peaks is higher than max_peaks.

    Returns
    -------
    trajs : :class:`pd.DataFrame`
    """

    log.info('Initializing peak detection')

    _parameters = DEFAULT_PARAMETERS.copy()
    _parameters.update(parameters)
    parameters = _parameters

    # Scale parameters in pixels
    parameters['w_s'] /= metadata['PhysicalSizeX']
    parameters['w_s'] = np.round(parameters['w_s'])
    parameters['peak_radius'] /= metadata['PhysicalSizeX']

    # Find number of stacks to process
    # Only iteration over T and Z are assumed
    n_stack = metadata['SizeT'] * metadata['SizeZ']

    if parallel:

        # Snippet to allow multiprocessing while importing
        # module such as numpy (only needed on linux)
        if os.name == 'posix':
            subprocess.call("taskset -p 0xff %d" % os.getpid(),
                            shell=True)  # stdout=subprocess.DEVNULL) ## Py2.7 compat

        def init_worker():
            import signal
            signal.signal(signal.SIGINT, signal.SIG_IGN)

        ncore = multiprocessing.cpu_count() + 1
        pool = multiprocessing.Pool(processes=ncore, initializer=init_worker)

    print('kkk')

    # Build arguments list
    arguments = zip(data_iterator,
                    itertools.repeat(parameters),
                    range(n_stack))

    try:
        # Launch peak_detection
        if parallel:
            results = pool.imap_unordered(find_gaussian_peaks, arguments)
        else:
            results = map(find_gaussian_peaks, arguments)

        all_peaks = []
        print("ppppp")
        # Get unordered results and log progress
        for i, (pos, peaks) in enumerate(results):

            n_peaks = len(peaks)
            percent_progression = (i + 1) / n_stack * 100

            if show_progress:
                message = ("%i/%i - %i peaks detected on stack n°%i" %
                           ((i + 1), n_stack, n_peaks, pos))
                print_progress(percent_progression, message)

            all_peaks.append((pos, peaks))

        if show_progress:
            print_progress(-1)

    except KeyboardInterrupt:
        if parallel:
            pool.terminate()
            pool.join()
        raise Exception('Detection has been canceled by user')

    if parallel:
        pool.close()
        pool.terminate()

    # Sort peaks and remove index used to sort
    all_peaks.sort(key=lambda x: x[0])
    all_peaks = [x[1] for x in all_peaks]

    # Store peaks in pd.DataFrame
    index = []
    peaks_df = []

    for n, peaks in enumerate(all_peaks):
        if peaks.any():
            for peak in peaks:
                peaks_df.append(peak)
                index.append((n,))

    if not peaks_df:
        return pd.DataFrame([])

    log.info('Terminating peak detection')

    peaks_df = pd.DataFrame(peaks_df, columns=['x', 'y', 'w', 'I'], dtype='float')
    peaks_df.index = pd.MultiIndex.from_tuples(index, names=['t_stamp'])

    t_stamp = peaks_df.index.get_level_values('t_stamp').values
    peaks_df['t'] = t_stamp // metadata['SizeZ']
    peaks_df['z'] = t_stamp % metadata['SizeZ']

    peaks_df['label'] = np.arange(len(peaks_df))

    peaks_df = peaks_df.reset_index(level=['t_stamp', 'label'])
    peaks_df['t_stamp'] = peaks_df['t'].copy()
    peaks_df.set_index(['t_stamp', 'label'], inplace=True)

    peaks_df['x'] *= metadata["PhysicalSizeX"]
    peaks_df['y'] *= metadata["PhysicalSizeY"]
    peaks_df['z'] *= metadata["PhysicalSizeZ"]
    peaks_df['w'] *= metadata["PhysicalSizeX"]
    peaks_df['t'] *= metadata["TimeIncrement"]

    return peaks_df

def find_gaussian_peaks(args):  # pragma: no cover
    """
    Buffer function for _find_gaussian_peaks
    """
    frame, detection_parameters, i = args
    return (i, _find_gaussian_peaks(frame, **detection_parameters))

def _find_gaussian_peaks(image, w_s=15, peak_radius=1.5,
                         threshold=27., max_peaks=1e4):  # pragma: no cover
    """
    This function implements the Gaussian peak detection described
    in Segré et al. Nature Methods **5**, 8 (2008). It is based on a
    likelyhood ratio test on the presence or absence of a Gaussian
    peak on a patch moving over the input 2D image and a successive
    sub-pixel localization of the peaks by a least square fit.  This
    detection is followed recursively by a _deflation_ of the image
    from the detected peaks and a new detection, until no more peaks
    are found

    Parameters
    ----------

    image: a 2D array
        the input image
    w_s: int, optional
        the width of the sliding window over which the hypothesis ratio
        is computed :math:`w_s` in the article. It should be wide enough
        to contain some background to allow a proper noise evaluation.
    peak_radius: float, optional
        typical radius of the peaks to detect. It must be higher than one
        (as peaks less that a pixel wide would yield bogus results
    thershold: float, optional
        Criterium for a positive detection (i.e. the null hypothesis is false).
        Corresponds to the :mat:`\chi^2` parameter in the Constant False
        Alarm Rate section of the article supplementary text (p. 12).
        A higher `threshold` corresponds to a more stringent test.
        According to the authors, this parameters needs to be adjusted
        once for a given data set.

    Returns
    -------

    peaks: ndarray
        peaks is a Nx4 array, where N is the number of detected peaks in the
        image. Each line gives the x position, y position, width,
        and (background corrected) intensity of a detected peak (in that order).

    """
    peaks_coords = glrt_detection(image, peak_radius,
                                  w_s, threshold)
    peaks = gauss_estimation(image, peaks_coords, w_s)
    d_image = image_deflation(image, peaks, w_s)
    peaks_coords = glrt_detection(d_image, peak_radius,
                                  w_s, threshold)
    while len(peaks_coords) > 0 and len(peaks) < max_peaks:
        new_peaks = gauss_estimation(d_image, peaks_coords, w_s)
        # in case the 2D gauss fit fails
        if len(new_peaks) < 1:
            break
        peaks.extend(new_peaks[:])
        d_image = image_deflation(d_image, new_peaks, w_s)
        peaks_coords = glrt_detection(d_image, peak_radius,
                                      w_s, threshold)
    peaks = np.array(peaks)
    return peaks

def image_deflation(image, peaks, w_s):  # pragma: no cover
    """
    Substracts the detected Gaussian peaks from the input image and
    returns the deflated image.
    """
    d_image = image.copy()
    for peak in peaks:
        xc, yc, width, I = peak
        xc_rel = w_s // 2 + xc - np.floor(xc)
        yc_rel = w_s // 2 + yc - np.floor(yc)
        low_x = int(xc - w_s // 2)
        low_y = int(yc - w_s // 2)

        params = xc_rel, yc_rel, width, I, 0
        deflated_peak = gauss_continuous(params, w_s)
        d_image[low_x:low_x + w_s,
                low_y:low_y + w_s] -= deflated_peak.reshape((w_s, w_s))
    return d_image

def gauss_estimation(image, peaks_coords, w_s):  # pragma: no cover
    """
    Least square fit of a 2D Gauss peaks (with radial symmetry)
    on regions of width `w_s` centered on each element
    of `peaks_coords`.

    Parameters:
    ----------
    image : 2D array
        a greyscale 2D input image.
    peaks_coords: iterable of pairs of int.
       The peaks_coords should contain `(x, y)` pairs
       corresponding to the approximate peak center,
       in pixels.
    """
    peaks = []
    for coords in peaks_coords:
        low_x, low_y = coords - w_s // 2
        try:
            patch = image[low_x: low_x + w_s,
                          low_y: low_y + w_s]
            params, success = gauss_estimate(patch, w_s)
            xc, yc, width, I, bg = params
            if success and I > 0 and width < w_s:
                peaks.append([xc + low_x, yc + low_y, width, I])
        except IndexError:
            log.error('peak too close from the edge\n'
                      'use a smaller window\n'
                      'peak @ (%i, %i) discarded' % (coords[0], coords[1]))
            continue
    return peaks

def glrt_detection(image, r0, w_s, threshold):  # pragma: no cover
    """
    Implements the Generalized Likelyhood Ratio Test, by
    computing equation 4 in Segré et al. Supplementary Note (p. 12)
    in a window sliding other the image.

    Parameters:
    ----------
    image: array
        the 2D input image
    r0: float
        the detected Gaussian peak 1/e radius
    w_s: int
        Size of the sliding window over which the test is
        computed ( :math:`w_s` in the article).
    threshold: float
        Criterium for a positive detection (i.e. the null hypothesis is false).
        Corresponds to the :mat:`\chi^2` parameter in the Constant False
        Alarm Rate section of the article supplementary text (p. 12).
        A higher `threshold` corresponds to a more stringent test.
        According to the authors, this parameters needs to be adjusted
        once for a given data set.

    Returns:
    --------

    peaks_coords: array
        An Nx2 array containing (x, y) pairs of the detected peaks
        in integer pixel coordinates.
    """

    if isinstance(image, np.ma.core.MaskedArray):
        mask = image.mask
        image = image.data
    elif isinstance(image, np.ndarray):
        mask = None
    else:
        raise Exception("Image has to be np.ndarray or np.ma.core.MaskedArray")

    w, h = image.shape
    g_patch = gauss_patch(r0, w_s)
    g_patch -= g_patch.mean()
    g_squaresum = np.sum(g_patch ** 2)

    hmap = []
    for i, j in np.ndindex((w - w_s, h - w_s)):
        tmp = image[int(i): int(i + w_s), int(j): int(j + w_s)]
        h = hypothesis_map(tmp,
                           g_patch,
                           g_squaresum)
        hmap.append(h)
    hmap = np.array(hmap)

    try:
        hmap = -2 * hmap.reshape((w - w_s, h - w_s))
        peaks_coords = feature.peak_local_max(hmap, 3,
                                              threshold_abs=threshold)
        peaks_coords += w_s / 2
        if isinstance(mask, np.ndarray):
            peaks_coords = list(filter(lambda x: not mask[x[0], x[1]], peaks_coords))

        return peaks_coords

    except ValueError:
        return np.array([])

def hypothesis_map(patch, g_patch, g_squaresum):  # pragma: no cover
    """
    Computes the ratio for a given patch position.
    """
    w_s = patch.shape[0]
    # mean = patch.mean()
    multiplicative = g_patch * patch

    intensity = multiplicative.sum()
    normalisation = w_s * patch.std()
    ratio = (w_s ** 2 / 2.) * np.log(1 - (intensity
                                          / normalisation) ** 2
                                     / g_squaresum)
    return ratio

def gauss_estimate(patch, w_s):  # pragma: no cover
    """
    Least square 2D gauss fit
    """
    params0 = [w_s / 2., w_s / 2., 3.,
               np.float(patch.max() - patch.min()), np.float(patch.min())]
    errfunc = lambda p: patch.flatten() - gauss_continuous(p, w_s)
    return leastsq(errfunc, params0, xtol=0.01)

def gauss_continuous(params, w_s):  # pragma: no cover
    """2D gauss function with a float center position"""
    xc, yc, width, I, bg = params
    xc = np.float(xc)
    yc = np.float(yc)
    x = np.exp(- (np.arange(0, w_s) - xc) ** 2 / width ** 2)
    y = np.exp(- (np.arange(0, w_s) - yc) ** 2 / width ** 2)
    g_patch = I * np.outer(x, y) + bg
    return g_patch.flatten()

def gauss_patch(r0, w_s):  # pragma: no cover
    """
    Computes an w_s by w_s image with a
    power normalized  Gaussian peak with radial symmetry
    at its center.
    """
    x = y = np.exp(- (np.arange(w_s) - w_s // 2) ** 2 / r0 ** 2)
    A = 1. / (np.sqrt(np.pi) * r0)
    g_patch = A * np.outer(x, y)
    return g_patch

def gauss_discrete(r0, i, j, w_s):  # pragma: no cover
    """
    2D gauss function with a discrete center position
    """
    i -= w_s // 2
    j -= w_s // 2
    A = 1. / (np.sqrt(np.pi) * r0)
    return A * np.exp(-(i ** 2 + j ** 2) / r0 ** 2)
<|endoftext|>


<commit_before># Copyright 2014 Chris Matteri
# Released under the MIT License (http://opensource.org/licenses/mit-license.php)

from setup import ExtensionInstaller

def loader():
    return CCWXRXVP2Installer()

class CCWXRXVP2Installer(ExtensionInstaller):
    def __init__(self):
        super(CCWXRXVP2Installer, self).__init__(
            version="1.0",
            name='ccwxrxvp2',
            description='Driver for the Davis Vantage Pro 2 used with the '
                        'CC1101 Weather Receiver.',
            author="Chris Matteri",
            author_email="chrismatteri@gmail.com",
            config={
                'Station': {
                    'station_type': 'CCWXRXVP2'},
                'CCWXRXVP2': {
                    'poll_interval': '2.5',
                    'driver': 'user.ccwxrxvp2'}},
            files=[('bin/user', ['bin/user/ccwxrxvp2.py'])]
            )
<commit_msg>Corrected default poll interval to the value used by Davis stations with default ID of 1.<commit_after># Copyright 2014 Chris Matteri
# Released under the MIT License (http://opensource.org/licenses/mit-license.php)

from setup import ExtensionInstaller

def loader():
    return CCWXRXVP2Installer()

class CCWXRXVP2Installer(ExtensionInstaller):
    def __init__(self):
        super(CCWXRXVP2Installer, self).__init__(
            version="1.0",
            name='ccwxrxvp2',
            description='Driver for the Davis Vantage Pro 2 used with the '
                        'CC1101 Weather Receiver.',
            author="Chris Matteri",
            author_email="chrismatteri@gmail.com",
            config={
                'Station': {
                    'station_type': 'CCWXRXVP2'},
                'CCWXRXVP2': {
                    'poll_interval': '2.5625',
                    'driver': 'user.ccwxrxvp2'}},
            files=[('bin/user', ['bin/user/ccwxrxvp2.py'])]
            )
<|endoftext|>


<commit_before>from acl import AuthManager
from core import Core
from user import UserManager

version = '0.6.2'
<commit_msg>Version bump to fix jenkins<commit_after>from acl import AuthManager
from core import Core
from user import UserManager

version = '0.6.3'
<|endoftext|>


<commit_before>import tokenize, io, keyword, ast, dis, warnings, textwrap

def tokeni(string):
    lex = tokenize.tokenize(io.BytesIO(string.encode('utf8')).readline)
    for tok in list(lex)[1:-1]:
        if keyword.iskeyword(tok.string): tip = tok.string.upper()
        else: tip = tokenize.tok_name[tok.exact_type]
        print('\t' + tip + repr(tok.string))

def stablo_parsiranja(string):
    warnings.simplefilter('ignore')
    try: import parser
    except ImportError: return print(textwrap.dedent('''\
        U verziji 3.10 Python je prešao na novi parser, koji više nije
        zasnovan na beskontekstnoj gramatici (i nije dostupan kroz
        Pythonovu standardnu biblioteku). To je u skladu s onim što smo
        rekli na nastavi, da sve više parsera producira direktno AST umjesto
        stabala parsiranja. Više o razlozima možete pročitati u PEP617.
        Ako ipak želite vidjeti stablo parsiranja za gornji komad koda,
        pokrenite ovaj program pod Pythonom 3.9 ili nižim.'''))
    def ispis(t, razina, nastavak=False):
        if not nastavak: print(end=' '*2*razina)
        if len(t) == 2:
            if isinstance(t[~0], str):
                tip, sadržaj = t
                if keyword.iskeyword(sadržaj): tip = sadržaj.upper()
                else: tip = tokenize.tok_name[tip]
                print(tip + repr(sadržaj))
            else:
                print(t[0] - 256, end='>')
                ispis(t[1], razina, True)
        else:
            print(t[0] - 256)
            for podstablo in t[1:]:
                ispis(podstablo, razina + 1)
    ispis(parser.suite(string).tolist(), 0)

def apstablo(string): print(ast.dump(ast.parse(string)))

def bytecode(string): dis.dis(string)

if __name__ == '__main__':
    primjer = 'for x in 2, 3: print(x)'
    print(primjer)
    for funkcija in tokeni, stablo_parsiranja, apstablo, bytecode:
        print(funkcija.__name__.center(75, '-'))
        funkcija(primjer)
        print()

# Module:
#     body = [...]:
#         For:
#             target = Name(id='x', ctx=Store())
#             iter = Tuple(elts=[Num(n=2), Num(n=3)], ctx=Load())
#             body = [...]
#                 Expr:
#                     value = Call:
#                         func = Name(id='print', ctx=Load())
#                         args = [Name(id='x', ctx=Load())]
#                         keywords = []
#             orelse = []

<commit_msg>Overhaul regarding version switching<commit_after>import tokenize, io, keyword, ast, dis, warnings, textwrap

def tokeni(string):
    lex = tokenize.tokenize(io.BytesIO(string.encode('utf8')).readline)
    for tok in list(lex)[1:-1]:
        if keyword.iskeyword(tok.string): tip = tok.string.upper()
        else: tip = tokenize.tok_name[tok.exact_type]
        print('\t' + tip + repr(tok.string))

def stablo_parsiranja(string):
    warnings.simplefilter('ignore')
    try: import parser
    except ImportError: return print(textwrap.dedent('''\
        U verziji 3.10 Python je prešao na novi parser, koji više nije
        zasnovan na beskontekstnoj gramatici (i nije dostupan kroz
        Pythonovu standardnu biblioteku). To je u skladu s onim što smo
        rekli na nastavi, da sve više parsera producira direktno AST umjesto
        stabala parsiranja. Više o razlozima možete pročitati u PEP617.
        Ako ipak želite vidjeti stablo parsiranja za gornji komad koda,
        pokrenite ovaj program pod Pythonom 3.9 ili nižim.'''))
    def ispis(t, razina, nastavak=False):
        if not nastavak: print(end=' '*2*razina)
        if len(t) == 2:
            if isinstance(t[~0], str):
                tip, sadržaj = t
                if keyword.iskeyword(sadržaj): tip = sadržaj.upper()
                else: tip = tokenize.tok_name[tip]
                print(tip + repr(sadržaj))
            else:
                print(t[0] - 256, end='>')
                ispis(t[1], razina, True)
        else:
            print(t[0] - 256)
            for podstablo in t[1:]:
                ispis(podstablo, razina + 1)
    ispis(parser.suite(string).tolist(), 0)

def apstablo(string):
    try: print(ast.dump(ast.parse(string), indent=4))
    except TypeError:
        print(ast.dump(ast.parse(string)))
        print('Za ljepši ispis pokrenite ovo pod Pythonom 3.9 ili kasnijim.')

def bytecode(string): dis.dis(string)

def izvrši(string): exec(string)

def naredba(string): print(string)

if __name__ == '__main__':
    primjer = 'for x in 2, 3: print(x)'
    for funkcija in naredba, tokeni, stablo_parsiranja, \
                    apstablo, bytecode, izvrši:
        print(funkcija.__name__.center(75, '-'))
        funkcija(primjer)
        print()

# Module:
#     body = [...]:
#         For:
#             target = Name(id='x', ctx=Store())
#             iter = Tuple(elts=[Num(n=2), Num(n=3)], ctx=Load())
#             body = [...]
#                 Expr:
#                     value = Call:
#                         func = Name(id='print', ctx=Load())
#                         args = [Name(id='x', ctx=Load())]
#                         keywords = []
#             orelse = []

<|endoftext|>