<commit_before>
import sys
from spyvm import model, shadow, constants
from rpython.rlib.objectmodel import import_from_mixin
from rpython.rlib.rfloat import string_to_float

# Disables all optimized strategies, for debugging.
only_list_storage = False

class AbstractStorageStrategy(object):
    _immutable_fields_ = []
    _attrs_ = []
    _settled_ = True
    strategy_tag = 'abstract'
    needs_objspace = False

    def set_initial_storage(self, space, w_obj, size):
        raise NotImplementedError("Abstract base class")
    def set_storage_for_list(self, space, w_obj, collection):
        raise NotImplementedError("Abstract base class")
    def set_storage_copied_from(self, space, w_obj, w_source_obj, reuse_storage=False):
        raise NotImplementedError("Abstract base class")

    def store(self, space, w_obj, n0, w_val):
        if self.can_contain(space, w_val):
            self.do_store(space, w_obj, n0, w_val)
        new_strategy = find_strategy_for_object(w_val)
        return w_obj.store_with_new_strategy(space, new_strategy, n0, w_val)

    def can_contain(self, space, w_val):
        raise NotImplementedError("Abstract base class")
    def fetch(self, space, w_obj, n0):
        raise NotImplementedError("Abstract base class")
    def do_store(self, space, w_obj, n0, w_val):
        raise NotImplementedError("Abstract base class")

class AbstractListStorageStrategy(AbstractStorageStrategy):
    strategy_tag = 'abstract-list'

    def storage(self, w_obj):
        return w_obj.list_storage
    def set_initial_storage(self, space, w_obj, size):
        w_obj.list_storage = self.initial_storage(space, size)
    def set_storage_for_list(self, space, w_obj, collection):
        w_obj.list_storage = self.storage_for_list(space, collection)
    def set_storage_copied_from(self, space, w_obj, w_source_obj, reuse_storage=False):
        w_obj.list_storage = self.copy_storage_from(space, w_source_obj, reuse_storage)

    def initial_storage(self, space, size):
        raise NotImplementedError("Abstract base class")
    def storage_for_list(self, space, collection):
        raise NotImplementedError("Abstract base class")
    def copy_storage_from(self, space, w_obj, reuse_storage):
        old_strategy = w_obj.strategy
        if old_strategy == self and reuse_storage:
            return self.storage(w_obj)
        else:
            # This can be overridden and optimized (reuse_storage flag, less temporary storage)
            return self.storage_for_list(space, w_obj.fetch_all(space))

class AbstractIntStorageStrategy(AbstractStorageStrategy):
    strategy_tag = 'abstract-int'

    def storage(self, w_obj):
        return w_obj.int_storage
    def set_initial_storage(self, space, w_obj, size):
        w_obj.int_storage = self.initial_storage(space, size)
    def set_storage_for_list(self, space, w_obj, collection):
        w_obj.int_storage = self.storage_for_list(space, collection)
    def set_storage_copied_from(self, space, w_obj, w_source_obj, reuse_storage=False):
        w_obj.int_storage = self.copy_storage_from(space, w_source_obj, reuse_storage)

    def initial_storage(self, space, size):
        raise NotImplementedError("Abstract base class")
    def storage_for_list(self, space, collection):
        raise NotImplementedError("Abstract base class")
    def copy_storage_from(self, space, w_obj, reuse_storage):
        old_strategy = w_obj.strategy
        if old_strategy == self and reuse_storage:
            return self.storage(w_obj)
        else:
            # This can be overridden and optimized (reuse_storage flag, less temporary storage)
            return self.storage_for_list(space, w_obj.fetch_all(space))

class SingletonMeta(type):
    def __new__(cls, name, bases, dct):
        result = type.__new__(cls, name, bases, dct)
        result.singleton = result()
        return result

# this is the typical "initial" storage strategy, for when every slot
# in an object is still nil. No storage is allocated.
class AllNilStorageStrategy(AbstractStorageStrategy):
    __metaclass__ = SingletonMeta
    strategy_tag = 'allnil'

    def can_contain(self, space, w_obj):
        return w_obj == model.w_nil
    def fetch(self, space, w_obj, n0):
        return model.w_nil
    def do_store(self, space, w_obj, n0, w_val):
        pass

    def set_initial_storage(self, space, w_obj, size):
        pass
    def set_storage_for_list(self, space, w_obj, collection):
        pass
    def set_storage_copied_from(self, space, w_obj, w_source_obj, reuse_storage=False):
        pass

# This is the regular storage strategy that does not result in any
# optimizations but can handle every case. Applicable for both
# fixed-sized and var-sized objects.
class ListStorageStrategy(AbstractListStorageStrategy):
    __metaclass__ = SingletonMeta
    strategy_tag = 'list'

    def can_contain(self, space, w_val):
        return True
    def fetch(self, space, w_obj, n0):
        return self.storage(w_obj)[n0]
    def do_store(self, space, w_obj, n0, w_val):
        # TODO enable generalization by maintaining a counter of elements that are nil.
        self.storage(w_obj)[n0] = w_val
    def initial_storage(self, space, size):
        return [model.w_nil] * size
    def storage_for_list(self, space, collection):
        return [x for x in collection]
    def copy_storage_from(self, space, w_obj, reuse_storage=False):
        length = w_obj.basic_size()
        return [w_obj.strategy.fetch(space, w_obj, i) for i in range(length)]

class AbstractValueOrNilStorageStrategy(AbstractIntStorageStrategy):
    needs_objspace = True
    strategy_tag = 'abstract-valueOrNil'

    def can_contain(self, space, w_val):
        return w_val == model.w_nil or (isinstance(w_val, self.wrapper_class) and self.unwrap(space, w_val) != self.nil_value)

    def fetch(self, space, w_obj, n0):
        val = self.storage(w_obj)[n0]
        if val == self.nil_value:
            return space.w_nil
        else:
            return self.wrap(space, val)

    def do_store(self, space, w_obj, n0, w_val):
        store = self.storage(w_obj)
        if w_val == space.w_nil:
                store[n0] = self.nil_value
        else:
            store[n0] = self.unwrap(space, w_val)

    def initial_storage(self, space, size):
        return [self.nil_value] * size

    def storage_for_list(self, space, collection):
        length = len(collection)
        store = self.initial_storage(length)
        for i in range(length):
            if collection[i] != space.w_nil:
                store[i] = self.unwrap(space, collection[i])
        return store

class SmallIntegerOrNilStorageStrategy(AbstractValueOrNilStorageStrategy):
    __metaclass__ = SingletonMeta
    strategy_tag = 'float-orNil'
    nil_value = constants.MAXINT
    wrapper_class = model.W_SmallInteger
    def wrap(self, space, val): return space.wrap_int(val)
    def unwrap(self, space, w_val): return space.unwrap_int(w_val)

class FloatOrNilStorageStrategy(AbstractValueOrNilStorageStrategy):
    __metaclass__ = SingletonMeta
    strategy_tag = 'smallint-orNil'
    nil_value = string_to_float("-nan")
    wrapper_class = model.W_Float
    def wrap(self, space, val): return space.wrap_float(val)
    def unwrap(self, space, w_val): return space.unwrap_float(w_val)

def find_strategy_for_object(space, var):
    return find_strategy_for_objects(space, [var])

def find_strategy_for_objects(space, vars):
    if only_list_storage:
        ListStorageStrategy.singleton

    specialized_strategies = 3
    all_nil_can_handle = True
    small_int_can_handle = True
    float_can_handle = True
    for w_obj in vars:
        if all_nil_can_handle and not AllNilStorageStrategy.singleton.can_contain(space, w_obj):
            all_nil_can_handle = False
            specialized_strategies = specialized_strategies - 1
        if small_int_can_handle and not SmallIntegerOrNilStorageStrategy.singleton.can_contain(space, w_obj):
            small_int_can_handle = False
            specialized_strategies = specialized_strategies - 1
        if float_can_handle and not FloatOrNilStorageStrategy.singleton.can_contain(space, w_obj):
            float_can_handle = False
            specialized_strategies = specialized_strategies - 1

        if specialized_strategies <= 0:
            return ListStorageStrategy.singleton

    if all_nil_can_handle:
        return AllNilStorageStrategy.singleton
    if small_int_can_handle:
        return SmallIntegerOrNilStorageStrategy.singleton
    if float_can_handle:
        return FloatOrNilStorageStrategy.singleton

    # If this happens, please look for a bug in the code above.
    assert False, "No strategy could be found for list %r" % vars

def empty_strategy(s_containing_class):
    if s_containing_class is None:
        # This is a weird and rare special case for w_nil
        return ListStorageStrategy.singleton
    if not s_containing_class.isvariable() or only_list_storage:
        return ListStorageStrategy.singleton

    # A newly allocated object contains only nils.
    return AllNilStorageStrategy.singleton

def strategy_for_list(s_containing_class, vars):
    if s_containing_class is None:
        # This is a weird and rare special case for w_nil
        return ListStorageStrategy.singleton
    try:
        is_variable = s_containing_class.isvariable()
    except AttributeError:
        # TODO - This happens during bootstrapping phase, when filling in generic objects.
        # Ths class object shadows are not yet synchronized.
        return ListStorageStrategy.singleton

    if is_variable:
        return find_strategy_for_objects(s_containing_class.space, vars)
    else:
        return ListStorageStrategy.singleton
<commit_msg>Fixes<commit_after>
import sys
from spyvm import model, shadow, constants
from rpython.rlib.objectmodel import import_from_mixin
from rpython.rlib.rfloat import string_to_float

# Disables all optimized strategies, for debugging.
only_list_storage = False

class AbstractStorageStrategy(object):
    _immutable_fields_ = []
    _attrs_ = []
    _settled_ = True
    strategy_tag = 'abstract'
    needs_objspace = False

    def set_initial_storage(self, space, w_obj, size):
        raise NotImplementedError("Abstract base class")
    def set_storage_for_list(self, space, w_obj, collection):
        raise NotImplementedError("Abstract base class")
    def set_storage_copied_from(self, space, w_obj, w_source_obj, reuse_storage=False):
        raise NotImplementedError("Abstract base class")

    def store(self, space, w_obj, n0, w_val):
        if self.can_contain(space, w_val):
            return self.do_store(space, w_obj, n0, w_val)
        new_strategy = find_strategy_for_object(space, w_val)
        return w_obj.store_with_new_strategy(space, new_strategy, n0, w_val)

    def can_contain(self, space, w_val):
        raise NotImplementedError("Abstract base class")
    def fetch(self, space, w_obj, n0):
        raise NotImplementedError("Abstract base class")
    def do_store(self, space, w_obj, n0, w_val):
        raise NotImplementedError("Abstract base class")

class AbstractListStorageStrategy(AbstractStorageStrategy):
    strategy_tag = 'abstract-list'

    def storage(self, w_obj):
        return w_obj.list_storage
    def set_initial_storage(self, space, w_obj, size):
        w_obj.list_storage = self.initial_storage(space, size)
    def set_storage_for_list(self, space, w_obj, collection):
        w_obj.list_storage = self.storage_for_list(space, collection)
    def set_storage_copied_from(self, space, w_obj, w_source_obj, reuse_storage=False):
        w_obj.list_storage = self.copy_storage_from(space, w_source_obj, reuse_storage)

    def initial_storage(self, space, size):
        raise NotImplementedError("Abstract base class")
    def storage_for_list(self, space, collection):
        raise NotImplementedError("Abstract base class")
    def copy_storage_from(self, space, w_obj, reuse_storage):
        old_strategy = w_obj.strategy
        if old_strategy == self and reuse_storage:
            return self.storage(w_obj)
        else:
            # This can be overridden and optimized (reuse_storage flag, less temporary storage)
            return self.storage_for_list(space, w_obj.fetch_all(space))

class AbstractIntStorageStrategy(AbstractStorageStrategy):
    strategy_tag = 'abstract-int'

    def storage(self, w_obj):
        return w_obj.int_storage
    def set_initial_storage(self, space, w_obj, size):
        w_obj.int_storage = self.initial_storage(space, size)
    def set_storage_for_list(self, space, w_obj, collection):
        w_obj.int_storage = self.storage_for_list(space, collection)
    def set_storage_copied_from(self, space, w_obj, w_source_obj, reuse_storage=False):
        w_obj.int_storage = self.copy_storage_from(space, w_source_obj, reuse_storage)

    def initial_storage(self, space, size):
        raise NotImplementedError("Abstract base class")
    def storage_for_list(self, space, collection):
        raise NotImplementedError("Abstract base class")
    def copy_storage_from(self, space, w_obj, reuse_storage):
        old_strategy = w_obj.strategy
        if old_strategy == self and reuse_storage:
            return self.storage(w_obj)
        else:
            # This can be overridden and optimized (reuse_storage flag, less temporary storage)
            return self.storage_for_list(space, w_obj.fetch_all(space))

class SingletonMeta(type):
    def __new__(cls, name, bases, dct):
        result = type.__new__(cls, name, bases, dct)
        result.singleton = result()
        return result

# this is the typical "initial" storage strategy, for when every slot
# in an object is still nil. No storage is allocated.
class AllNilStorageStrategy(AbstractStorageStrategy):
    __metaclass__ = SingletonMeta
    strategy_tag = 'allnil'

    def can_contain(self, space, w_obj):
        return w_obj == model.w_nil
    def fetch(self, space, w_obj, n0):
        return model.w_nil
    def do_store(self, space, w_obj, n0, w_val):
        pass

    def set_initial_storage(self, space, w_obj, size):
        pass
    def set_storage_for_list(self, space, w_obj, collection):
        pass
    def set_storage_copied_from(self, space, w_obj, w_source_obj, reuse_storage=False):
        pass

# This is the regular storage strategy that does not result in any
# optimizations but can handle every case. Applicable for both
# fixed-sized and var-sized objects.
class ListStorageStrategy(AbstractListStorageStrategy):
    __metaclass__ = SingletonMeta
    strategy_tag = 'list'

    def can_contain(self, space, w_val):
        return True
    def fetch(self, space, w_obj, n0):
        return self.storage(w_obj)[n0]
    def do_store(self, space, w_obj, n0, w_val):
        # TODO enable generalization by maintaining a counter of elements that are nil.
        self.storage(w_obj)[n0] = w_val
    def initial_storage(self, space, size):
        return [model.w_nil] * size
    def storage_for_list(self, space, collection):
        return [x for x in collection]
    def copy_storage_from(self, space, w_obj, reuse_storage=False):
        length = w_obj.basic_size()
        return [w_obj.strategy.fetch(space, w_obj, i) for i in range(length)]

class AbstractValueOrNilStorageStrategy(AbstractIntStorageStrategy):
    needs_objspace = True
    strategy_tag = 'abstract-valueOrNil'

    def can_contain(self, space, w_val):
        return w_val == model.w_nil or (isinstance(w_val, self.wrapper_class) and self.unwrap(space, w_val) != self.nil_value)

    def fetch(self, space, w_obj, n0):
        val = self.storage(w_obj)[n0]
        if val == self.nil_value:
            return space.w_nil
        else:
            return self.wrap(space, val)

    def do_store(self, space, w_obj, n0, w_val):
        store = self.storage(w_obj)
        if w_val == model.w_nil:
                store[n0] = self.nil_value
        else:
            store[n0] = self.unwrap(space, w_val)

    def initial_storage(self, space, size):
        return [self.nil_value] * size

    def storage_for_list(self, space, collection):
        length = len(collection)
        store = self.initial_storage(space, length)
        for i in range(length):
            if collection[i] != model.w_nil:
                store[i] = self.unwrap(space, collection[i])
        return store

class SmallIntegerOrNilStorageStrategy(AbstractValueOrNilStorageStrategy):
    __metaclass__ = SingletonMeta
    strategy_tag = 'float-orNil'
    nil_value = constants.MAXINT
    wrapper_class = model.W_SmallInteger
    def wrap(self, space, val): return space.wrap_int(val)
    def unwrap(self, space, w_val): return space.unwrap_int(w_val)

class FloatOrNilStorageStrategy(AbstractValueOrNilStorageStrategy):
    __metaclass__ = SingletonMeta
    strategy_tag = 'smallint-orNil'
    nil_value = string_to_float("-nan")
    wrapper_class = model.W_Float
    def wrap(self, space, val): return space.wrap_float(val)
    def unwrap(self, space, w_val): return space.unwrap_float(w_val)

def find_strategy_for_object(space, var):
    return find_strategy_for_objects(space, [var])

def find_strategy_for_objects(space, vars):
    if only_list_storage:
        ListStorageStrategy.singleton

    specialized_strategies = 3
    all_nil_can_handle = True
    small_int_can_handle = True
    float_can_handle = True
    for w_obj in vars:
        if all_nil_can_handle and not AllNilStorageStrategy.singleton.can_contain(space, w_obj):
            all_nil_can_handle = False
            specialized_strategies = specialized_strategies - 1
        if small_int_can_handle and not SmallIntegerOrNilStorageStrategy.singleton.can_contain(space, w_obj):
            small_int_can_handle = False
            specialized_strategies = specialized_strategies - 1
        if float_can_handle and not FloatOrNilStorageStrategy.singleton.can_contain(space, w_obj):
            float_can_handle = False
            specialized_strategies = specialized_strategies - 1

        if specialized_strategies <= 0:
            return ListStorageStrategy.singleton

    if all_nil_can_handle:
        return AllNilStorageStrategy.singleton
    if small_int_can_handle:
        return SmallIntegerOrNilStorageStrategy.singleton
    if float_can_handle:
        return FloatOrNilStorageStrategy.singleton

    # If this happens, please look for a bug in the code above.
    assert False, "No strategy could be found for list %r" % vars

def empty_strategy(s_containing_class):
    if s_containing_class is None:
        # This is a weird and rare special case for w_nil
        return ListStorageStrategy.singleton
    if not s_containing_class.isvariable() or only_list_storage:
        return ListStorageStrategy.singleton

    # A newly allocated object contains only nils.
    return AllNilStorageStrategy.singleton

def strategy_for_list(s_containing_class, vars):
    if s_containing_class is None:
        # This is a weird and rare special case for w_nil
        return ListStorageStrategy.singleton
    try:
        is_variable = s_containing_class.isvariable()
    except AttributeError:
        # TODO - This happens during bootstrapping phase, when filling in generic objects.
        # Ths class object shadows are not yet synchronized.
        return ListStorageStrategy.singleton

    if is_variable:
        return find_strategy_for_objects(s_containing_class.space, vars)
    else:
        return ListStorageStrategy.singleton
<|endoftext|>


<commit_before># tests: mypy

from abc import ABCMeta, abstractproperty
from typing import List, Union
import tensorflow as tf

from neuralmonkey.encoders.attentive import Attentive
from neuralmonkey.checking import assert_shape
from neuralmonkey.nn.projection import linear

class EncoderWrapper(Attentive):

    def __init__(self, name, encoders, multi_attention_type, attention_state_size, use_sentinels=False):
        self.name = name
        self.encoders = encoders
        self._multi_attention_type = multi_attention_type
        self._attention_state_size = attention_state_size
        self._use_sentinels = use_sentinels

        self.encoded = tf.concat(1, [e.encoded for e in encoders])

    def create_attention_object(self, runtime=False):
        return self._multi_attention_type(
            [e._attention_tensor for e in self.encoders],
            [e._attention_mask for e in self.encoders],
            self._attention_state_size,
            "attention_{}".format(self.name),
            use_sentinels=self._use_sentinels)

    def feed_dict(*args, **kwargs):
        return {}

    @property
    def _attention_tensor(self):
        raise NotImplementedError("Encoder wrapper does not contain the"
                                  " attention tensor")

    @property
    def _attention_mask(self):
        raise NotImplementedError("Encoder wrapper does not contain the"
                                  " attention mask")

class MultiAttention(metaclass=ABCMeta):

    def __init__(self,
                 encoders_tensors: List[tf.Tensor],
                 encoders_masks: List[tf.Tensor],
                 state_size: int,
                 scope: Union[tf.VariableScope, str],
                 **kwargs) -> None:
        self._encoders_tensors = encoders_tensors
        self._encoders_masks = encoders_masks
        self._state_size = state_size
        self._scope = scope
        self.attentions_in_time = []

        for e_m in self._encoders_masks:
            assert_shape(e_m, [None, -1])

        for e_t in self._encoders_tensors:
            assert_shape(e_t, [None, -1, -1])

    @abstractproperty
    def attention(self, decoder_state, decoder_input):
        raise NotImplementedError("Abstract method")

    @property
    def attn_size(self):
        return self._state_size

class FlatMultiAttention(MultiAttention):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        if "use_sentinels" in kwargs:
            self._use_sentinels = kwargs.get("use_sentinels")

        with tf.variable_scope(self._scope):
            self.attn_v = tf.get_variable(
                "attn_v", [1, 1, self._state_size],
                initializer=tf.random_normal_initializer(stddev=.001))

            self.encoder_state_projections = []
            self.encoder_attn_biases = []

            for i, encoder_tensor in enumerate(self._encoders_tensors):
                encoder_state_size = encoder_tensor.get_shape()[2].value
                encoder_tensor_shape = tf.shape(encoder_tensor)

                proj_matrix = tf.get_variable(
                    "proj_matrix_{}".format(i),
                    [encoder_state_size, self._state_size],
                    initializer=tf.random_normal_initializer(stddev=0.001))

                proj_bias = tf.get_variable(
                    "proj_bias_{}".format(i),
                    initializer=tf.zeros_initializer([self._state_size]))

                encoder_tensor_2d = tf.reshape(
                    encoder_tensor, [-1, encoder_state_size])

                projected_2d = tf.matmul(
                    encoder_tensor_2d, proj_matrix) + proj_bias
                assert_shape(projected_2d, [None, self._state_size])

                projection = tf.reshape(projected_2d, [encoder_tensor_shape[0],
                                                       encoder_tensor_shape[1],
                                                       self._state_size])

                self.encoder_state_projections.append(projection)

                attn_bias = tf.get_variable(
                    "attn_bias_{}".format(i), [],
                    initializer=tf.constant_initializer(0.0))

                self.encoder_attn_biases.append(attn_bias)

            self.masks_concat = tf.concat(1, self._encoders_masks)

    def attention(self, decoder_state, decoder_prev_state, decoder_input):

        with tf.variable_scope(self._scope):
            projected_state = linear(decoder_state, self._state_size)
            projected_state = tf.expand_dims(projected_state, 1)

            assert_shape(projected_state, [None, 1, self._state_size])

            logits = []

            for proj, bias in zip(self.encoder_state_projections,
                                  self.encoder_attn_biases):

                logits.append(tf.reduce_sum(
                    self.attn_v * tf.tanh(projected_state + proj), [2]) + bias)

            if self._use_sentinels:
                sentinel_value = self._sentinel(decoder_state,
                                                decoder_prev_state,
                                                decoder_input)

                sentinel_bias = tf.get_variable(
                    "sentinel_bias", [],
                    initializer=tf.constant_initializer(0.0))

                projected_sentinel = linear(sentinel_value, self._state_size,
                                            scope="sentinel_projection")
                sentinel_logit = tf.reduce_sum(
                    self.attn_v * tf.tanh(projected_state
                                          + projected_sentinel),
                    [1]) + sentinel_bias

                logits.append(tf.expand_dims(sentinel_logit, 1))

            logits_concat = tf.concat(1, logits)
            softmax_concat = tf.nn.softmax(logits_concat) * self.masks_concat
            norm = tf.reduce_sum(softmax_concat, 1, keep_dims=True) + 1e-8
            attentions = softmax_concat / norm

            self.attentions_in_time.append(attentions)

            projections_concat = tf.concat(1, self.encoder_state_projections)

            contexts = tf.reduce_sum(
                tf.expand_dims(attentions, 2) * projections_concat, [1])

            return contexts

    def _sentinel(self, state, prev_state, input_):

        with tf.variable_scope("sentinel"):

            decoder_state_size = state.get_shape()[-1].value
            concatenation = tf.concat(1, [prev_state, input_])

            gate = tf.nn.sigmoid(linear(concatenation, decoder_state_size))
            sentinel_value = gate * state

            return sentinel_value

class HierarchicalMultiAttention(MultiAttention):

    def __init__(self, *args, **kwargs) -> None:
        super().__init__(*args, **kwargs)
<commit_msg>make sentinel work<commit_after># tests: mypy

from abc import ABCMeta, abstractproperty
from typing import List, Union
import tensorflow as tf

from neuralmonkey.encoders.attentive import Attentive
from neuralmonkey.checking import assert_shape
from neuralmonkey.nn.projection import linear

class EncoderWrapper(Attentive):

    def __init__(self, name, encoders, multi_attention_type, attention_state_size, use_sentinels=False):
        self.name = name
        self.encoders = encoders
        self._multi_attention_type = multi_attention_type
        self._attention_state_size = attention_state_size
        self._use_sentinels = use_sentinels

        self.encoded = tf.concat(1, [e.encoded for e in encoders])

    def create_attention_object(self, runtime=False):
        return self._multi_attention_type(
            [e._attention_tensor for e in self.encoders],
            [e._attention_mask for e in self.encoders],
            self._attention_state_size,
            "attention_{}".format(self.name),
            use_sentinels=self._use_sentinels)

    def feed_dict(*args, **kwargs):
        return {}

    @property
    def _attention_tensor(self):
        raise NotImplementedError("Encoder wrapper does not contain the"
                                  " attention tensor")

    @property
    def _attention_mask(self):
        raise NotImplementedError("Encoder wrapper does not contain the"
                                  " attention mask")

class MultiAttention(metaclass=ABCMeta):

    def __init__(self,
                 encoders_tensors: List[tf.Tensor],
                 encoders_masks: List[tf.Tensor],
                 state_size: int,
                 scope: Union[tf.VariableScope, str],
                 **kwargs) -> None:
        self._encoders_tensors = encoders_tensors
        self._encoders_masks = encoders_masks
        self._state_size = state_size
        self._scope = scope
        self.attentions_in_time = []

        for e_m in self._encoders_masks:
            assert_shape(e_m, [None, -1])

        for e_t in self._encoders_tensors:
            assert_shape(e_t, [None, -1, -1])

    @abstractproperty
    def attention(self, decoder_state, decoder_input):
        raise NotImplementedError("Abstract method")

    @property
    def attn_size(self):
        return self._state_size

class FlatMultiAttention(MultiAttention):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        if "use_sentinels" in kwargs:
            self._use_sentinels = kwargs.get("use_sentinels")

        with tf.variable_scope(self._scope):
            self.attn_v = tf.get_variable(
                "attn_v", [1, 1, self._state_size],
                initializer=tf.random_normal_initializer(stddev=.001))

            self.encoder_state_projections = []
            self.encoder_attn_biases = []

            for i, encoder_tensor in enumerate(self._encoders_tensors):
                encoder_state_size = encoder_tensor.get_shape()[2].value
                encoder_tensor_shape = tf.shape(encoder_tensor)

                proj_matrix = tf.get_variable(
                    "proj_matrix_{}".format(i),
                    [encoder_state_size, self._state_size],
                    initializer=tf.random_normal_initializer(stddev=0.001))

                proj_bias = tf.get_variable(
                    "proj_bias_{}".format(i),
                    initializer=tf.zeros_initializer([self._state_size]))

                encoder_tensor_2d = tf.reshape(
                    encoder_tensor, [-1, encoder_state_size])

                projected_2d = tf.matmul(
                    encoder_tensor_2d, proj_matrix) + proj_bias
                assert_shape(projected_2d, [None, self._state_size])

                projection = tf.reshape(projected_2d, [encoder_tensor_shape[0],
                                                       encoder_tensor_shape[1],
                                                       self._state_size])

                self.encoder_state_projections.append(projection)

                attn_bias = tf.get_variable(
                    "attn_bias_{}".format(i), [],
                    initializer=tf.constant_initializer(0.0))

                self.encoder_attn_biases.append(attn_bias)

            if self._use_sentinels:
                self._encoders_masks.append(
                    tf.ones([tf.shape(self._encoders_masks[0])[0], 1]))

            self.masks_concat = tf.concat(1, self._encoders_masks)

    def attention(self, decoder_state, decoder_prev_state, decoder_input):

        with tf.variable_scope(self._scope):
            projected_state = linear(decoder_state, self._state_size)
            projected_state = tf.expand_dims(projected_state, 1)

            assert_shape(projected_state, [None, 1, self._state_size])

            logits = []

            for proj, bias in zip(self.encoder_state_projections,
                                  self.encoder_attn_biases):

                logits.append(tf.reduce_sum(
                    self.attn_v * tf.tanh(projected_state + proj), [2]) + bias)

            if self._use_sentinels:
                sentinel_value = self._sentinel(decoder_state,
                                                decoder_prev_state,
                                                decoder_input)

                sentinel_bias = tf.get_variable(
                    "sentinel_bias", [],
                    initializer=tf.constant_initializer(0.0))

                projected_sentinel = tf.expand_dims(
                    linear(sentinel_value, self._state_size,
                           scope="sentinel_projection"), 1)
                sentinel_logit = tf.reduce_sum(
                    self.attn_v *
                    tf.tanh(projected_state + projected_sentinel),
                    [2]) + sentinel_bias
                assert_shape(sentinel_logit, [None, 1])

                logits.append(sentinel_logit)

            logits_concat = tf.concat(1, logits)
            softmax_concat = tf.nn.softmax(logits_concat) * self.masks_concat
            norm = tf.reduce_sum(softmax_concat, 1, keep_dims=True) + 1e-8
            attentions = softmax_concat / norm

            self.attentions_in_time.append(attentions)

            projections_concat = tf.concat(
                1, self.encoder_state_projections +
                [projected_sentinel] if self._use_sentinels else [])

            contexts = tf.reduce_sum(
                tf.expand_dims(attentions, 2) * projections_concat, [1])

            return contexts

    def _sentinel(self, state, prev_state, input_):

        with tf.variable_scope("sentinel"):

            decoder_state_size = state.get_shape()[-1].value
            concatenation = tf.concat(1, [prev_state, input_])

            gate = tf.nn.sigmoid(linear(concatenation, decoder_state_size))
            sentinel_value = gate * state

            assert_shape(sentinel_value, [None, decoder_state_size])

            return sentinel_value

class HierarchicalMultiAttention(MultiAttention):

    def __init__(self, *args, **kwargs) -> None:
        super().__init__(*args, **kwargs)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
@author: Sebastian Flennerhag
@date: 12/01/2017
"""

from mlens.ensemble import StackingEnsemble
from mlens.ensemble._setup import name_estimators, name_base, _check_names
from mlens.ensemble._clone import _clone_base_estimators
from mlens.ensemble._clone import _clone_preprocess_cases
from mlens.utils.utils import name_columns
from mlens.metrics import rmse
from mlens.metrics.metrics import rmse_scoring
from sklearn.linear_model import Lasso
from sklearn.neighbors import KNeighborsRegressor
from sklearn.svm import SVR
from sklearn.ensemble import RandomForestRegressor
from sklearn.preprocessing import StandardScaler, MinMaxScaler
from sklearn.model_selection import RandomizedSearchCV
from scipy.stats import uniform, randint
import numpy as np
from pandas import DataFrame

# training data
np.random.seed(100)
X = np.random.random((1000, 10))

# noisy output, y = x0 * x1 + x2^2 + x3 - x4^(1/4) + e
y = X[:, 0] * X[:, 1] + X[:, 2] ** 2 + X[:, 3] - X[:, 4] ** (1 / 4)

# Change scales
X[:, 0] *= 10
X[:, 1] += 10
X[:, 2] *= 5
X[:, 3] *= 3
X[:, 4] /= 10

# meta estimator
meta = SVR()

# Create ensemble with preprocessing pipelines
base_pipelines = {'sc':
                  ([StandardScaler()],
                   [('ls', Lasso()), ('kn', KNeighborsRegressor())]),
                  'mm':
                  ([MinMaxScaler()], [SVR()]),
                  'np':
                  ([], [('rf', RandomForestRegressor(random_state=100))])}

ensemble = StackingEnsemble(meta, base_pipelines, folds=10, shuffle=False,
                            scorer=rmse._score_func, n_jobs=1,
                            random_state=100)

params = {'sc-ls__alpha': uniform(0.0005, 0.005),
          'np-rf__max_depth': randint(2, 6),
          'np-rf__max_features': randint(2, 5),
          'np-rf__min_samples_leaf': randint(5, 12),
          'sc-kn__n_neighbors': randint(6, 12),
          'mm-svr__C': uniform(10, 20),
          'meta-svr__C': uniform(10, 20)}

# Ensembles without preprocessing
ens1 = StackingEnsemble(Lasso(alpha=0.001, random_state=100),
                        [('svr', SVR()),
                         ('rf', RandomForestRegressor(random_state=100))],
                        random_state=100, n_jobs=-1)

ens2 = StackingEnsemble(Lasso(alpha=0.001, random_state=100),
                        [SVR(), RandomForestRegressor(random_state=100)],
                        random_state=100, n_jobs=-1)

ens3 = StackingEnsemble(Lasso(alpha=0.001, random_state=100),
                        {'no_prep':
                         ([], [SVR(),
                               RandomForestRegressor(random_state=100)])},
                        random_state=100, n_jobs=-1)

def test_naming():

    named_meta = name_estimators([meta], 'meta-')
    named_base = name_base(base_pipelines)

    assert isinstance(named_meta, dict)
    assert isinstance(named_meta['meta-svr'], SVR)
    assert isinstance(named_base, dict)
    assert len(named_base) == 6

def test_check_names():

    preprocess = [(case, _check_names(p[0])) for case, p in
                  base_pipelines.items()]

    base_estimators = [(case, _check_names(p[1])) for case, p in
                       base_pipelines.items()]

    assert isinstance(base_estimators, list)
    assert isinstance(preprocess, list)
    assert len(base_estimators) == 3
    assert len(preprocess) == 3
    assert isinstance(base_estimators[0], tuple)
    assert isinstance(preprocess[0], tuple)

def test_clone():

    preprocess = [(case, _check_names(p[0])) for case, p in
                  base_pipelines.items()]
    base_estimators = [(case, _check_names(p[1])) for case, p in
                       base_pipelines.items()]

    base_ = _clone_base_estimators(base_estimators)
    preprocess_ = _clone_preprocess_cases(preprocess)
    base_columns_ = name_columns(base_)

    assert isinstance(preprocess_, list)
    assert isinstance(preprocess_[0], tuple)
    assert isinstance(preprocess_[0][1], list)
    assert isinstance(base_, dict)
    assert isinstance(base_['mm'], list)
    assert isinstance(base_['mm'][0], tuple)
    assert isinstance(base_columns_, list)
    assert len(base_columns_) == 4

def test_no_preprocess_ensemble():
    ens1.fit(X, y)
    ens2.fit(X, y)
    ens3.fit(X, y)

    p1 = ens1.predict(X)
    p2 = ens2.predict(X)
    p3 = ens3.predict(X)

    assert (p1 == p2).all()
    assert (p1 == p3).all()

    # Test that DataFrames runs through
    ens1.set_params(**{'as_df': True, 'scorer': rmse._score_func})
    ens1.fit(DataFrame(X), y)

    assert str(ens1.scores_['rf'])[:16] == '0.171221793126'
    assert str(ens1.scores_['svr'])[:16] == '0.171500953229'

def test_preprocess_ensemble():

    ensemble.set_params(**{'np-rf__min_samples_leaf': 9,
                           'meta-svr__C': 16.626146983723014,
                           'sc-kn__n_neighbors': 9,
                           'np-rf__max_features': 4,
                           'mm-svr__C': 11.834807428701293,
                           'sc-ls__alpha': 0.0014284293508642438,
                           'np-rf__max_depth': 4,
                           'n_jobs': -1,
                           'verbose': 0,
                           'shuffle': False,
                           'random_state': 100})

    ensemble.fit(X[:900], y[:900])

    score1 = rmse(ensemble, X[900:], y[900:])
    score2 = rmse_scoring(y[900:], ensemble.predict(X[900:]))

    assert score1 == -score2
    assert ensemble.get_params()['n_jobs'] == -1
    assert str(score1)[:16] == '-0.0522364178463'

def test_grid_search():

    ensemble.set_params(**{'n_jobs': 1})

    grid = RandomizedSearchCV(ensemble, param_distributions=params,
                              n_iter=2, cv=2, scoring=rmse,
                              n_jobs=-1, random_state=100)
    grid.fit(X, y)

    assert str(grid.best_score_)[:16] == '-0.0626352824626'
<commit_msg>made test fit verbose<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
@author: Sebastian Flennerhag
@date: 12/01/2017
"""

from mlens.ensemble import StackingEnsemble
from mlens.ensemble._setup import name_estimators, name_base, _check_names
from mlens.ensemble._clone import _clone_base_estimators
from mlens.ensemble._clone import _clone_preprocess_cases
from mlens.utils.utils import name_columns
from mlens.metrics import rmse
from mlens.metrics.metrics import rmse_scoring
from sklearn.linear_model import Lasso
from sklearn.neighbors import KNeighborsRegressor
from sklearn.svm import SVR
from sklearn.ensemble import RandomForestRegressor
from sklearn.preprocessing import StandardScaler, MinMaxScaler
from sklearn.model_selection import RandomizedSearchCV
from scipy.stats import uniform, randint
import numpy as np
from pandas import DataFrame

# training data
np.random.seed(100)
X = np.random.random((1000, 10))

# noisy output, y = x0 * x1 + x2^2 + x3 - x4^(1/4) + e
y = X[:, 0] * X[:, 1] + X[:, 2] ** 2 + X[:, 3] - X[:, 4] ** (1 / 4)

# Change scales
X[:, 0] *= 10
X[:, 1] += 10
X[:, 2] *= 5
X[:, 3] *= 3
X[:, 4] /= 10

# meta estimator
meta = SVR()

# Create ensemble with preprocessing pipelines
base_pipelines = {'sc':
                  ([StandardScaler()],
                   [('ls', Lasso()), ('kn', KNeighborsRegressor())]),
                  'mm':
                  ([MinMaxScaler()], [SVR()]),
                  'np':
                  ([], [('rf', RandomForestRegressor(random_state=100))])}

ensemble = StackingEnsemble(meta, base_pipelines, folds=10, shuffle=False,
                            scorer=rmse._score_func, n_jobs=1,
                            random_state=100)

params = {'sc-ls__alpha': uniform(0.0005, 0.005),
          'np-rf__max_depth': randint(2, 6),
          'np-rf__max_features': randint(2, 5),
          'np-rf__min_samples_leaf': randint(5, 12),
          'sc-kn__n_neighbors': randint(6, 12),
          'mm-svr__C': uniform(10, 20),
          'meta-svr__C': uniform(10, 20)}

# Ensembles without preprocessing
ens1 = StackingEnsemble(Lasso(alpha=0.001, random_state=100),
                        [('svr', SVR()),
                         ('rf', RandomForestRegressor(random_state=100))],
                        random_state=100, n_jobs=-1)

ens2 = StackingEnsemble(Lasso(alpha=0.001, random_state=100),
                        [SVR(), RandomForestRegressor(random_state=100)],
                        random_state=100, n_jobs=-1)

ens3 = StackingEnsemble(Lasso(alpha=0.001, random_state=100),
                        {'no_prep':
                         ([], [SVR(),
                               RandomForestRegressor(random_state=100)])},
                        random_state=100, n_jobs=-1)

def test_naming():

    named_meta = name_estimators([meta], 'meta-')
    named_base = name_base(base_pipelines)

    assert isinstance(named_meta, dict)
    assert isinstance(named_meta['meta-svr'], SVR)
    assert isinstance(named_base, dict)
    assert len(named_base) == 6

def test_check_names():

    preprocess = [(case, _check_names(p[0])) for case, p in
                  base_pipelines.items()]

    base_estimators = [(case, _check_names(p[1])) for case, p in
                       base_pipelines.items()]

    assert isinstance(base_estimators, list)
    assert isinstance(preprocess, list)
    assert len(base_estimators) == 3
    assert len(preprocess) == 3
    assert isinstance(base_estimators[0], tuple)
    assert isinstance(preprocess[0], tuple)

def test_clone():

    preprocess = [(case, _check_names(p[0])) for case, p in
                  base_pipelines.items()]
    base_estimators = [(case, _check_names(p[1])) for case, p in
                       base_pipelines.items()]

    base_ = _clone_base_estimators(base_estimators)
    preprocess_ = _clone_preprocess_cases(preprocess)
    base_columns_ = name_columns(base_)

    assert isinstance(preprocess_, list)
    assert isinstance(preprocess_[0], tuple)
    assert isinstance(preprocess_[0][1], list)
    assert isinstance(base_, dict)
    assert isinstance(base_['mm'], list)
    assert isinstance(base_['mm'][0], tuple)
    assert isinstance(base_columns_, list)
    assert len(base_columns_) == 4

def test_no_preprocess_ensemble():
    ens1.fit(X, y)
    ens2.fit(X, y)
    ens3.fit(X, y)

    p1 = ens1.predict(X)
    p2 = ens2.predict(X)
    p3 = ens3.predict(X)

    assert (p1 == p2).all()
    assert (p1 == p3).all()

    # Test that DataFrames runs through
    ens1.set_params(**{'as_df': True, 'scorer': rmse._score_func})
    ens1.fit(DataFrame(X), y)

    assert str(ens1.scores_['rf'])[:16] == '0.171221793126'
    assert str(ens1.scores_['svr'])[:16] == '0.171500953229'

def test_preprocess_ensemble():

    ensemble.set_params(**{'np-rf__min_samples_leaf': 9,
                           'meta-svr__C': 16.626146983723014,
                           'sc-kn__n_neighbors': 9,
                           'np-rf__max_features': 4,
                           'mm-svr__C': 11.834807428701293,
                           'sc-ls__alpha': 0.0014284293508642438,
                           'np-rf__max_depth': 4,
                           'n_jobs': -1,
                           'verbose': 2,
                           'shuffle': False,
                           'random_state': 100})

    ensemble.fit(X[:900], y[:900])

    score1 = rmse(ensemble, X[900:], y[900:])
    score2 = rmse_scoring(y[900:], ensemble.predict(X[900:]))

    assert score1 == -score2
    assert ensemble.get_params()['n_jobs'] == -1
    assert str(score1)[:16] == '-0.0522364178463'

def test_grid_search():

    ensemble.set_params(**{'n_jobs': 1})

    grid = RandomizedSearchCV(ensemble, param_distributions=params,
                              n_iter=2, cv=2, scoring=rmse,
                              n_jobs=-1, random_state=100)
    grid.fit(X, y)

    assert str(grid.best_score_)[:16] == '-0.0626352824626'
<|endoftext|>


<commit_before>from interface.definitions import MCDAProblem, Alternative, Criterion
import numpy as np
from utils.matrice_tools import apply_on_each_index, apply_on_each_element

class MCDAProxy(object):
    def __init__(self, problem):
        self.problem = problem

    def apply_on_each_crit(self, func):
        x_dim = len(self.problem.alternativesList)
        y_dim = len(self.problem.alternativesList[0].criteriaList) if x_dim > 0 else 0

        result = np.zeros((x_dim, y_dim))

        for x in range(x_dim):
            for y in range(y_dim):
                result[x, y] = func(self.problem.alternativesList[x].criteriaList[y])

        return result

    def alt_count(self):
        return len(self.problem.alternativesList)

    def crit_count(self):
        return len(self.problem.alternativesList[0].criteriaList) if self.alt_count() > 0 else 0

    def criterion(self, i, j):
        if i >= self.alt_count() or j >= self.crit_count():
            raise ValueError('index out of bounds')

        return self.problem.alternativesList[i].criteriaList[j]

    def weight(self, i):
        return self.problem.criteria_weights[i]

    def value(self, i, j):
        return self.criterion(i, j).value

    def direction(self, i, j):
        return self.criterion(i, j).minmax

    def p(self, i):
        return self.problem.criteria_preference[i]

    def q(self, i):
        return self.problem.criteria_indifference[i]

    def veto(self, i):
        return self.problem.veto_thresholds[i]

    def alternative(self, i):
        return self.problem.alternativesList[i]

def electre_is(problem):
    assert isinstance(problem, MCDAProblem), "This metod works only with MCDAproblems"

    pr = MCDAProxy(problem)

    def js(a, b):
        def predicate(j):
            return pr.value(a, j) >= pr.value(b, j) - pr.q(j)

        indexes = filter(predicate, range(pr.crit_count()))
        return list(indexes)

    def jq(a, b):
        def predicate(j):
            return pr.value(b, j) - pr.p(j) <= pr.value(a, j) < pr.value(b, j) - pr.q(j)

        indexes = filter(predicate, range(pr.crit_count()))
        return list(indexes)

    def jp(a, b):
        def predicate(j):
            return pr.value(b, j) > pr.value(a, j) + pr.p(j)

        indexes = filter(predicate, range(pr.crit_count()))
        return list(indexes)

    def fi(a, b, j):
        nom = pr.p(j) + pr.value(a, j) - pr.value(b, j)
        denom = pr.p(j) - pr.q(j)
        return float(nom)/denom

    def concordance(a, b):
        if len(jp(a, b)) == pr.crit_count():
            return 0
        if len(js(a, b)) == pr.crit_count():
            return 1

        part1 = sum(pr.weight(j) for j in js(a, b))

        part2 = sum(fi(a, b, j) * pr.weight(j) for j in jq(a, b))

        norm = sum(pr.weight(j) for j in range(pr.crit_count()))

        return float(part1 + part2) / norm

    s = 0.60
    dim = pr.alt_count()
    concordance_mtx = apply_on_each_index(concordance, (dim, dim))

    def condition1(val):
        return 0.5 < val < s

    def filter_by_condition(value):
        if condition1(value):
            return value

        return 0

    filtered_matrix = apply_on_each_element(filter_by_condition, concordance_mtx)
    # print(filtered_matrix)

    import itertools as it

    # failed_dict = {(a, b): [] for a, b in it.permutations(range(pr.alt_count()), 2)}

    def condition2(a, b, j):
        def w(x, y, i):
            norm = sum(pr.weight(jj) for jj in range(pr.crit_count()))
            nom = 1 - concordance_mtx[x, y] - pr.weight(i) / norm
            denom = 1 - s - pr.weight(i) / norm
            return float(nom) / denom

        # if not (pr.value(a, j) + pr.veto(j) >= pr.value(b, j) + pr.q(j) + w(a, b, j)):
        #     failed_dict[(a, b)].append(j)

        return pr.value(a, j) + pr.veto(j) >= pr.value(b, j) + pr.q(j) + w(a, b, j)

    def filter2(a, b):
        return all(condition2(a, b, j) for j in range(pr.crit_count()))

    filtered2_matrix = apply_on_each_index(filter2, (dim, dim))

    result_matrix = filtered_matrix * filtered2_matrix

    non_zero = result_matrix.nonzero()
    graph_dict = {k: [] for k in non_zero[0]}
    for i, k in enumerate(non_zero[0]):
        graph_dict[k].append(non_zero[1][i])

    filtered = list(filter(lambda x: x not in non_zero[1], non_zero[0]))
    best_alternatives = [pr.alternative(i) for i in filtered]

    return best_alternatives, graph_dict

def main():
    problem = MCDAProblem()

    path_root = '../input_files/electre_test_input/%s'

    problem.read_performance_table(path_root % 'testAlternatives.csv', delimiter=',')
    problem.read_criteria_weights(path_root % 'weights.csv', delimiter=',')
    problem.read_veto_thresholds(path_root % 'vetos.csv', delimiter=',')
    problem.read_preference(path_root % 'preference.csv', delimiter=',')
    problem.read_indifference(path_root % 'indifference.csv', delimiter=',')

    best_alternatives, graph = electre_is(problem)

    if len(best_alternatives):
        print('Najlepsze alternatywy')
        print(', '.join(alt.name for alt in best_alternatives))
    else:
        print('Dla zadanych parametrow brak rozwiazan')
        print('struktura grafu:')
        print(graph)

if __name__ == '__main__':
    main()
<commit_msg>unused import removed<commit_after>from interface.definitions import MCDAProblem, Alternative, Criterion
import numpy as np
from utils.matrice_tools import apply_on_each_index, apply_on_each_element

class MCDAProxy(object):
    def __init__(self, problem):
        self.problem = problem

    def apply_on_each_crit(self, func):
        x_dim = len(self.problem.alternativesList)
        y_dim = len(self.problem.alternativesList[0].criteriaList) if x_dim > 0 else 0

        result = np.zeros((x_dim, y_dim))

        for x in range(x_dim):
            for y in range(y_dim):
                result[x, y] = func(self.problem.alternativesList[x].criteriaList[y])

        return result

    def alt_count(self):
        return len(self.problem.alternativesList)

    def crit_count(self):
        return len(self.problem.alternativesList[0].criteriaList) if self.alt_count() > 0 else 0

    def criterion(self, i, j):
        if i >= self.alt_count() or j >= self.crit_count():
            raise ValueError('index out of bounds')

        return self.problem.alternativesList[i].criteriaList[j]

    def weight(self, i):
        return self.problem.criteria_weights[i]

    def value(self, i, j):
        return self.criterion(i, j).value

    def direction(self, i, j):
        return self.criterion(i, j).minmax

    def p(self, i):
        return self.problem.criteria_preference[i]

    def q(self, i):
        return self.problem.criteria_indifference[i]

    def veto(self, i):
        return self.problem.veto_thresholds[i]

    def alternative(self, i):
        return self.problem.alternativesList[i]

def electre_is(problem):
    assert isinstance(problem, MCDAProblem), "This metod works only with MCDAproblems"

    pr = MCDAProxy(problem)

    def js(a, b):
        def predicate(j):
            return pr.value(a, j) >= pr.value(b, j) - pr.q(j)

        indexes = filter(predicate, range(pr.crit_count()))
        return list(indexes)

    def jq(a, b):
        def predicate(j):
            return pr.value(b, j) - pr.p(j) <= pr.value(a, j) < pr.value(b, j) - pr.q(j)

        indexes = filter(predicate, range(pr.crit_count()))
        return list(indexes)

    def jp(a, b):
        def predicate(j):
            return pr.value(b, j) > pr.value(a, j) + pr.p(j)

        indexes = filter(predicate, range(pr.crit_count()))
        return list(indexes)

    def fi(a, b, j):
        nom = pr.p(j) + pr.value(a, j) - pr.value(b, j)
        denom = pr.p(j) - pr.q(j)
        return float(nom)/denom

    def concordance(a, b):
        if len(jp(a, b)) == pr.crit_count():
            return 0
        if len(js(a, b)) == pr.crit_count():
            return 1

        part1 = sum(pr.weight(j) for j in js(a, b))

        part2 = sum(fi(a, b, j) * pr.weight(j) for j in jq(a, b))

        norm = sum(pr.weight(j) for j in range(pr.crit_count()))

        return float(part1 + part2) / norm

    s = 0.60
    dim = pr.alt_count()
    concordance_mtx = apply_on_each_index(concordance, (dim, dim))

    def condition1(val):
        return 0.5 < val < s

    def filter_by_condition(value):
        if condition1(value):
            return value

        return 0

    filtered_matrix = apply_on_each_element(filter_by_condition, concordance_mtx)
    # print(filtered_matrix)

    # import itertools as it

    # failed_dict = {(a, b): [] for a, b in it.permutations(range(pr.alt_count()), 2)}

    def condition2(a, b, j):
        def w(x, y, i):
            norm = sum(pr.weight(jj) for jj in range(pr.crit_count()))
            nom = 1 - concordance_mtx[x, y] - pr.weight(i) / norm
            denom = 1 - s - pr.weight(i) / norm
            return float(nom) / denom

        # if not (pr.value(a, j) + pr.veto(j) >= pr.value(b, j) + pr.q(j) + w(a, b, j)):
        #     failed_dict[(a, b)].append(j)

        return pr.value(a, j) + pr.veto(j) >= pr.value(b, j) + pr.q(j) + w(a, b, j)

    def filter2(a, b):
        return all(condition2(a, b, j) for j in range(pr.crit_count()))

    filtered2_matrix = apply_on_each_index(filter2, (dim, dim))

    result_matrix = filtered_matrix * filtered2_matrix

    non_zero = result_matrix.nonzero()
    graph_dict = {k: [] for k in non_zero[0]}
    for i, k in enumerate(non_zero[0]):
        graph_dict[k].append(non_zero[1][i])

    filtered = list(filter(lambda x: x not in non_zero[1], non_zero[0]))
    best_alternatives = [pr.alternative(i) for i in filtered]

    return best_alternatives, graph_dict

def main():
    problem = MCDAProblem()

    path_root = '../input_files/electre_test_input/%s'

    problem.read_performance_table(path_root % 'testAlternatives.csv', delimiter=',')
    problem.read_criteria_weights(path_root % 'weights.csv', delimiter=',')
    problem.read_veto_thresholds(path_root % 'vetos.csv', delimiter=',')
    problem.read_preference(path_root % 'preference.csv', delimiter=',')
    problem.read_indifference(path_root % 'indifference.csv', delimiter=',')

    best_alternatives, graph = electre_is(problem)

    if len(best_alternatives):
        print('Najlepsze alternatywy')
        print(', '.join(alt.name for alt in best_alternatives))
    else:
        print('Dla zadanych parametrow brak rozwiazan')
        print('struktura grafu:')
        print(graph)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""Tests for futures.py."""

import unittest

from . import futures

class FutureTests(unittest.TestCase):

    def testInitialState(self):
        f = futures.Future()
        self.assertFalse(f.cancelled())
        self.assertFalse(f.running())
        self.assertFalse(f.done())

    def testInitEventLoopPositional(self):
        # Make sure Future does't accept a positional argument
        self.assertRaises(TypeError, futures.Future, 42)

    def testCancel(self):
        f = futures.Future()
        self.assertTrue(f.cancel())
        self.assertTrue(f.cancelled())
        self.assertFalse(f.running())
        self.assertTrue(f.done())
        self.assertRaises(futures.CancelledError, f.result)
        self.assertRaises(futures.CancelledError, f.exception)
        self.assertRaises(futures.InvalidStateError, f.set_result, None)
        self.assertRaises(futures.InvalidStateError, f.set_exception, None)
        self.assertFalse(f.cancel())

    def testResult(self):
        f = futures.Future()
        f.set_result(42)
        self.assertFalse(f.cancelled())
        self.assertFalse(f.running())
        self.assertTrue(f.done())
        self.assertEqual(f.result(), 42)
        self.assertEqual(f.exception(), None)
        self.assertRaises(futures.InvalidStateError, f.set_result, None)
        self.assertRaises(futures.InvalidStateError, f.set_exception, None)
        self.assertFalse(f.cancel())

    def testException(self):
        exc = RuntimeError()
        f = futures.Future()
        f.set_exception(exc)
        self.assertFalse(f.cancelled())
        self.assertFalse(f.running())
        self.assertTrue(f.done())
        self.assertRaises(RuntimeError, f.result)
        self.assertEqual(f.exception(), exc)
        self.assertRaises(futures.InvalidStateError, f.set_result, None)
        self.assertRaises(futures.InvalidStateError, f.set_exception, None)
        self.assertFalse(f.cancel())

    def testDoneCallbacks(self):
        class MyEventLoop:
            def call_soon(self, fn, future):
                fn(future)

        bag = []
        def make_callback(num):
            def bag_appender(future):
                bag.append(num)
            return bag_appender

        f = futures.Future(event_loop=MyEventLoop())
        f.add_done_callback(make_callback(42))
        f.add_done_callback(make_callback(17))

        self.assertEquals(bag, [])
        f.set_result('foo')
        self.assertEqual(bag, [42, 17])
        self.assertEqual(f.result(), 'foo')

    def testYieldFromTwice(self):
        f = futures.Future()
        def fixture():
            yield 'A'
            x = yield from f
            yield 'B', x
            y = yield from f
            yield 'C', y
        g = fixture()
        self.assertEqual(next(g), 'A')  # yield 'A'.
        self.assertEqual(next(g), f)  # First yield from f.
        f.set_result(42)
        self.assertEqual(next(g), ('B', 42))  # yield 'B', x.
        # The second "yield from f" does not yield f.
        self.assertEqual(next(g), ('C', 42))  # yield 'C', y.

if __name__ == '__main__':
    unittest.main()
<commit_msg>Add more unit tests for Future.(add|remove)_done_callback<commit_after>"""Tests for futures.py."""

import unittest

from . import futures

class FutureTests(unittest.TestCase):

    def testInitialState(self):
        f = futures.Future()
        self.assertFalse(f.cancelled())
        self.assertFalse(f.running())
        self.assertFalse(f.done())

    def testInitEventLoopPositional(self):
        # Make sure Future does't accept a positional argument
        self.assertRaises(TypeError, futures.Future, 42)

    def testCancel(self):
        f = futures.Future()
        self.assertTrue(f.cancel())
        self.assertTrue(f.cancelled())
        self.assertFalse(f.running())
        self.assertTrue(f.done())
        self.assertRaises(futures.CancelledError, f.result)
        self.assertRaises(futures.CancelledError, f.exception)
        self.assertRaises(futures.InvalidStateError, f.set_result, None)
        self.assertRaises(futures.InvalidStateError, f.set_exception, None)
        self.assertFalse(f.cancel())

    def testResult(self):
        f = futures.Future()
        f.set_result(42)
        self.assertFalse(f.cancelled())
        self.assertFalse(f.running())
        self.assertTrue(f.done())
        self.assertEqual(f.result(), 42)
        self.assertEqual(f.exception(), None)
        self.assertRaises(futures.InvalidStateError, f.set_result, None)
        self.assertRaises(futures.InvalidStateError, f.set_exception, None)
        self.assertFalse(f.cancel())

    def testException(self):
        exc = RuntimeError()
        f = futures.Future()
        f.set_exception(exc)
        self.assertFalse(f.cancelled())
        self.assertFalse(f.running())
        self.assertTrue(f.done())
        self.assertRaises(RuntimeError, f.result)
        self.assertEqual(f.exception(), exc)
        self.assertRaises(futures.InvalidStateError, f.set_result, None)
        self.assertRaises(futures.InvalidStateError, f.set_exception, None)
        self.assertFalse(f.cancel())

    def testYieldFromTwice(self):
        f = futures.Future()
        def fixture():
            yield 'A'
            x = yield from f
            yield 'B', x
            y = yield from f
            yield 'C', y
        g = fixture()
        self.assertEqual(next(g), 'A')  # yield 'A'.
        self.assertEqual(next(g), f)  # First yield from f.
        f.set_result(42)
        self.assertEqual(next(g), ('B', 42))  # yield 'B', x.
        # The second "yield from f" does not yield f.
        self.assertEqual(next(g), ('C', 42))  # yield 'C', y.

# A fake event loop for tests. All it does is implement a call_soon method
# that immediately invokes the given function.
class _FakeEventLoop:
    def call_soon(self, fn, future):
        fn(future)

class FutureDoneCallbackTests(unittest.TestCase):

    def _make_callback(self, bag, thing):
        # Create a callback function that appends thing to bag.
        def bag_appender(future):
            bag.append(thing)
        return bag_appender

    def _new_future(self):
        return futures.Future(event_loop=_FakeEventLoop())

    def testCallbacksInvokedOnSetResult(self):
        bag = []
        f = self._new_future()
        f.add_done_callback(self._make_callback(bag, 42))
        f.add_done_callback(self._make_callback(bag, 17))

        self.assertEqual(bag, [])
        f.set_result('foo')
        self.assertEqual(bag, [42, 17])
        self.assertEqual(f.result(), 'foo')

    def testCallbacksInvokedOnSetException(self):
        bag = []
        f = self._new_future()
        f.add_done_callback(self._make_callback(bag, 100))

        self.assertEqual(bag, [])
        exc = RuntimeError()
        f.set_exception(exc)
        self.assertEqual(bag, [100])
        self.assertEqual(f.exception(), exc)

    def testRemoveDoneCallback(self):
        bag = []
        f = self._new_future()
        cb1 = self._make_callback(bag, 1)
        cb2 = self._make_callback(bag, 2)
        cb3 = self._make_callback(bag, 3)

        # Add one cb1 and one cb2.
        f.add_done_callback(cb1)
        f.add_done_callback(cb2)

        # One instance of cb2 removed. Now there's only one cb1.
        self.assertEqual(f.remove_done_callback(cb2), 1)

        # Never had any cb3 in there.
        self.assertEqual(f.remove_done_callback(cb3), 0)

        # After this there will be 6 instances of cb1 and one of cb2.
        f.add_done_callback(cb2)
        for i in range(5):
            f.add_done_callback(cb1)

        # Remove all instances of cb1. One cb2 remains.
        self.assertEqual(f.remove_done_callback(cb1), 6)

        self.assertEqual(bag, [])
        f.set_result('foo')
        self.assertEqual(bag, [2])
        self.assertEqual(f.result(), 'foo')

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/python
import os
import click
import csv
from magic import Magic

class Scanner:
    def __init__(self, **options):
        self.options = options
        self.mime = options['mime']
        self.magic = Magic(magic_file='magic.db', mime=self.mime, uncompress=True)
        self._reset()

    def _reset(self):
        self.types = {}
        self.files = {}
        self.total = 0

    def scan_file(self, file):
        return self.magic.from_file(file)

    def scan(self, path):
        for root, directory, files in os.walk(path):
            for file in files:
                self.total += 1
                path = os.path.join(root, file)
                mime = self.scan_file(path)
                if not mime in self.types: self.types[mime] = []
                self.files[path] = {}
                self.files[path]['mime'] = mime
                self.types[mime].append(path)

class Reporter:
    def __init__(self, options, scanner):
        self.options = options
        self.scanner = scanner
        self.outfile = options['target']

    def report(self):
        raise NotImplemented()

class ReportCSV(Reporter):
    def report(self):
        writer = csv.writer(self.options['target'])

        headers = ["filename", ["type", "mimetype"][self.scanner.mime]]

        writer.writerow(headers)
        for f, details in self.scanner.files.iteritems():
            line = [f, details['mime']]
            writer.writerow(line)

class ReportAggregate(Reporter):
    def report(self):
        for t, fs in self.scanner.types.iteritems():
            self.outfile.write("%-35s: %d (%d%%)\n" % (t, len(fs),
                len(fs)*100/self.scanner.total))

outputmodes = {
    "csv":      ReportCSV,
    "report":   ReportAggregate,
}

@click.command()
@click.argument('path', type=click.Path(exists=True))
@click.option('--mime', is_flag=True, help="Show mime types")
@click.option('--format', type=click.Choice(outputmodes.keys()),
    default="report", help="Choose output format")
@click.option('--target', type=click.File('w'), default='-')
def main(path, **options):
    scanner = Scanner(**options)
    scanner.scan(path)
    reporter = outputmodes[options['format']](options, scanner)
    reporter.report()

if __name__ == "__main__":
    main()
<commit_msg>--size to report file sizes too<commit_after>#!/usr/bin/python
import os
import click
import csv
from magic import Magic

class Scanner:
    def __init__(self, **options):
        self.options = options
        self.mime = options['mime']
        self.magic = Magic(magic_file='magic.db', mime=self.mime, uncompress=True)
        self._reset()

    def _reset(self):
        self.types = {}
        self.files = {}
        self.total = 0

    def scan_file(self, file):
        return self.magic.from_file(file)

    def scan(self, path):
        for root, directory, files in os.walk(path):
            for file in files:
                self.total += 1
                path = os.path.join(root, file)
                mime = self.scan_file(path)
                if not mime in self.types: self.types[mime] = []
                self.files[path] = {}
                self.files[path]['mime'] = mime
                if self.options['size']:
                    stat = os.stat(path)
                    self.files[path]['size'] = stat.st_size
                self.types[mime].append(path)

class Reporter:
    def __init__(self, options, scanner):
        self.options = options
        self.scanner = scanner
        self.outfile = options['target']

    def report(self):
        raise NotImplemented()

class ReportCSV(Reporter):
    def report(self):
        writer = csv.writer(self.options['target'])

        headers = ["filename", ["type", "mimetype"][self.scanner.mime]]
        if self.options['size']:
            headers.append('size')

        writer.writerow(headers)
        for f, details in self.scanner.files.iteritems():
            line = [f, details['mime']]
            if self.options['size']:
                line.append(details['size'])
            writer.writerow(line)

class ReportAggregate(Reporter):
    def report(self):
        for t, fs in self.scanner.types.iteritems():
            self.outfile.write("%-35s: %d (%d%%)\n" % (t, len(fs),
                len(fs)*100/self.scanner.total))

outputmodes = {
    "csv":      ReportCSV,
    "report":   ReportAggregate,
}

@click.command()
@click.argument('path', type=click.Path(exists=True))
@click.option('--mime', is_flag=True, help="Show mime types")
@click.option('--format', type=click.Choice(outputmodes.keys()),
    default="report", help="Choose output format")
@click.option('--size', is_flag=True, help="Show file sizes")
@click.option('--target', type=click.File('w'), default='-')
def main(path, **options):
    scanner = Scanner(**options)
    scanner.scan(path)
    reporter = outputmodes[options['format']](options, scanner)
    reporter.report()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2016, Jonathan Kelley
#
# License: the mit license
#    Permission is hereby granted, free of charge, to any person obtaining a copy of
#    this software and associated documentation files (the "Software"), to deal in
#    the Software without restriction, including without limitation the rights to
#    use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
#    the Software, and to permit persons to whom the Software is furnished to do so,
#    subject to the above copyright notice and this permission notice shall being included
#    in all copies or substantial portions of the Software.

# -- config --
# description: A test file to validate functionality and get up and running
# param: name - Your name
# param: age - Your age
# http_method: post
# lock: False
# -- end config --

from extension import Constants
from extension import ParamHandle as Param
import datetime

# Spawn Instances
p = Param()    # <class> Parame ter manipulation
params = p.list()    # <dict>  A dict of params
dt = datetime.datetime.now()  #<datetime> object

# ************************************
# *  DEFINE PARAMETERS AND VALIDATE  *
# ************************************
arguement = {} # The actual API params we pass

param = "name"
name = Param()
name.value = params[param]
name.name = param
name.require = True
name.max_length = 128
arguement[param] = name.get()

param = "age"
age = Param()
age.value = params[param]
age.name = param
age.require = True
age.max_length = 3
age.max_int = 99
arguement[param] = age.get()
# Calculate cake day
birthyear =  dt.year - int(arguement['age'])

# *************
# *  RESULTS  *
# *************
print(("{status} current_datetime={datetime}\n"
  "{status} name={whom}\n"
  "{status} age={age}\n"
  "{status} status=You were born in {year}").format(
  status=Constants.API_RETURN_STRING,
  age=arguement['age'],
  whom=arguement['name'],
  year=birthyear,
  datetime=dt))

exit(0)
<commit_msg>Update test extension<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2016, Jonathan Kelley
#
# License: the mit license
#    Permission is hereby granted, free of charge, to any person obtaining a copy of
#    this software and associated documentation files (the "Software"), to deal in
#    the Software without restriction, including without limitation the rights to
#    use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
#    the Software, and to permit persons to whom the Software is furnished to do so,
#    subject to the above copyright notice and this permission notice shall being included
#    in all copies or substantial portions of the Software.

# -- config --
# description: A test file to validate functionality and get up and running
# param: name - Your name
# param: age - Your age
# http_method: post
# lock: False
# -- end config --

from extension import Constants
from extension import ParamHandle as Param
import datetime

# Spawn Instances
p = Param()    # <class> Parame ter manipulation
params = p.list()    # <dict>  A dict of params
dt = datetime.datetime.now()  #<datetime> object

# ************************************
# *  DEFINE PARAMETERS AND VALIDATE  *
# ************************************
arguement = {} # The actual API params we pass

param = "name"
name = Param()
name.value = params[param]
name.name = param
name.require = True
name.max_length = 128
arguement[param] = name.get()

param = "age"
age = Param()
age.value = params[param]
age.name = param
age.require = True
age.max_length = 3
age.max_int = 99
arguement[param] = age.get()
# Calculate cake day
birthyear =  dt.year - int(arguement['age'])

# *************
# *  RESULTS  *
# *************
print(("{status} current_datetime={datetime}\n"
  "{status} name={whom}\n"
  "{status} age={age}\n"
  "{status} status={u} was born in {year}").format(
  status=Constants.API_RETURN_STRING,
  age=arguement['age'],
  whom=arguement['name'],
  u=arguement['name'].title(),
  year=birthyear,
  datetime=dt))

exit(0)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Author: François Rossigneux <francois.rossigneux@inria.fr>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""This blueprint defines all URLs and answers."""

import flask
import socket
from execo_g5k import get_resource_attributes
from kwapi.utils import cfg, log
from pandas import read_hdf
from hdf5 import get_probe_path

LOG = log.getLogger(__name__)

web_opts = [
    cfg.StrOpt('hdf5_dir',
               required=True,
               ),
]
cfg.CONF.register_opts(web_opts)

blueprint = flask.Blueprint('v1', __name__)

@blueprint.route('/')
def welcome():
    """Returns detailed information about this specific version of the API."""
    return 'Welcome to Kwapi!'

@blueprint.route('/probe-ids/')
def list_probes_ids():
    """Returns all known probes IDs."""
    message = {}
    message['probe_ids'] = flask.request.collector.database.keys()
    response = flask.jsonify(message)
    response.headers.add('Access-Control-Allow-Origin', '*')
    return response

@blueprint.route('/measurements/')
def retrieve_measurements():
    """Returns measurements."""
    hostname = socket.getfqdn().split('.')
    site = hostname[1] if len(hostname) >= 2 else hostname[0]

    args = flask.request.args

    if 'job_id' in args:
        job_info = get_resource_attributes('sites/' + site + '/jobs/' + args['job_id'])
        start_time = job_info['started_at']
        end_time = start_time + job_info['walltime']
        nodes = list(set(job_info['resources_by_type']['cores']))
        probes = [site + '.' + node.split('.')[0] for node in nodes]
    else:
        probes = [site + '.' + node for node in args['probes'].split(',')]
        start_time = args['start_time']
        end_time = args['end_time']

    message = {}
    for probe in probes:
        df = read_hdf(cfg.CONF.hdf5_dir + '/store.h5', get_probe_path(probe),
            where=['index>=' +str(start_time), 'index<=' +str(end_time)])
        measurements = df.to_json(orient='index')
        LOG.info( probe)
        LOG.info( measurements)

    message = {}
    for probe in probes:
        message[probe] = {
            'start_time': start_time,
            'end_time': end_time,
            'kwh': kwh
        }
    response = flask.jsonify(message)
    response.headers.add('Access-Control-Allow-Origin', '*')
    return response<commit_msg>Large improvements to plugin hdf5 that now returns values<commit_after># -*- coding: utf-8 -*-
#
# Author: François Rossigneux <francois.rossigneux@inria.fr>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""This blueprint defines all URLs and answers."""

import time
import flask
import socket
from execo_g5k import get_resource_attributes
from kwapi.utils import cfg, log
from pandas import read_hdf
from hdf5 import get_probe_path

LOG = log.getLogger(__name__)

web_opts = [
    cfg.StrOpt('hdf5_dir',
               required=True,
               ),
]
cfg.CONF.register_opts(web_opts)

blueprint = flask.Blueprint('v1', __name__)

@blueprint.route('/')
def welcome():
    """Returns detailed information about this specific version of the API."""
    return 'Welcome to Kwapi!'

@blueprint.route('/probe-ids/')
def list_probes_ids():
    """Returns all known probes IDs."""
    message = {}
    message['probe_ids'] = flask.request.collector.database.keys()
    response = flask.jsonify(message)
    response.headers.add('Access-Control-Allow-Origin', '*')
    return response

@blueprint.route('/metrics/')
def retrieve_measurements():
    """Returns measurements."""
    hostname = socket.getfqdn().split('.')
    site = hostname[1] if len(hostname) >= 2 else hostname[0]

    args = flask.request.args
    probes = None
    if 'job_id' in args:
        job_info = get_resource_attributes('sites/' + site + '/jobs/' + args['job_id'])
        start_time = job_info['started_at']
        end_time = start_time + job_info['walltime']
        nodes = list(set(job_info['resources_by_type']['cores']))
        probes = [site + '.' + node.split('.')[0] for node in nodes]
    elif 'probes' in args:
        LOG.info(args['probes'].split(','))
        probes = [site + '.' + node for node in args['probes'].split(',')]
        start_time = args['start_time'] if 'start_time' in args else time.time() - 24 * 3600
        end_time = args['end_time'] if 'end_time' in args else time.time()

    if probes:
        message = {'start_time': start_time, 'end_time': end_time, 'probes': {}}
        for probe in probes:
            message['probes'][probe] = []
            df = read_hdf(cfg.CONF.hdf5_dir + '/store.h5', get_probe_path(probe),
                where=['index>=' +str(start_time), 'index<=' +str(end_time)])
            for ts, mes in df.iterrows():
                message['probes'][probe].append((ts, mes[0]))

    response = flask.jsonify(message)
    response.headers.add('Access-Control-Allow-Origin', '*')
    return response<|endoftext|>