<commit_before>from typing import List, Tuple, Optional, Union

from hwt.hdl.operatorDefs import OpDefinition
from hwt.hdl.value import HValue
from hwt.synthesizer.rtlLevel.mainBases import RtlSignalBase
from hwt.synthesizer.rtlLevel.rtlSignal import RtlSignal
from hwtHls.hlsStreamProc.ssa.phi import SsaPhi, TmpVariable

class SsaInstrBranch():

    def __init__(self, parent: "SsaBasicBlock"):
        self.parent = parent
        self.targets: List[Tuple[Optional[RtlSignal], "SsaBasicBlock"]] = []

    def addTarget(self, cond: Optional[RtlSignal], target: "SsaBasicBlock"):
        self.targets.append((cond, target))
        target.predecessors.append(self.parent)
        if isinstance(cond, SsaPhi):
            cond.users.append(self)

    def replaceTargetBlock(self, orig_block:"SsaBasicBlock", new_block:"SsaBasicBlock"):
        for i, (c, b) in enumerate(self.targets):
            if b is orig_block:
                self.targets[i] = (c, new_block)

    def __len__(self):
        return len(self.targets)

    def iter_blocks(self):
        for (_, t) in self.targets:
            yield t

    def __repr__(self):
        return f"<{self.__class__.__name__} {self.targets}>"

ValOrVal = Union[TmpVariable, RtlSignal, HValue, SsaPhi]

class SsaInstr():

    def __init__(self,
                 dst: Union[RtlSignal, TmpVariable],
                 src: Union[ValOrVal,
                            Tuple[OpDefinition, List[ValOrVal]]
                ]):
        assert isinstance(dst, RtlSignalBase), dst
        self.dst = dst
        self.src = src
        if isinstance(src, SsaPhi):
            src.users.append(self)
        elif isinstance(src, tuple):
            for op in src[1]:
                if isinstance(op, SsaPhi):
                    op.users.append(self)

    def iterInputs(self):
        src = self.src
        if isinstance(src, tuple):
            yield from src[1]
        else:
            yield src

    def replaceInput(self, orig_expr: SsaPhi, new_expr: Union[SsaPhi, HValue]):
        src = self.src
        if orig_expr is src:
            orig_expr.users.remove(self)
            self.src = new_expr
            if isinstance(new_expr, SsaPhi):
                new_expr.users.append(self)
        else:
            assert orig_expr in src[1]
            self.src = (src[0], tuple(new_expr if o is orig_expr else o for o in src[1]))
            orig_expr.users.remove(self)
            if isinstance(new_expr, SsaPhi):
                new_expr.users.append(self)

    def __repr__(self):
        dst = self.dst
        src = self.src
        if isinstance(src, (TmpVariable, RtlSignal, HValue, SsaPhi)):
            return f"{dst} = {src}"
        else:
            _src = ", ".join(repr(s) for s in src[1])
            return f"{dst} = {src[0].id:s} {_src:s}"

<commit_msg>TmpVariable -> HlsTmpVariable<commit_after>from typing import List, Tuple, Optional, Union

from hwt.hdl.operatorDefs import OpDefinition
from hwt.hdl.value import HValue
from hwt.synthesizer.rtlLevel.mainBases import RtlSignalBase
from hwt.synthesizer.rtlLevel.rtlSignal import RtlSignal
from hwtHls.hlsStreamProc.ssa.phi import SsaPhi
from hwtHls.tmpVariable import HlsTmpVariable

class SsaInstrBranch():

    def __init__(self, parent: "SsaBasicBlock"):
        self.parent = parent
        self.targets: List[Tuple[Optional[RtlSignal], "SsaBasicBlock"]] = []

    def addTarget(self, cond: Optional[RtlSignal], target: "SsaBasicBlock"):
        self.targets.append((cond, target))
        target.predecessors.append(self.parent)
        if isinstance(cond, SsaPhi):
            cond.users.append(self)

    def replaceTargetBlock(self, orig_block:"SsaBasicBlock", new_block:"SsaBasicBlock"):
        for i, (c, b) in enumerate(self.targets):
            if b is orig_block:
                self.targets[i] = (c, new_block)

    def __len__(self):
        return len(self.targets)

    def iter_blocks(self):
        for (_, t) in self.targets:
            yield t

    def __repr__(self):
        return f"<{self.__class__.__name__} {self.targets}>"

ValOrVal = Union[HlsTmpVariable, RtlSignal, HValue, SsaPhi]

class SsaInstr():

    def __init__(self,
                 dst: Union[RtlSignal, HlsTmpVariable],
                 src: Union[ValOrVal,
                            Tuple[OpDefinition, List[ValOrVal]]
                ]):
        assert isinstance(dst, RtlSignalBase), dst
        self.dst = dst
        self.src = src
        if isinstance(src, SsaPhi):
            src.users.append(self)
        elif isinstance(src, tuple):
            for op in src[1]:
                if isinstance(op, SsaPhi):
                    op.users.append(self)

    def iterInputs(self):
        src = self.src
        if isinstance(src, tuple):
            yield from src[1]
        else:
            yield src

    def replaceInput(self, orig_expr: SsaPhi, new_expr: Union[SsaPhi, HValue]):
        src = self.src
        if orig_expr is src:
            orig_expr.users.remove(self)
            self.src = new_expr
            if isinstance(new_expr, SsaPhi):
                new_expr.users.append(self)
        else:
            assert orig_expr in src[1]
            self.src = (src[0], tuple(new_expr if o is orig_expr else o for o in src[1]))
            orig_expr.users.remove(self)
            if isinstance(new_expr, SsaPhi):
                new_expr.users.append(self)

    def __repr__(self):
        dst = self.dst
        src = self.src
        if isinstance(src, (HlsTmpVariable, RtlSignal, HValue, SsaPhi)):
            return f"{dst} = {src}"
        else:
            _src = ", ".join(repr(s) for s in src[1])
            return f"{dst} = {src[0].id:s} {_src:s}"

<|endoftext|>


<commit_before># Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""*Experimental* support for running Keras models on the TPU.

To use, wrap your model with the `keras_support.tpu_model` function.

Example usage:

```
# Must activate before building TPU models
keras_support.setup_tpu_session(master_address)

image = tf.keras.layers.Input(shape=(28, 28, 3), name='image')
c1 = tf.keras.layers.Conv2D(filters=16, kernel_size=(3, 3))( image)
flattened = tf.keras.layers.Flatten()(c1)
logits = tf.keras.layers.Dense(10, activation='softmax')(flattened)
model = tf.keras.Model(inputs=[image], outputs=[logits])
model = keras_support.tpu_model(model)

# Only TF optimizers are currently supported.
model.compile(optimizer=tf.train.AdamOptimizer(), ...)

# `images` and `labels` should be Numpy arrays.  Support for tensor input
# (e.g. datasets) is planned.
model.fit(images, labels)

# Invoke before shutting down
keras_support.shutdown_tpu_session()
```
"""

# pylint: disable=protected-access

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import collections
import re

from tensorflow.contrib.framework.python.framework import experimental
from tensorflow.contrib.tpu.python.ops import tpu_ops
from tensorflow.contrib.tpu.python.tpu import tpu
from tensorflow.core.protobuf import config_pb2
from tensorflow.python.client import session as tf_session
from tensorflow.python.estimator import model_fn as model_fn_lib
from tensorflow.python.framework import ops
from tensorflow.python.framework import tensor_spec
from tensorflow.python.keras._impl.keras import backend as K
from tensorflow.python.keras._impl.keras import layers
from tensorflow.python.keras._impl.keras import models
from tensorflow.python.keras._impl.keras import optimizers as keras_optimizers
from tensorflow.python.keras._impl.keras.layers import embeddings
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.platform import tf_logging as logging

class TPUEmbedding(embeddings.Embedding):
  """TPU compatible embedding layer.

  The default Keras layer is not TPU compatible.  This layer is a drop-in
  replacement: it has the same behavior and will work on CPU and GPU devices.
  """

  def __init__(self, *args, **kw):
    super(TPUEmbedding, self).__init__(*args, **kw)

  def build(self, input_shape):
    if input_shape[0] is None:
      raise ValueError(
          'TPUEmbeddings must have a fixed input_length or input shape.')
    return super(TPUEmbedding, self).build(input_shape)

  def call(self, inputs):
    if K.dtype(inputs) != 'int32':
      inputs = math_ops.cast(inputs, 'int32')

    inputs = array_ops.one_hot(inputs, self.input_dim)
    return math_ops.tensordot(inputs, self.embeddings, 1)

class CompiledTPUOp(
    collections.namedtuple(
        'CompiledTPUOp',
        ['tpu_execute_op', 'infeed_tensors', 'infeed_op', 'outfeed_op'])):
  pass

def _valid_name(tensor_name):
  """Return a valid tensor name (strips '/', ':', etc)."""
  return re.sub('[^a-zA-Z0-9_-]+', '', tensor_name)

class TPUFunction(object):
  """K.function compatible interface for invoking a TPU compiled function.

  Recompilation is triggered on-demand for each set of new inputs shapes: the
  results are cached for future execution.  We expect most computations will
  be dominated by a standard batch-size, followed by a straggler batch for
  the end of training or evaluation.

  All `inputs` and `outputs` will be loaded via the infeed and outfeed queues
  instead of being injected as `feed_dict` items or fetches.
  """

  def __init__(self, model, execution_mode):
    self.model = model
    self.execution_mode = execution_mode
    self._compilation_cache = {}

  def _specialize_model(self, input_specs):
    """Specialize `self.model` (a Keras model) for the given input shapes."""
    # Re-create our input and output layers inside our subgraph.  They will be
    # attached to the true computation when we clone our model in `tpu_fn`.
    K.set_learning_phase(
        self.execution_mode == model_fn_lib.ModeKeys.TRAIN
    )

    # functools.partial and callable objects are not supported by tpu.rewrite
    def _model_fn():
      """Compute fit/eval/predict for the TPU."""
      is_training = self.execution_mode == model_fn_lib.ModeKeys.TRAIN
      is_test = self.execution_mode == model_fn_lib.ModeKeys.EVAL
      is_predict = self.execution_mode == model_fn_lib.ModeKeys.PREDICT

      # During train/eval, we infeed our features as well as labels.
      if is_training or is_test:
        infeed_layers = self.model._input_layers + self.model._output_layers
      else:
        infeed_layers = self.model._input_layers

      # Generate our infeed operation to read features & labels.
      infeed_tensors = tpu_ops.infeed_dequeue_tuple(
          dtypes=[spec.dtype for spec in input_specs],
          shapes=[spec.shape for spec in input_specs],
          name='infeed-%s' % self.execution_mode)

      assert len(infeed_tensors) == len(infeed_layers), (
          'Infeed inputs did not match model: %s vs %s', (infeed_layers,
                                                          infeed_tensors))

      tpu_targets = []
      tpu_inputs = []

      # Sort infeed outputs into inputs and labels for calling our Keras model.
      for tensor, layer in zip(infeed_tensors, infeed_layers):
        if layer in self.model._input_layers:
          tpu_inputs.append(layers.Input(name=layer.name, tensor=tensor))
        if layer in self.model._output_layers:
          tpu_targets.append(tensor)

      # Call our model with our infeed inputs (re-using the weights).
      model_outputs = self.model(tpu_inputs)
      child_model = models.Model(inputs=tpu_inputs, outputs=model_outputs)
      if is_training or is_test:
        child_model.compile(
            optimizer=self.model.optimizer,
            loss=self.model.loss,
            loss_weights=self.model.loss_weights,
            metrics=self.model.metrics,
            weighted_metrics=self.model.weighted_metrics,
            target_tensors=tpu_targets,
        )

      # Compute our outfeed depending on the execution mode
      if is_training:
        child_model._make_train_function()
        self._outfeed_spec = [
            tensor_spec.TensorSpec(tensor.shape, tensor.dtype, tensor.name)
            for tensor in child_model.train_function.outputs
        ]
        return [
            child_model.train_function.updates_op,
            tpu_ops.outfeed_enqueue_tuple(
                child_model.train_function.outputs, name='oufeed-enqueue-train')
        ]
      elif is_test:
        child_model._make_test_function()
        self._outfeed_spec = [
            tensor_spec.TensorSpec(tensor.shape, tensor.dtype, tensor.name)
            for tensor in child_model.test_function.outputs
        ]
        return [
            tpu_ops.outfeed_enqueue_tuple(
                child_model.test_function.outputs, name='outfeed-enqueue-test')
        ]
      elif is_predict:
        child_model._make_predict_function()
        self._outfeed_spec = [
            tensor_spec.TensorSpec(tensor.shape, tensor.dtype, tensor.name)
            for tensor in child_model.predict_function.outputs
        ]
        return [
            tpu_ops.outfeed_enqueue_tuple(
                child_model.predict_function.outputs,
                name='outfeed-enqueue-predict',
            )
        ]
      else:
        assert False, 'Unexpected execution mode: %s' % self.execution_mode

    # Capture outfeed metadata computed during the rewrite.
    self._outfeed_spec = None

    tpu_execute_op = tpu.rewrite(_model_fn)

    # Generate CPU side operations to enqueue features/labels and dequeue
    # outputs from the model call.
    with ops.device('/device:TPU:0'):
      infeed_tensors = []
      for spec in input_specs:
        infeed_tensors.append(
            array_ops.placeholder(
                dtype=spec.dtype,
                shape=spec.shape,
                name='infeed-enqueue-%s' % spec.name))

      infeed_op = tpu_ops.infeed_enqueue_tuple(
          infeed_tensors, [spec.shape for spec in input_specs],
          name='infeed-enqueue-%s' % self.execution_mode)

      outfeed_op = tpu_ops.outfeed_dequeue_tuple(
          dtypes=[spec.dtype for spec in self._outfeed_spec],
          shapes=[spec.shape for spec in self._outfeed_spec],
          name='outfeed-dequeue-%s' % self.execution_mode)

    return CompiledTPUOp(tpu_execute_op, infeed_tensors, infeed_op, outfeed_op)

  def __call__(self, inputs):
    assert isinstance(inputs, list)

    # Strip sample weight from inputs
    if (self.execution_mode == model_fn_lib.ModeKeys.TRAIN or
        self.execution_mode == model_fn_lib.ModeKeys.EVAL):
      input_tensors = self.model._feed_inputs + self.model._feed_targets
      inputs = inputs[:len(input_tensors)]
    else:
      input_tensors = self.model._feed_inputs

    # Compute an input specification (used to generate infeed enqueue and
    # dequeue operations).  We use the shape from our input array and the
    # dtype from our model.  A user may pass in a float64 for a float32
    # input: for model compatibility we still must generate a float32 infeed.
    input_specs = []
    for tensor, ary in zip(input_tensors, inputs):
      input_specs.append(
          tensor_spec.TensorSpec(ary.shape, tensor.dtype,
                                 _valid_name(tensor.name)))

    # XLA requires every operation in the graph has a fixed shape.  To
    # handle varying batch sizes we recompile a new sub-graph for each
    # unique input shape.
    shape_key = tuple([tuple(spec.shape.as_list()) for spec in input_specs])

    if shape_key not in self._compilation_cache:
      logging.info('New input shapes; (re-)compiling: mode=%s, %s',
                   self.execution_mode, input_specs)
      self._compilation_cache[shape_key] = self._specialize_model(input_specs)

    compiled_model = self._compilation_cache[shape_key]

    infeed_dict = {}
    for tensor, value in zip(compiled_model.infeed_tensors, inputs):
      infeed_dict[tensor] = value

    session = K.get_session()
    _, _, outfeed_outputs = session.run([
        compiled_model.infeed_op, compiled_model.tpu_execute_op,
        compiled_model.outfeed_op
    ], infeed_dict)

    return outfeed_outputs

@experimental
def setup_tpu_session(master):
  """Initializes and returns a Keras/TF session connected the TPU `master`."""
  session = tf_session.Session(
      target=master, config=config_pb2.ConfigProto(isolate_session_state=True))
  K.set_session(session)
  K.get_session().run(tpu.initialize_system())
  return session

@experimental
def shutdown_tpu_session(session=None):
  """Shutdown the TPU attached to session.

  This should be called to cleanly shut down the TPU system before the client
  exits.

  Args:
    session: Session to shutdown, or None to use the default session.

  Returns:

  """
  if session is None:
    session = K.get_session()

  session.run(tpu.shutdown_system())

class KerasTPUModel(models.Model):
  """TPU compatible Keras model wrapper."""

  def __init__(self, inputs, outputs, name=None):
    super(models.Model, self).__init__(
        inputs=inputs,
        outputs=outputs,
        name=name,
    )
    self.predict_function = None
    self.test_function = None
    self.train_function = None

  def compile(self,
              optimizer,
              loss=None,
              metrics=None,
              loss_weights=None,
              sample_weight_mode=None,
              weighted_metrics=None,
              target_tensors=None,
              **kwargs):
    if sample_weight_mode:
      raise ValueError('sample_weight_mode not supported for TPU execution.')
    if weighted_metrics:
      raise ValueError('weighted_metrics not supported for TPU execution.')
    if target_tensors:
      raise ValueError('target_tensors is not supported for TPU execution.')

    super(KerasTPUModel, self).compile(optimizer, loss, metrics, loss_weights,
                                       sample_weight_mode, weighted_metrics,
                                       target_tensors, **kwargs)

    # Keras optimizers are not compatible with TPU rewrite
    if not isinstance(self.optimizer, keras_optimizers.TFOptimizer):
      raise ValueError(
          'Optimizer must be a TFOptimizer, got: %s' % self.optimizer)

  def _make_train_function(self):
    if not self.train_function:
      self.train_function = TPUFunction(self, model_fn_lib.ModeKeys.TRAIN)

    return self.train_function

  def _make_test_function(self):
    if not self.test_function:
      self.test_function = TPUFunction(self, model_fn_lib.ModeKeys.EVAL)
    return self.test_function

  def _make_predict_function(self):
    if not self.predict_function:
      self.predict_function = TPUFunction(self, model_fn_lib.ModeKeys.PREDICT)
    return self.predict_function

  def cpu_model(self):
    cpu_model = models.Model(
        inputs=self.inputs,
        outputs=self.outputs,
        name=self.name,
    )

    if self.optimizer:
      cpu_model.compile(
          optimizer=self.optimizer,
          loss=self.loss,
          metrics=self.metrics,
          loss_weights=self.loss_weights,
      )

    return cpu_model

def _validate_shapes(model):
  """Validate that all layers in `model` have constant shape."""
  for layer in model.layers:
    if isinstance(layer.input_shape, tuple):
      input_shapes = [layer.input_shape]
    else:
      input_shapes = layer.input_shape

    if isinstance(layer.output_shape, tuple):
      output_shapes = [layer.output_shape]
    else:
      output_shapes = layer.output_shape

    for shape in input_shapes + output_shapes:
      for dim in shape[1:]:
        if dim is None:
          raise ValueError(
              """
Layer %(layer)s has a variable shape in a non-batch dimension.  TPU models must
have constant shapes for all operations.

You may have to specify `input_length` for RNN/TimeDistributed layers.

Layer: %(layer)s
Input shape: %(input_shape)s
Output shape: %(output_shape)s
  """ % {
      'layer': layer,
      'input_shape': layer.input_shape,
      'output_shape': layer.output_shape
      })

@experimental
def tpu_model(model):
  _validate_shapes(model)
  return KerasTPUModel(
      inputs=model.inputs, outputs=model.outputs, name=model.name)
<commit_msg>Use the new compile op and print user-friendly error message without mixing with infeed/outfeed if compilation fails.<commit_after># Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""*Experimental* support for running Keras models on the TPU.

To use, wrap your model with the `keras_support.tpu_model` function.

Example usage:

```
# Must activate before building TPU models
keras_support.setup_tpu_session(master_address)

image = tf.keras.layers.Input(shape=(28, 28, 3), name='image')
c1 = tf.keras.layers.Conv2D(filters=16, kernel_size=(3, 3))( image)
flattened = tf.keras.layers.Flatten()(c1)
logits = tf.keras.layers.Dense(10, activation='softmax')(flattened)
model = tf.keras.Model(inputs=[image], outputs=[logits])
model = keras_support.tpu_model(model)

# Only TF optimizers are currently supported.
model.compile(optimizer=tf.train.AdamOptimizer(), ...)

# `images` and `labels` should be Numpy arrays.  Support for tensor input
# (e.g. datasets) is planned.
model.fit(images, labels)

# Invoke before shutting down
keras_support.shutdown_tpu_session()
```
"""

# pylint: disable=protected-access

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import collections
import re
import time

from tensorflow.contrib.framework.python.framework import experimental
from tensorflow.contrib.tpu.proto import compilation_result_pb2 as tpu_compilation_result
from tensorflow.contrib.tpu.python.ops import tpu_ops
from tensorflow.contrib.tpu.python.tpu import tpu
from tensorflow.core.protobuf import config_pb2
from tensorflow.python.client import session as tf_session
from tensorflow.python.estimator import model_fn as model_fn_lib
from tensorflow.python.framework import ops
from tensorflow.python.framework import tensor_spec
from tensorflow.python.keras._impl.keras import backend as K
from tensorflow.python.keras._impl.keras import layers
from tensorflow.python.keras._impl.keras import models
from tensorflow.python.keras._impl.keras import optimizers as keras_optimizers
from tensorflow.python.keras._impl.keras.layers import embeddings
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.platform import tf_logging as logging

class TPUEmbedding(embeddings.Embedding):
  """TPU compatible embedding layer.

  The default Keras layer is not TPU compatible.  This layer is a drop-in
  replacement: it has the same behavior and will work on CPU and GPU devices.
  """

  def build(self, input_shape):
    if input_shape[0] is None:
      raise ValueError(
          'TPUEmbeddings must have a fixed input_length or input shape.')
    return super(TPUEmbedding, self).build(input_shape)

  def call(self, inputs):
    if K.dtype(inputs) != 'int32':
      inputs = math_ops.cast(inputs, 'int32')

    inputs = array_ops.one_hot(inputs, self.input_dim)
    return math_ops.tensordot(inputs, self.embeddings, 1)

class TPUModelOp(
    collections.namedtuple(
        'TPUModelOp',
        ['compile_op', 'execute_op', 'infeed_tensors', 'infeed_op',
         'outfeed_op'])):
  pass

def _valid_name(tensor_name):
  """Return a valid tensor name (strips '/', ':', etc)."""
  return re.sub('[^a-zA-Z0-9_-]+', '', tensor_name)

class TPUFunction(object):
  """K.function compatible interface for invoking a TPU compiled function.

  Recompilation is triggered on-demand for each set of new inputs shapes: the
  results are cached for future execution.  We expect most computations will
  be dominated by a standard batch-size, followed by a straggler batch for
  the end of training or evaluation.

  All `inputs` and `outputs` will be loaded via the infeed and outfeed queues
  instead of being injected as `feed_dict` items or fetches.
  """

  def __init__(self, model, execution_mode):
    self.model = model
    self.execution_mode = execution_mode
    self._compilation_cache = {}

  def _specialize_model(self, input_specs):
    """Specialize `self.model` (a Keras model) for the given input shapes."""
    # Re-create our input and output layers inside our subgraph.  They will be
    # attached to the true computation when we clone our model in `tpu_fn`.
    K.set_learning_phase(
        self.execution_mode == model_fn_lib.ModeKeys.TRAIN
    )

    # functools.partial and callable objects are not supported by tpu.rewrite
    def _model_fn():
      """Compute fit/eval/predict for the TPU."""
      is_training = self.execution_mode == model_fn_lib.ModeKeys.TRAIN
      is_test = self.execution_mode == model_fn_lib.ModeKeys.EVAL
      is_predict = self.execution_mode == model_fn_lib.ModeKeys.PREDICT

      # During train/eval, we infeed our features as well as labels.
      if is_training or is_test:
        infeed_layers = self.model._input_layers + self.model._output_layers
      else:
        infeed_layers = self.model._input_layers

      # Generate our infeed operation to read features & labels.
      infeed_tensors = tpu_ops.infeed_dequeue_tuple(
          dtypes=[spec.dtype for spec in input_specs],
          shapes=[spec.shape for spec in input_specs],
          name='infeed-%s' % self.execution_mode)

      assert len(infeed_tensors) == len(infeed_layers), (
          'Infeed inputs did not match model: %s vs %s', (infeed_layers,
                                                          infeed_tensors))

      tpu_targets = []
      tpu_inputs = []

      # Sort infeed outputs into inputs and labels for calling our Keras model.
      for tensor, layer in zip(infeed_tensors, infeed_layers):
        if layer in self.model._input_layers:
          tpu_inputs.append(layers.Input(name=layer.name, tensor=tensor))
        if layer in self.model._output_layers:
          tpu_targets.append(tensor)

      # Call our model with our infeed inputs (re-using the weights).
      model_outputs = self.model(tpu_inputs)
      child_model = models.Model(inputs=tpu_inputs, outputs=model_outputs)
      if is_training or is_test:
        child_model.compile(
            optimizer=self.model.optimizer,
            loss=self.model.loss,
            loss_weights=self.model.loss_weights,
            metrics=self.model.metrics,
            weighted_metrics=self.model.weighted_metrics,
            target_tensors=tpu_targets,
        )

      # Compute our outfeed depending on the execution mode
      if is_training:
        child_model._make_train_function()
        self._outfeed_spec = [
            tensor_spec.TensorSpec(tensor.shape, tensor.dtype, tensor.name)
            for tensor in child_model.train_function.outputs
        ]
        return [
            child_model.train_function.updates_op,
            tpu_ops.outfeed_enqueue_tuple(
                child_model.train_function.outputs, name='oufeed-enqueue-train')
        ]
      elif is_test:
        child_model._make_test_function()
        self._outfeed_spec = [
            tensor_spec.TensorSpec(tensor.shape, tensor.dtype, tensor.name)
            for tensor in child_model.test_function.outputs
        ]
        return [
            tpu_ops.outfeed_enqueue_tuple(
                child_model.test_function.outputs, name='outfeed-enqueue-test')
        ]
      elif is_predict:
        child_model._make_predict_function()
        self._outfeed_spec = [
            tensor_spec.TensorSpec(tensor.shape, tensor.dtype, tensor.name)
            for tensor in child_model.predict_function.outputs
        ]
        return [
            tpu_ops.outfeed_enqueue_tuple(
                child_model.predict_function.outputs,
                name='outfeed-enqueue-predict',
            )
        ]
      else:
        assert False, 'Unexpected execution mode: %s' % self.execution_mode

    # Capture outfeed metadata computed during the rewrite.
    self._outfeed_spec = None

    compile_op, execute_op = tpu.split_compile_and_replicate(
        _model_fn, inputs=[[]])

    # Generate CPU side operations to enqueue features/labels and dequeue
    # outputs from the model call.
    with ops.device('/device:TPU:0'):
      infeed_tensors = []
      for spec in input_specs:
        infeed_tensors.append(
            array_ops.placeholder(
                dtype=spec.dtype,
                shape=spec.shape,
                name='infeed-enqueue-%s' % spec.name))

      infeed_op = tpu_ops.infeed_enqueue_tuple(
          infeed_tensors, [spec.shape for spec in input_specs],
          name='infeed-enqueue-%s' % self.execution_mode)

      outfeed_op = tpu_ops.outfeed_dequeue_tuple(
          dtypes=[spec.dtype for spec in self._outfeed_spec],
          shapes=[spec.shape for spec in self._outfeed_spec],
          name='outfeed-dequeue-%s' % self.execution_mode)

    return TPUModelOp(
        compile_op, execute_op, infeed_tensors, infeed_op, outfeed_op)

  def _test_model_compiles(self, tpu_model_ops):
    """Verifies that the given TPUModelOp can be compiled via XLA."""
    session = K.get_session()

    logging.info('Started compiling')
    start_time = time.clock()

    result = session.run(tpu_model_ops.compile_op)
    proto = tpu_compilation_result.CompilationResultProto()
    proto.ParseFromString(result)
    if proto.status_error_message:
      raise RuntimeError(
          'Compilation failed: {}'.format(proto.status_error_message))

    end_time = time.clock()
    logging.info('Finished compiling. Time elapsed: %s secs',
                 end_time - start_time)

  def __call__(self, inputs):
    assert isinstance(inputs, list)

    # Strip sample weight from inputs
    if (self.execution_mode == model_fn_lib.ModeKeys.TRAIN or
        self.execution_mode == model_fn_lib.ModeKeys.EVAL):
      input_tensors = self.model._feed_inputs + self.model._feed_targets
      inputs = inputs[:len(input_tensors)]
    else:
      input_tensors = self.model._feed_inputs

    # Compute an input specification (used to generate infeed enqueue and
    # dequeue operations).  We use the shape from our input array and the
    # dtype from our model.  A user may pass in a float64 for a float32
    # input: for model compatibility we still must generate a float32 infeed.
    input_specs = []
    for tensor, ary in zip(input_tensors, inputs):
      input_specs.append(
          tensor_spec.TensorSpec(ary.shape, tensor.dtype,
                                 _valid_name(tensor.name)))

    # XLA requires every operation in the graph has a fixed shape.  To
    # handle varying batch sizes we recompile a new sub-graph for each
    # unique input shape.
    shape_key = tuple([tuple(spec.shape.as_list()) for spec in input_specs])

    if shape_key not in self._compilation_cache:
      logging.info('New input shapes; (re-)compiling: mode=%s, %s',
                   self.execution_mode, input_specs)
      new_tpu_model_ops = self._specialize_model(input_specs)
      self._compilation_cache[shape_key] = new_tpu_model_ops
      self._test_model_compiles(new_tpu_model_ops)

    tpu_model_ops = self._compilation_cache[shape_key]

    infeed_dict = {}
    for tensor, value in zip(tpu_model_ops.infeed_tensors, inputs):
      infeed_dict[tensor] = value

    session = K.get_session()
    _, _, outfeed_outputs = session.run([
        tpu_model_ops.infeed_op, tpu_model_ops.execute_op,
        tpu_model_ops.outfeed_op
    ], infeed_dict)

    return outfeed_outputs

@experimental
def setup_tpu_session(master):
  """Initializes and returns a Keras/TF session connected the TPU `master`."""
  session = tf_session.Session(
      target=master, config=config_pb2.ConfigProto(isolate_session_state=True))
  K.set_session(session)
  K.get_session().run(tpu.initialize_system())
  return session

@experimental
def shutdown_tpu_session(session=None):
  """Shutdown the TPU attached to session.

  This should be called to cleanly shut down the TPU system before the client
  exits.

  Args:
    session: Session to shutdown, or None to use the default session.

  Returns:

  """
  if session is None:
    session = K.get_session()

  session.run(tpu.shutdown_system())

class KerasTPUModel(models.Model):
  """TPU compatible Keras model wrapper."""

  def __init__(self, inputs, outputs, name=None):
    super(models.Model, self).__init__(
        inputs=inputs,
        outputs=outputs,
        name=name,
    )
    self.predict_function = None
    self.test_function = None
    self.train_function = None

  def compile(self,
              optimizer,
              loss=None,
              metrics=None,
              loss_weights=None,
              sample_weight_mode=None,
              weighted_metrics=None,
              target_tensors=None,
              **kwargs):
    if sample_weight_mode:
      raise ValueError('sample_weight_mode not supported for TPU execution.')
    if weighted_metrics:
      raise ValueError('weighted_metrics not supported for TPU execution.')
    if target_tensors:
      raise ValueError('target_tensors is not supported for TPU execution.')

    super(KerasTPUModel, self).compile(optimizer, loss, metrics, loss_weights,
                                       sample_weight_mode, weighted_metrics,
                                       target_tensors, **kwargs)

    # Keras optimizers are not compatible with TPU rewrite
    if not isinstance(self.optimizer, keras_optimizers.TFOptimizer):
      raise ValueError(
          'Optimizer must be a TFOptimizer, got: %s' % self.optimizer)

  def _make_train_function(self):
    if not self.train_function:
      self.train_function = TPUFunction(self, model_fn_lib.ModeKeys.TRAIN)

    return self.train_function

  def _make_test_function(self):
    if not self.test_function:
      self.test_function = TPUFunction(self, model_fn_lib.ModeKeys.EVAL)
    return self.test_function

  def _make_predict_function(self):
    if not self.predict_function:
      self.predict_function = TPUFunction(self, model_fn_lib.ModeKeys.PREDICT)
    return self.predict_function

  def cpu_model(self):
    cpu_model = models.Model(
        inputs=self.inputs,
        outputs=self.outputs,
        name=self.name,
    )

    if self.optimizer:
      cpu_model.compile(
          optimizer=self.optimizer,
          loss=self.loss,
          metrics=self.metrics,
          loss_weights=self.loss_weights,
      )

    return cpu_model

def _validate_shapes(model):
  """Validate that all layers in `model` have constant shape."""
  for layer in model.layers:
    if isinstance(layer.input_shape, tuple):
      input_shapes = [layer.input_shape]
    else:
      input_shapes = layer.input_shape

    if isinstance(layer.output_shape, tuple):
      output_shapes = [layer.output_shape]
    else:
      output_shapes = layer.output_shape

    for shape in input_shapes + output_shapes:
      for dim in shape[1:]:
        if dim is None:
          raise ValueError(
              """
Layer %(layer)s has a variable shape in a non-batch dimension.  TPU models must
have constant shapes for all operations.

You may have to specify `input_length` for RNN/TimeDistributed layers.

Layer: %(layer)s
Input shape: %(input_shape)s
Output shape: %(output_shape)s
  """ % {
      'layer': layer,
      'input_shape': layer.input_shape,
      'output_shape': layer.output_shape
      })

@experimental
def tpu_model(model):
  _validate_shapes(model)
  return KerasTPUModel(
      inputs=model.inputs, outputs=model.outputs, name=model.name)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2015, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------
"""
Depends on:
- https://github.com/numenta/nupic/pull/2491
- https://github.com/numenta/nupic/pull/2495
"""

import json
import operator
import os
import random
import sys
import time

import numpy

from nupic.data.inference_shifter import InferenceShifter
from nupic.frameworks.opf.modelfactory import ModelFactory
from nupic.research.monitor_mixin.trace import CountsTrace

from htmresearch.data.sequence_generator import SequenceGenerator

MIN_ORDER = 6
MAX_ORDER = 7
NUM_PREDICTIONS = [1, 2]
NUM_RANDOM = 1
PERTURB_AFTER = 10000

NUM_SYMBOLS = SequenceGenerator.numSymbols(MAX_ORDER, max(NUM_PREDICTIONS))
RANDOM_START = NUM_SYMBOLS
RANDOM_END = NUM_SYMBOLS + 1000

MODEL_PARAMS = {
  "model": "CLA",
  "version": 1,
  "predictAheadTime": None,
  "modelParams": {
    "inferenceType": "TemporalMultiStep",
    "sensorParams": {
      "verbosity" : 0,
      "encoders": {
        "element": {
          "fieldname": u"element",
          "name": u"element",
          "type": "SDRCategoryEncoder",
          "categoryList": range(max(RANDOM_END, NUM_SYMBOLS)),
          "n": 2048,
          "w": 41
        }
      },
      "sensorAutoReset" : None,
    },
      "spEnable": False,
      "spParams": {
        "spVerbosity" : 0,
        "globalInhibition": 1,
        "columnCount": 2048,
        "inputWidth": 0,
        "numActiveColumnsPerInhArea": 40,
        "seed": 1956,
        "columnDimensions": 0.5,
        "synPermConnected": 0.1,
        "synPermActiveInc": 0.1,
        "synPermInactiveDec": 0.01,
        "maxBoost": 0.0
    },
    "tpEnable" : True,
    "tpParams": {
      "verbosity": 0,
        "columnCount": 2048,
        "cellsPerColumn": 32,
        "inputWidth": 2048,
        "seed": 1960,
        "temporalImp": "monitored_tm_py",
        "newSynapseCount": 32,
        "maxSynapsesPerSegment": 40,
        "maxSegmentsPerCell": 128,
        "initialPerm": 0.31,
        "connectedPerm": 0.50,
        "permanenceInc": 0.1,
        "permanenceDec": 0.1,
        "predictedSegmentDecrement": 0.01,
        "globalDecay": 0.0,
        "maxAge": 0,
        "minThreshold": 18,
        "activationThreshold": 18,
        "outputType": "normal",
        "pamLength": 1,
      },
      "clParams": {
        "implementation": "cpp",
        "regionName" : "CLAClassifierRegion",
        "clVerbosity" : 0,
        "alpha": 0.0001,
        "steps": "1",
      },
      "trainSPNetOnlyIfRequested": False,
    },
}

def generateSequences(numPredictions, perturbed=False):
  sequences = []

  # # Generated sequences
  # generator = SequenceGenerator(seed=42)

  # for order in xrange(MIN_ORDER, MAX_ORDER+1):
  #   sequences += generator.generate(order, numPredictions)

  # # Subutai's sequences
  # """
  # Make sure to change parameter 'categoryList' above to: "categoryList": range(18)
  # """
  # sequences = [
  #   [0, 1, 2, 3, 4, 5],
  #   [6, 3, 2, 5, 1, 7],
  #   [8, 9, 10, 11, 12, 13],
  #   [14, 1, 2, 3, 15, 16],
  #   [17, 4, 2, 3, 1, 5]
  # ]

  # # Two orders of sequences
  # sequences = [
  #   [4, 2, 5, 0],
  #   [4, 5, 2, 3],
  #   [1, 2, 5, 3],
  #   [1, 5, 2, 0],
  #   [5, 3, 6, 2, 0],
  #   [5, 2, 6, 3, 4],
  #   [1, 3, 6, 2, 4],
  #   [1, 2, 6, 3, 0]
  # ]

  # # Two orders of sequences (easier)
  # # """
  # # Make sure to change parameter 'categoryList' above to: "categoryList": range(13)
  # # """
  # sequences = [
  #   [4, 2, 5, 0],
  #   [4, 5, 2, 3],
  #   [1, 2, 5, 3],
  #   [1, 5, 2, 0],
  #   [11, 9, 12, 8, 6],
  #   [11, 8, 12, 9, 10],
  #   [7, 9, 12, 8, 10],
  #   [7, 8, 12, 9, 6]
  # ]

  # # Two orders of sequences (isolating the problem)
  # sequences = [
  #   [1, 5, 2, 0],
  #   [5, 2, 6, 3, 4]
  # ]
  # random.seed(100) # 100 fails, 300 works (results depend on order of training)

  if numPredictions == 1:
    # Hardcoded set of sequences
    if perturbed:
      sequences = [
        [6, 8, 7, 4, 2, 3, 5],
        [1, 8, 7, 4, 2, 3, 0],
        [6, 3, 4, 2, 7, 8, 0],
        [1, 3, 4, 2, 7, 8, 5],
        [0, 9, 7, 8, 5, 3, 4, 6],
        [2, 9, 7, 8, 5, 3, 4, 1],
        [0, 4, 3, 5, 8, 7, 9, 1],
        [2, 4, 3, 5, 8, 7, 9, 6]
      ]
    else:
      sequences = [
        [6, 8, 7, 4, 2, 3, 0],
        [1, 8, 7, 4, 2, 3, 5],
        [6, 3, 4, 2, 7, 8, 5],
        [1, 3, 4, 2, 7, 8, 0],
        [0, 9, 7, 8, 5, 3, 4, 1],
        [2, 9, 7, 8, 5, 3, 4, 6],
        [0, 4, 3, 5, 8, 7, 9, 6],
        [2, 4, 3, 5, 8, 7, 9, 1]
      ]

  if numPredictions == 2:
    # Hardcoded set of sequences with multiple predictions (2)
    # Make sure to set NUM_PREDICTIONS = 2 above
    if perturbed:
      sequences = [
        [4, 8, 3, 10, 9, 6, 0],
        [4, 8, 3, 10, 9, 6, 7],
        [5, 8, 3, 10, 9, 6, 1],
        [5, 8, 3, 10, 9, 6, 2],
        [4, 6, 9, 10, 3, 8, 2],
        [4, 6, 9, 10, 3, 8, 1],
        [5, 6, 9, 10, 3, 8, 7],
        [5, 6, 9, 10, 3, 8, 0],
        [4, 3, 8, 6, 1, 10, 11, 0],
        [4, 3, 8, 6, 1, 10, 11, 7],
        [5, 3, 8, 6, 1, 10, 11, 9],
        [5, 3, 8, 6, 1, 10, 11, 2],
        [4, 11, 10, 1, 6, 8, 3, 2],
        [4, 11, 10, 1, 6, 8, 3, 9],
        [5, 11, 10, 1, 6, 8, 3, 7],
        [5, 11, 10, 1, 6, 8, 3, 0]
      ]
    else:
      sequences = [
        [4, 8, 3, 10, 9, 6, 1],
        [4, 8, 3, 10, 9, 6, 2],
        [5, 8, 3, 10, 9, 6, 0],
        [5, 8, 3, 10, 9, 6, 7],
        [4, 6, 9, 10, 3, 8, 7],
        [4, 6, 9, 10, 3, 8, 0],
        [5, 6, 9, 10, 3, 8, 2],
        [5, 6, 9, 10, 3, 8, 1],
        [4, 3, 8, 6, 1, 10, 11, 9],
        [4, 3, 8, 6, 1, 10, 11, 2],
        [5, 3, 8, 6, 1, 10, 11, 0],
        [5, 3, 8, 6, 1, 10, 11, 7],
        [4, 11, 10, 1, 6, 8, 3, 7],
        [4, 11, 10, 1, 6, 8, 3, 0],
        [5, 11, 10, 1, 6, 8, 3, 2],
        [5, 11, 10, 1, 6, 8, 3, 9]
      ]

  print "Sequences generated:"
  for sequence in sequences:
    print sequence
  print

  return sequences

def getEncoderMapping(model):
  encoder = model._getEncoder().encoders[0][1]
  mapping = dict()

  for i in range(NUM_SYMBOLS):
    mapping[i] = set(encoder.encode(i).nonzero()[0])

  return mapping

def classify(mapping, activeColumns, numPredictions):
  scores = [(len(encoding & activeColumns), i) for i, encoding in mapping.iteritems()]
  random.shuffle(scores)  # break ties randomly
  return [i for _, i in sorted(scores, reverse=True)[:numPredictions]]

class Runner(object):

  def __init__(self, numPredictions, resultsDir):
    self.numPredictions = numPredictions

    if not os.path.exists(resultsDir):
      os.makedirs(resultsDir)

    self.resultsFile = open(os.path.join(resultsDir, "0.log"), 'w')

    self.model = ModelFactory.create(MODEL_PARAMS)
    self.model.enableInference({"predictedField": "element"})
    self.shifter = InferenceShifter()
    self.mapping = getEncoderMapping(self.model)

    self.sequences = generateSequences(self.numPredictions)
    self.correct = []
    self.numPredictedActiveCells = []
    self.numPredictedInactiveCells = []
    self.numUnpredictedActiveColumns = []

    self.currentSequence = random.choice(self.sequences)
    self.iteration = 0
    self.perturbed = False
    self.randoms = []

  def step(self):
    element = self.currentSequence.pop(0)

    randomFlag = (len(self.currentSequence) == 0)
    self.randoms.append(randomFlag)

    if len(self.currentSequence) == 0:
      if randomFlag:
        self.currentSequence.append(random.randrange(RANDOM_START, RANDOM_END))

      if self.iteration > PERTURB_AFTER and not self.perturbed:
        print "PERTURBING"
        self.sequences = generateSequences(self.numPredictions, perturbed=True)
        self.perturbed = True

      sequence = random.choice(self.sequences)

      self.currentSequence += sequence

    result = self.shifter.shift(self.model.run({"element": element}))
    tm = self.model._getTPRegion().getSelf()._tfdr

    tm.mmClearHistory()
    # Use custom classifier (uses predicted cells to make predictions)
    predictiveColumns = set([tm.columnForCell(cell) for cell in tm.predictiveCells])
    topPredictions = classify(self.mapping, predictiveColumns, self.numPredictions)

    truth = None if (self.randoms[-1] or
                     len(self.randoms) >= 2 and self.randoms[-2]) else self.currentSequence[0]

    data = {"iteration": self.iteration,
            "current": element,
            "reset": False,
            "random": randomFlag,
            "train": True,
            "predictions": topPredictions,
            "truth": truth}

    self.resultsFile.write(json.dumps(data) + '\n')
    self.resultsFile.flush()

    self.iteration += 1

if __name__ == "__main__":
  outdir = sys.argv[1]

  if not os.path.exists(outdir):
    os.makedirs(outdir)

  runners = []

  for numPredictions in NUM_PREDICTIONS:
    resultsDir = os.path.join(outdir, "num_predictions{0}".format(numPredictions))
    runners.append(Runner(numPredictions, resultsDir))

  for i in iter(int, 1):
    for runner in runners:
      runner.step()
<commit_msg>Revert "Try same TM params as continuous case"<commit_after>#!/usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2015, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------
"""
Depends on:
- https://github.com/numenta/nupic/pull/2491
- https://github.com/numenta/nupic/pull/2495
"""

import json
import operator
import os
import random
import sys
import time

import numpy

from nupic.data.inference_shifter import InferenceShifter
from nupic.frameworks.opf.modelfactory import ModelFactory
from nupic.research.monitor_mixin.trace import CountsTrace

from htmresearch.data.sequence_generator import SequenceGenerator

MIN_ORDER = 6
MAX_ORDER = 7
NUM_PREDICTIONS = [1, 2]
NUM_RANDOM = 1
PERTURB_AFTER = 10000

NUM_SYMBOLS = SequenceGenerator.numSymbols(MAX_ORDER, max(NUM_PREDICTIONS))
RANDOM_START = NUM_SYMBOLS
RANDOM_END = NUM_SYMBOLS + 1000

MODEL_PARAMS = {
  "model": "CLA",
  "version": 1,
  "predictAheadTime": None,
  "modelParams": {
    "inferenceType": "TemporalMultiStep",
    "sensorParams": {
      "verbosity" : 0,
      "encoders": {
        "element": {
          "fieldname": u"element",
          "name": u"element",
          "type": "SDRCategoryEncoder",
          "categoryList": range(max(RANDOM_END, NUM_SYMBOLS)),
          "n": 2048,
          "w": 41
        }
      },
      "sensorAutoReset" : None,
    },
      "spEnable": False,
      "spParams": {
        "spVerbosity" : 0,
        "globalInhibition": 1,
        "columnCount": 2048,
        "inputWidth": 0,
        "numActiveColumnsPerInhArea": 40,
        "seed": 1956,
        "columnDimensions": 0.5,
        "synPermConnected": 0.1,
        "synPermActiveInc": 0.1,
        "synPermInactiveDec": 0.01,
        "maxBoost": 0.0
    },
    "tpEnable" : True,
    "tpParams": {
      "verbosity": 0,
        "columnCount": 2048,
        "cellsPerColumn": 32,
        "inputWidth": 2048,
        "seed": 1960,
        "temporalImp": "monitored_tm_py",
        "newSynapseCount": 20,
        "maxSynapsesPerSegment": 128,
        "maxSegmentsPerCell": 128,
        "initialPerm": 0.21,
        "connectedPerm": 0.50,
        "permanenceInc": 0.1,
        "permanenceDec": 0.1,
        "predictedSegmentDecrement": 0.01,
        "globalDecay": 0.0,
        "maxAge": 0,
        "minThreshold": 15,
        "activationThreshold": 15,
        "outputType": "normal",
        "pamLength": 1,
      },
      "clParams": {
        "implementation": "cpp",
        "regionName" : "CLAClassifierRegion",
        "clVerbosity" : 0,
        "alpha": 0.0001,
        "steps": "1",
      },
      "trainSPNetOnlyIfRequested": False,
    },
}

def generateSequences(numPredictions, perturbed=False):
  sequences = []

  # # Generated sequences
  # generator = SequenceGenerator(seed=42)

  # for order in xrange(MIN_ORDER, MAX_ORDER+1):
  #   sequences += generator.generate(order, numPredictions)

  # # Subutai's sequences
  # """
  # Make sure to change parameter 'categoryList' above to: "categoryList": range(18)
  # """
  # sequences = [
  #   [0, 1, 2, 3, 4, 5],
  #   [6, 3, 2, 5, 1, 7],
  #   [8, 9, 10, 11, 12, 13],
  #   [14, 1, 2, 3, 15, 16],
  #   [17, 4, 2, 3, 1, 5]
  # ]

  # # Two orders of sequences
  # sequences = [
  #   [4, 2, 5, 0],
  #   [4, 5, 2, 3],
  #   [1, 2, 5, 3],
  #   [1, 5, 2, 0],
  #   [5, 3, 6, 2, 0],
  #   [5, 2, 6, 3, 4],
  #   [1, 3, 6, 2, 4],
  #   [1, 2, 6, 3, 0]
  # ]

  # # Two orders of sequences (easier)
  # # """
  # # Make sure to change parameter 'categoryList' above to: "categoryList": range(13)
  # # """
  # sequences = [
  #   [4, 2, 5, 0],
  #   [4, 5, 2, 3],
  #   [1, 2, 5, 3],
  #   [1, 5, 2, 0],
  #   [11, 9, 12, 8, 6],
  #   [11, 8, 12, 9, 10],
  #   [7, 9, 12, 8, 10],
  #   [7, 8, 12, 9, 6]
  # ]

  # # Two orders of sequences (isolating the problem)
  # sequences = [
  #   [1, 5, 2, 0],
  #   [5, 2, 6, 3, 4]
  # ]
  # random.seed(100) # 100 fails, 300 works (results depend on order of training)

  if numPredictions == 1:
    # Hardcoded set of sequences
    if perturbed:
      sequences = [
        [6, 8, 7, 4, 2, 3, 5],
        [1, 8, 7, 4, 2, 3, 0],
        [6, 3, 4, 2, 7, 8, 0],
        [1, 3, 4, 2, 7, 8, 5],
        [0, 9, 7, 8, 5, 3, 4, 6],
        [2, 9, 7, 8, 5, 3, 4, 1],
        [0, 4, 3, 5, 8, 7, 9, 1],
        [2, 4, 3, 5, 8, 7, 9, 6]
      ]
    else:
      sequences = [
        [6, 8, 7, 4, 2, 3, 0],
        [1, 8, 7, 4, 2, 3, 5],
        [6, 3, 4, 2, 7, 8, 5],
        [1, 3, 4, 2, 7, 8, 0],
        [0, 9, 7, 8, 5, 3, 4, 1],
        [2, 9, 7, 8, 5, 3, 4, 6],
        [0, 4, 3, 5, 8, 7, 9, 6],
        [2, 4, 3, 5, 8, 7, 9, 1]
      ]

  if numPredictions == 2:
    # Hardcoded set of sequences with multiple predictions (2)
    # Make sure to set NUM_PREDICTIONS = 2 above
    if perturbed:
      sequences = [
        [4, 8, 3, 10, 9, 6, 0],
        [4, 8, 3, 10, 9, 6, 7],
        [5, 8, 3, 10, 9, 6, 1],
        [5, 8, 3, 10, 9, 6, 2],
        [4, 6, 9, 10, 3, 8, 2],
        [4, 6, 9, 10, 3, 8, 1],
        [5, 6, 9, 10, 3, 8, 7],
        [5, 6, 9, 10, 3, 8, 0],
        [4, 3, 8, 6, 1, 10, 11, 0],
        [4, 3, 8, 6, 1, 10, 11, 7],
        [5, 3, 8, 6, 1, 10, 11, 9],
        [5, 3, 8, 6, 1, 10, 11, 2],
        [4, 11, 10, 1, 6, 8, 3, 2],
        [4, 11, 10, 1, 6, 8, 3, 9],
        [5, 11, 10, 1, 6, 8, 3, 7],
        [5, 11, 10, 1, 6, 8, 3, 0]
      ]
    else:
      sequences = [
        [4, 8, 3, 10, 9, 6, 1],
        [4, 8, 3, 10, 9, 6, 2],
        [5, 8, 3, 10, 9, 6, 0],
        [5, 8, 3, 10, 9, 6, 7],
        [4, 6, 9, 10, 3, 8, 7],
        [4, 6, 9, 10, 3, 8, 0],
        [5, 6, 9, 10, 3, 8, 2],
        [5, 6, 9, 10, 3, 8, 1],
        [4, 3, 8, 6, 1, 10, 11, 9],
        [4, 3, 8, 6, 1, 10, 11, 2],
        [5, 3, 8, 6, 1, 10, 11, 0],
        [5, 3, 8, 6, 1, 10, 11, 7],
        [4, 11, 10, 1, 6, 8, 3, 7],
        [4, 11, 10, 1, 6, 8, 3, 0],
        [5, 11, 10, 1, 6, 8, 3, 2],
        [5, 11, 10, 1, 6, 8, 3, 9]
      ]

  print "Sequences generated:"
  for sequence in sequences:
    print sequence
  print

  return sequences

def getEncoderMapping(model):
  encoder = model._getEncoder().encoders[0][1]
  mapping = dict()

  for i in range(NUM_SYMBOLS):
    mapping[i] = set(encoder.encode(i).nonzero()[0])

  return mapping

def classify(mapping, activeColumns, numPredictions):
  scores = [(len(encoding & activeColumns), i) for i, encoding in mapping.iteritems()]
  random.shuffle(scores)  # break ties randomly
  return [i for _, i in sorted(scores, reverse=True)[:numPredictions]]

class Runner(object):

  def __init__(self, numPredictions, resultsDir):
    self.numPredictions = numPredictions

    if not os.path.exists(resultsDir):
      os.makedirs(resultsDir)

    self.resultsFile = open(os.path.join(resultsDir, "0.log"), 'w')

    self.model = ModelFactory.create(MODEL_PARAMS)
    self.model.enableInference({"predictedField": "element"})
    self.shifter = InferenceShifter()
    self.mapping = getEncoderMapping(self.model)

    self.sequences = generateSequences(self.numPredictions)
    self.correct = []
    self.numPredictedActiveCells = []
    self.numPredictedInactiveCells = []
    self.numUnpredictedActiveColumns = []

    self.currentSequence = random.choice(self.sequences)
    self.iteration = 0
    self.perturbed = False
    self.randoms = []

  def step(self):
    element = self.currentSequence.pop(0)

    randomFlag = (len(self.currentSequence) == 0)
    self.randoms.append(randomFlag)

    if len(self.currentSequence) == 0:
      if randomFlag:
        self.currentSequence.append(random.randrange(RANDOM_START, RANDOM_END))

      if self.iteration > PERTURB_AFTER and not self.perturbed:
        print "PERTURBING"
        self.sequences = generateSequences(self.numPredictions, perturbed=True)
        self.perturbed = True

      sequence = random.choice(self.sequences)

      self.currentSequence += sequence

    result = self.shifter.shift(self.model.run({"element": element}))
    tm = self.model._getTPRegion().getSelf()._tfdr

    tm.mmClearHistory()
    # Use custom classifier (uses predicted cells to make predictions)
    predictiveColumns = set([tm.columnForCell(cell) for cell in tm.predictiveCells])
    topPredictions = classify(self.mapping, predictiveColumns, self.numPredictions)

    truth = None if (self.randoms[-1] or
                     len(self.randoms) >= 2 and self.randoms[-2]) else self.currentSequence[0]

    data = {"iteration": self.iteration,
            "current": element,
            "reset": False,
            "random": randomFlag,
            "train": True,
            "predictions": topPredictions,
            "truth": truth}

    self.resultsFile.write(json.dumps(data) + '\n')
    self.resultsFile.flush()

    self.iteration += 1

if __name__ == "__main__":
  outdir = sys.argv[1]

  if not os.path.exists(outdir):
    os.makedirs(outdir)

  runners = []

  for numPredictions in NUM_PREDICTIONS:
    resultsDir = os.path.join(outdir, "num_predictions{0}".format(numPredictions))
    runners.append(Runner(numPredictions, resultsDir))

  for i in iter(int, 1):
    for runner in runners:
      runner.step()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from typing import Tuple, Optional, Union

from hwt.code import If
from hwt.hdl.types.defs import BIT
from hwt.hdl.types.struct import HStruct
from hwt.interfaces.std import Signal, Rst_n, Rst, Clk, VectSignal
from hwt.interfaces.utils import propagateClkRstn
from hwt.serializer.mode import serializeParamsUniq
from hwt.synthesizer.interfaceLevel.interfaceUtils.utils import packIntf, \
    connectPacked
from hwtLib.amba.axis_comp.base import AxiSCompBase
from hwtLib.amba.axis_comp.reg import AxiSReg
from hwtLib.handshaked.fifo import HandshakedFifo
from hwtLib.mem.fifoCopy import FifoCopy

@serializeParamsUniq
class AxiSRegCopy(AxiSCompBase, HandshakedFifo):
    """
    Same thing as a :class:`~.AxiSFifoCopy`
    just uses registers without fifo pointers

    .. hwt-autodoc::
    """

    def _declr(self):
        AxiSFifoCopy._declr_io(self)
        with self._paramsShared():
            self.reg = AxiSReg(self.intfCls)

    def _impl(self):
        reg = self.reg
        copy_en = self.dataOut_copy_frame & self.dataOut.valid
        non_data_signals = [reg.dataIn.ready, reg.dataIn.valid]
        if self.ID_WIDTH > 0:
            non_data_signals.append(reg.dataIn.id)
            If(self.dataOut_copy_frame,
               reg.dataIn.id(self.dataOut_replacement_id),
            ).Else(
              reg.dataIn.id(self.dataIn.id)
            )
        If(copy_en,
            reg.dataIn(reg.dataOut, exclude=non_data_signals)
        ).Else(
            reg.dataIn(self.dataIn, exclude=non_data_signals)
        )

        reg.dataIn.valid(copy_en | self.dataIn.valid)
        self.dataIn.ready(reg.dataIn.ready & ~copy_en)

        #self.dataOut(reg.dataOut, exclude=[reg.dataOut.ready, reg.dataOut.valid])
        self.dataOut(reg.dataOut)

        propagateClkRstn(self)

@serializeParamsUniq
class AxiSFifoCopy(AxiSCompBase, HandshakedFifo):
    """
    Synchronous fifo for axi-stream interface which can copy last frame or work
    as a regular fifo.

    :note: DEPTH > axis.MAX_FRAME_LEN

    :see: :class:`hwtLib.handshaked.fifo_copy.HandshakedFifoCopy`

    .. hwt-autodoc:: _example_AxiSFifoCopy
    """
    FIFO_CLS = FifoCopy

    def _declr_io(self):
        HandshakedFifo._declr_io(self)
        # these signals are sudes sorely in dataOut beat which has last=1
        self.dataOut_copy_frame = Signal()
        if self.ID_WIDTH > 0:
            self.dataOut_replacement_id = VectSignal(self.ID_WIDTH)

    def _declr(self)->None:
        assert self.DEPTH > 1 ,\
            "Fifo is too small, fifo pointers would not work correctly, use register(s) instead"
        HandshakedFifo._declr(self)

    def _impl(self, clk_rst: Optional[Tuple[
            Tuple[Clk, Union[Rst, Rst_n]],
            Tuple[Clk, Union[Rst, Rst_n]]]]=None):
        HandshakedFifo._impl(self, clk_rst=clk_rst)

    def _connect_fifo_in(self):
        rd = self.get_ready_signal
        vld = self.get_valid_signal
        din = self.dataIn
        fIn = self.fifo.dataIn
        wr_en = ~fIn.wait

        rd(din)(wr_en)
        fIn.data(packIntf(din, exclude=[vld(din), rd(din)]))
        fIn.en(vld(din) & wr_en)

    def _connect_fifo_out(self, out_clk, out_rst):
        rd = self.get_ready_signal
        vld = self.get_valid_signal

        fOut = self.fifo.dataOut
        dout = self.dataOut

        out_vld = self._reg("out_vld", def_val=0, clk=out_clk, rst=out_rst)
        vld(dout)(out_vld)
        non_data = [vld(dout), rd(dout),]
        data_connections = connectPacked(fOut.data,
                      dout,
                      exclude=non_data)
        if self.ID_WIDTH > 0:
            replace_id = self._reg("replace_id",
                HStruct(
                    (dout.id._dtype, "val"),
                    (BIT, "vld")
                ),
                def_val={"vld": 0}
            )
            id_from_fifo = [a for a in data_connections if a.dst is dout.id._sig]
            assert id_from_fifo
            If(replace_id.vld,
               dout.id(replace_id.val)
            ).Else(
                *id_from_fifo
            )
            If(dout.last & rd(dout) & vld(dout),
               replace_id.vld(self.dataOut_copy_frame),
               replace_id.val(self.dataOut_replacement_id),
            )

        fOut.en((rd(dout) | ~out_vld) & ~fOut.wait)
        If(rd(dout) | ~out_vld,
           out_vld(~fOut.wait)
        )
        self.fifo.dataOut_copy_frame.vld(out_vld & dout.last & rd(dout))
        self.fifo.dataOut_copy_frame.data(self.dataOut_copy_frame)

        return out_vld

def _example_AxiSFifoCopy():
    u = AxiSFifoCopy()
    u.DEPTH = 4
    u.ID_WIDTH = 2
    # u.EXPORT_SIZE = True
    # u.EXPORT_SPACE = True
    return u

if __name__ == "__main__":
    from hwt.synthesizer.utils import to_rtl_str
    u = _example_AxiSFifoCopy()

    print(to_rtl_str(u))
<commit_msg>grammar fix<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from typing import Tuple, Optional, Union

from hwt.code import If
from hwt.hdl.types.defs import BIT
from hwt.hdl.types.struct import HStruct
from hwt.interfaces.std import Signal, Rst_n, Rst, Clk, VectSignal
from hwt.interfaces.utils import propagateClkRstn
from hwt.serializer.mode import serializeParamsUniq
from hwt.synthesizer.interfaceLevel.interfaceUtils.utils import packIntf, \
    connectPacked
from hwtLib.amba.axis_comp.base import AxiSCompBase
from hwtLib.amba.axis_comp.reg import AxiSReg
from hwtLib.handshaked.fifo import HandshakedFifo
from hwtLib.mem.fifoCopy import FifoCopy

@serializeParamsUniq
class AxiSRegCopy(AxiSCompBase, HandshakedFifo):
    """
    Same thing as a :class:`~.AxiSFifoCopy`
    just uses registers without fifo pointers

    .. hwt-autodoc::
    """

    def _declr(self):
        AxiSFifoCopy._declr_io(self)
        with self._paramsShared():
            self.reg = AxiSReg(self.intfCls)

    def _impl(self):
        reg = self.reg
        copy_en = self.dataOut_copy_frame & self.dataOut.valid
        non_data_signals = [reg.dataIn.ready, reg.dataIn.valid]
        if self.ID_WIDTH > 0:
            non_data_signals.append(reg.dataIn.id)
            If(self.dataOut_copy_frame,
               reg.dataIn.id(self.dataOut_replacement_id),
            ).Else(
              reg.dataIn.id(self.dataIn.id)
            )
        If(copy_en,
            reg.dataIn(reg.dataOut, exclude=non_data_signals)
        ).Else(
            reg.dataIn(self.dataIn, exclude=non_data_signals)
        )

        reg.dataIn.valid(copy_en | self.dataIn.valid)
        self.dataIn.ready(reg.dataIn.ready & ~copy_en)

        #self.dataOut(reg.dataOut, exclude=[reg.dataOut.ready, reg.dataOut.valid])
        self.dataOut(reg.dataOut)

        propagateClkRstn(self)

@serializeParamsUniq
class AxiSFifoCopy(AxiSCompBase, HandshakedFifo):
    """
    Synchronous fifo for axi-stream interface which can copy last frame or work
    as a regular fifo.

    :note: DEPTH > axis.MAX_FRAME_LEN

    :see: :class:`hwtLib.handshaked.fifo_copy.HandshakedFifoCopy`

    .. hwt-autodoc:: _example_AxiSFifoCopy
    """
    FIFO_CLS = FifoCopy

    def _declr_io(self):
        HandshakedFifo._declr_io(self)
        # these signals are used sorely in dataOut beat which has last=1
        self.dataOut_copy_frame = Signal()
        if self.ID_WIDTH > 0:
            self.dataOut_replacement_id = VectSignal(self.ID_WIDTH)

    def _declr(self)->None:
        assert self.DEPTH > 1 ,\
            "Fifo is too small, fifo pointers would not work correctly, use register(s) instead"
        HandshakedFifo._declr(self)

    def _impl(self, clk_rst: Optional[Tuple[
            Tuple[Clk, Union[Rst, Rst_n]],
            Tuple[Clk, Union[Rst, Rst_n]]]]=None):
        HandshakedFifo._impl(self, clk_rst=clk_rst)

    def _connect_fifo_in(self):
        rd = self.get_ready_signal
        vld = self.get_valid_signal
        din = self.dataIn
        fIn = self.fifo.dataIn
        wr_en = ~fIn.wait

        rd(din)(wr_en)
        fIn.data(packIntf(din, exclude=[vld(din), rd(din)]))
        fIn.en(vld(din) & wr_en)

    def _connect_fifo_out(self, out_clk, out_rst):
        rd = self.get_ready_signal
        vld = self.get_valid_signal

        fOut = self.fifo.dataOut
        dout = self.dataOut

        out_vld = self._reg("out_vld", def_val=0, clk=out_clk, rst=out_rst)
        vld(dout)(out_vld)
        non_data = [vld(dout), rd(dout),]
        data_connections = connectPacked(fOut.data,
                      dout,
                      exclude=non_data)
        if self.ID_WIDTH > 0:
            replace_id = self._reg("replace_id",
                HStruct(
                    (dout.id._dtype, "val"),
                    (BIT, "vld")
                ),
                def_val={"vld": 0}
            )
            id_from_fifo = [a for a in data_connections if a.dst is dout.id._sig]
            assert id_from_fifo
            If(replace_id.vld,
               dout.id(replace_id.val)
            ).Else(
                *id_from_fifo
            )
            If(dout.last & rd(dout) & vld(dout),
               replace_id.vld(self.dataOut_copy_frame),
               replace_id.val(self.dataOut_replacement_id),
            )

        fOut.en((rd(dout) | ~out_vld) & ~fOut.wait)
        If(rd(dout) | ~out_vld,
           out_vld(~fOut.wait)
        )
        self.fifo.dataOut_copy_frame.vld(out_vld & dout.last & rd(dout))
        self.fifo.dataOut_copy_frame.data(self.dataOut_copy_frame)

        return out_vld

def _example_AxiSFifoCopy():
    u = AxiSFifoCopy()
    u.DEPTH = 4
    u.ID_WIDTH = 2
    # u.EXPORT_SIZE = True
    # u.EXPORT_SPACE = True
    return u

if __name__ == "__main__":
    from hwt.synthesizer.utils import to_rtl_str
    u = _example_AxiSFifoCopy()

    print(to_rtl_str(u))
<|endoftext|>


<commit_before># @ 2021 KMEE - kmee.com.br
#   Luis Felipe Mileo <mileo@kmee.com.br>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)

import unittest

from odoo.tests import TransactionCase
from odoo.exceptions import ValidationError

class TestPaymentMode(TransactionCase):

    @classmethod
    def setUpClass(cls):
        super().setUpClass()

        if not cls.env['account.account'].search([]):
            raise unittest.SkipTest('No accounts defined')

        cls.company = cls.env.ref('base.main_company')
        cls.journal_c1 = cls.env['account.journal'].create({
            'name': 'Journal 1',
            'code': 'J1',
            'type': 'bank',
            'company_id': cls.company.id,
        })

        cls.account = cls.env['account.account'].search([
            ('reconcile', '=', True),
            ('company_id', '=', cls.company.id)
        ], limit=1)

        cls.type_240 = cls.env.ref(
            'l10n_br_account_payment_order.payment_mode_type_cnab240')

        cls.type_400 = cls.env.ref(
            'l10n_br_account_payment_order.payment_mode_type_cnab400')

        cls.type_500 = cls.env.ref(
            'l10n_br_account_payment_order.payment_mode_type_cnab500')

        cls.payment_mode_test_240 = cls.env['account.payment.mode'].create({
            'name': 'Banco Teste 240',
            'bank_account_link': 'fixed',
            'payment_method_id': cls.type_240.id,
            'company_id': cls.company.id,
            'fixed_journal_id': cls.journal_c1.id,
            'group_lines': False,
            'generate_move': False,
            'post_move': False,
        })

        cls.payment_mode_test_400 = cls.env['account.payment.mode'].create({
            'name': 'Banco Teste 400',
            'bank_account_link': 'fixed',
            'payment_method_id': cls.type_400.id,
            'company_id': cls.company.id,
            'fixed_journal_id': cls.journal_c1.id,
            'group_lines': False,
            'generate_move': False,
            'post_move': False,

        })

        cls.payment_mode_test_500 = cls.env['account.payment.mode'].create({
            'name': 'Banco Teste 500',
            'bank_account_link': 'fixed',
            'payment_method_id': cls.type_500.id,
            'company_id': cls.company.id,
            'fixed_journal_id': cls.journal_c1.id,
            'group_lines': False,
            'generate_move': False,
            'post_move': False,
        })

    def test_constrains(self):
        with self.assertRaises(ValidationError):
            self.payment_mode_test_240.write({
                'group_lines': True,
            })
        with self.assertRaises(ValidationError):
            self.payment_mode_test_240.write({
                'generate_move': True,
            })
        with self.assertRaises(ValidationError):
            self.payment_mode_test_240.write({
                'post_move': True,
            })
<commit_msg>[FIX] Test class definition.<commit_after># @ 2021 KMEE - kmee.com.br
#   Luis Felipe Mileo <mileo@kmee.com.br>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)

import unittest

from odoo.tests import SavepointCase
from odoo.exceptions import ValidationError

class TestPaymentMode(SavepointCase):

    @classmethod
    def setUpClass(cls):
        super().setUpClass()

        if not cls.env['account.account'].search([]):
            raise unittest.SkipTest('No accounts defined')

        cls.company = cls.env.ref('base.main_company')
        cls.journal_c1 = cls.env['account.journal'].create({
            'name': 'Journal 1',
            'code': 'J1',
            'type': 'bank',
            'company_id': cls.company.id,
        })

        cls.account = cls.env['account.account'].search([
            ('reconcile', '=', True),
            ('company_id', '=', cls.company.id)
        ], limit=1)

        cls.type_240 = cls.env.ref(
            'l10n_br_account_payment_order.payment_mode_type_cnab240')

        cls.type_400 = cls.env.ref(
            'l10n_br_account_payment_order.payment_mode_type_cnab400')

        cls.type_500 = cls.env.ref(
            'l10n_br_account_payment_order.payment_mode_type_cnab500')

        cls.payment_mode_test_240 = cls.env['account.payment.mode'].create({
            'name': 'Banco Teste 240',
            'bank_account_link': 'fixed',
            'payment_method_id': cls.type_240.id,
            'company_id': cls.company.id,
            'fixed_journal_id': cls.journal_c1.id,
            'group_lines': False,
            'generate_move': False,
            'post_move': False,
        })

        cls.payment_mode_test_400 = cls.env['account.payment.mode'].create({
            'name': 'Banco Teste 400',
            'bank_account_link': 'fixed',
            'payment_method_id': cls.type_400.id,
            'company_id': cls.company.id,
            'fixed_journal_id': cls.journal_c1.id,
            'group_lines': False,
            'generate_move': False,
            'post_move': False,

        })

        cls.payment_mode_test_500 = cls.env['account.payment.mode'].create({
            'name': 'Banco Teste 500',
            'bank_account_link': 'fixed',
            'payment_method_id': cls.type_500.id,
            'company_id': cls.company.id,
            'fixed_journal_id': cls.journal_c1.id,
            'group_lines': False,
            'generate_move': False,
            'post_move': False,
        })

    def test_constrains(self):
        with self.assertRaises(ValidationError):
            self.payment_mode_test_240.write({
                'group_lines': True,
            })
        with self.assertRaises(ValidationError):
            self.payment_mode_test_240.write({
                'generate_move': True,
            })
        with self.assertRaises(ValidationError):
            self.payment_mode_test_240.write({
                'post_move': True,
            })
<|endoftext|>


<commit_before>
"""
Test the debug message interface.
"""

import dbus

from gabbletest import exec_test
import constants as cs

path = '/org/freedesktop/Telepathy/debug'
iface = 'org.freedesktop.Telepathy.Debug'

def test(q, bus, conn, stream):
    messages = []

    def new_message(timestamp, string):
        messages.append((timestamp, string))

    debug = bus.get_object(conn.bus_name, path)
    debug_iface = dbus.Interface(debug, iface)
    debug_iface.connect_to_signal('NewDebugMessage', new_message)
    props_iface = dbus.Interface(debug, cs.PROPERTIES_IFACE)

    assert len(debug.GetMessages()) > 0

    # Turn signalling on and generate some messages.

    assert len(messages) == 0
    conn.Connect()
    q.expect('dbus-signal', signal='StatusChanged', args=[0, 1])
    assert props_iface.Get(iface, 'Enabled') == False
    props_iface.Set(iface, 'Enabled', True)

    conn.RequestChannel(
        cs.CHANNEL_TYPE_TEXT, cs.HT_CONTACT, conn.GetSelfHandle(), True)
    q.expect('dbus-signal', signal='NewChannel')
    assert len(messages) > 0

    # Turn signalling off and check we have no new messages.

    props_iface.Set(iface, 'Enabled', False)
    snapshot = list(messages)

    conn.Disconnect()
    q.expect('dbus-signal', signal='StatusChanged', args=[2, 1])

    assert snapshot == messages

if __name__ == '__main__':
    exec_test(test)

<commit_msg>test-debug.py: Fix to use correct iface and number of NewDebugMessage args.<commit_after>
"""
Test the debug message interface.
"""

import dbus

from gabbletest import exec_test
import constants as cs

path = '/org/freedesktop/Telepathy/debug'
iface = 'org.freedesktop.Telepathy.Debug.DRAFT'

def test(q, bus, conn, stream):
    messages = []

    def new_message(timestamp, domain, level, string):
        messages.append((timestamp, domain, level, string))

    debug = bus.get_object(conn.bus_name, path)
    debug_iface = dbus.Interface(debug, iface)
    debug_iface.connect_to_signal('NewDebugMessage', new_message)
    props_iface = dbus.Interface(debug, cs.PROPERTIES_IFACE)

    assert len(debug.GetMessages()) > 0

    # Turn signalling on and generate some messages.

    assert len(messages) == 0
    conn.Connect()
    q.expect('dbus-signal', signal='StatusChanged', args=[0, 1])
    assert props_iface.Get(iface, 'Enabled') == False
    props_iface.Set(iface, 'Enabled', True)

    conn.RequestChannel(
        cs.CHANNEL_TYPE_TEXT, cs.HT_CONTACT, conn.GetSelfHandle(), True)
    q.expect('dbus-signal', signal='NewChannel')
    assert len(messages) > 0

    # Turn signalling off and check we have no new messages.

    props_iface.Set(iface, 'Enabled', False)
    snapshot = list(messages)

    conn.Disconnect()
    q.expect('dbus-signal', signal='StatusChanged', args=[2, 1])

    assert snapshot == messages

if __name__ == '__main__':
    exec_test(test)

<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
# vim:fenc=utf-8
#
# Copyright © %YEAR%  <>
#
# Distributed under terms of the %LICENSE% license.

"""

"""

import logging

logger = logging.getLogger(__name__)
import argparse
# conda install -c conda-forge begins
# import begin

import PyQt4
from PyQt4.QtGui import QGridLayout, QLabel,\
    QPushButton, QLineEdit, QApplication, QWidget, QGridLayout, QSpinBox, QLineEdit, QCheckBox,\
        QComboBox, QTextEdit, QDialog, QMainWindow, QDoubleSpinBox

from PyQt4 import QtGui
import sys
import os.path
import copy

from .. import dictwidgetqt
import cylinders

from pyqtconfig import ConfigManager
import inspect
import collections

def get_default_args(obj):
    argspec = inspect.getargspec(obj.__init__)
    args = argspec.args[1:]
    defaults = argspec.defaults
    print "---- args"
    print args
    print defaults
    # import ipdb; ipdb.set_trace() #  noqa BREAKPOINT
    # dc = dict(zip(args, defaults))
    dc = collections.OrderedDict(zip(args, defaults))
    return dc

class CylindersWidget(QtGui.QWidget):
    def __init__(self, config_in, ncols=2):
        super(CylindersWidget, self).__init__()
        self.config_in = config_in
        self.ncols = ncols

        self.config = get_default_args(cylinders.CylinderGenerator)
        print "default args"
        print self.config
        self.init_ui()

    def run(self):
        print "generator args"
        new_cfg = self.configwg.config_as_dict()
        logger.debug(str(new_cfg))
        self.config = new_cfg
        print self.config
        self.gen = cylinders.CylinderGenerator(**self.config)
        self.gen.run()

    def _show_stats(self):
        df = self.gen.getStats()
        from .. import tablewidget

        dfmerne = df[["length", "volume", "surface"]].sum() / self.gen.area_volume
        print "merne"
        print dfmerne
        dfmernef = dfmerne.to_frame()
        dfmernef.insert(0, "", dfmernef.index)
        # import ipdb; ipdb.set_trace()
        tw = tablewidget.TableWidget(self, dataframe=dfmernef)
        self.mainLayout.addWidget(tw)

        from matplotlib.backends.backend_qt4agg import FigureCanvasQTAgg as FigureCanvas
        # from matplotlib.backends.backend_qt4agg import NavigationToolbar2QTAgg as NavigationToolbar
        import matplotlib.pyplot as plt

        self.figure = plt.figure()
        self.canvas = FigureCanvas(self.figure)
        # self.toolbar = NavigationToolbar(self.canvas, self)
        self.mainLayout.addWidget(self.canvas)
        plt.subplot(141)
        df[["length"]].boxplot()
        plt.subplot(142)
        df[['radius']].boxplot()

        plt.subplot(143)
        df[["surface"]].boxplot()

        plt.subplot(144)
        df[["volume"]].boxplot()

        # TODO take care about redrawing
        dfdescribe = df.describe()
        dfdescribe.insert(0, "", dfdescribe.index)
        tw = tablewidget.TableWidget(self, dataframe=dfdescribe)
        tw.show()
        tw.raise_()

        self.mainLayout.addWidget(tw)
        self.resize(600,700)

    def complicated_to_yaml(self, cfg):
        import yaml
        # convert values to json
        isconverted = {}
        for key, value in cfg.iteritems():
            if type(value) in (str, int, float, bool):

                isconverted[key] = False
                if type(value) is str:
                    pass

            else:
                isconverted[key] = True
                cfg[key] = yaml.dump(value, default_flow_style=True)
        return cfg

    def init_ui(self):
        self.mainLayout = QGridLayout(self)
        hide_keys = ["build", "gtree"]

        self.configwg = dictwidgetqt.DictWidget(self.config, hide_keys=hide_keys)
        self.mainLayout.addWidget(self.configwg)

        # self.mainLayout.setColumnMinimumWidth(text_col, 500)

        btn_accept = QPushButton("Run", self)
        btn_accept.clicked.connect(self.btnAccept)
        self.mainLayout.addWidget(btn_accept) # , (gd_max_i / 2), text_col)

        btn_accept = QPushButton("Save", self)
        btn_accept.clicked.connect(self.btnSave)
        self.mainLayout.addWidget(btn_accept) # , (gd_max_i / 2), text_col)
        # self.config.updated.connect(self.on_config_update)

    def btnAccept(self):

        logger.debug("btnAccept")
        logger.debug(str(self.config))
        self.run()
        self._show_stats()

    def btnSave(self):
        filename = "file{:05d}.jpg"
        try:
            filename = QtGui.QFileDialog.getSaveFileName(
                self,
                "Save file",
                "file{:05d}.jpg",
                ""
            )
        except:
            pass
        filename = str(filename)
        self.gen.saveVolumeToFile(filename=filename)

    def on_config_update(self):
        pass

    def config_as_dict(self):
        dictionary = self.config.as_dict()
        for key, value in dictionary.iteritems():
            from PyQt4.QtCore import pyqtRemoveInputHook
            pyqtRemoveInputHook()
            # import ipdb; ipdb.set_trace() #  noqa BREAKPOINT
            if type(value) == PyQt4.QtCore.QString:
                value = str(value)
            dictionary[key] = value

        return dictionary

def main():
    logger = logging.getLogger()

    logger.setLevel(logging.DEBUG)
    ch = logging.StreamHandler()
    logger.addHandler(ch)

    # create file handler which logs even debug messages
    # fh = logging.FileHandler('log.txt')
    # fh.setLevel(logging.DEBUG)
    # formatter = logging.Formatter(
    #     '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    # fh.setFormatter(formatter)
    # logger.addHandler(fh)
    # logger.debug('start')

    # input parser
    parser = argparse.ArgumentParser(
        description=__doc__
    )
    parser.add_argument(
        '-i', '--inputfile',
        default=None,
        # required=True,
        help='input file'
    )
    parser.add_argument(
        '-d', '--debug', action='store_true',
        help='Debug mode')
    args = parser.parse_args()

    if args.debug:
        ch.setLevel(logging.DEBUG)

    app = QApplication(sys.argv)
    cfg = {"bool": True, "int":1}
    cw = CylindersWidget(cfg)
    cw.show()
    app.exec_()

if __name__ == "__main__":
    main()<commit_msg>fix widget name<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
# vim:fenc=utf-8
#
# Copyright © %YEAR%  <>
#
# Distributed under terms of the %LICENSE% license.

"""

"""

import logging

logger = logging.getLogger(__name__)
import argparse
# conda install -c conda-forge begins
# import begin

import PyQt4
from PyQt4.QtGui import QGridLayout, QLabel,\
    QPushButton, QLineEdit, QApplication, QWidget, QGridLayout, QSpinBox, QLineEdit, QCheckBox,\
        QComboBox, QTextEdit, QDialog, QMainWindow, QDoubleSpinBox

from PyQt4 import QtGui
import sys
import os.path
import copy

from .. import dictwidgetqt
import cylinders

from pyqtconfig import ConfigManager
import inspect
import collections

def get_default_args(obj):
    argspec = inspect.getargspec(obj.__init__)
    args = argspec.args[1:]
    defaults = argspec.defaults
    print "---- args"
    print args
    print defaults
    # import ipdb; ipdb.set_trace() #  noqa BREAKPOINT
    # dc = dict(zip(args, defaults))
    dc = collections.OrderedDict(zip(args, defaults))
    return dc

class CylindersWidget(QtGui.QWidget):
    def __init__(self, config_in, ncols=2):
        super(CylindersWidget, self).__init__()
        self.config_in = config_in
        self.ncols = ncols

        self.config = get_default_args(cylinders.CylinderGenerator)
        print "default args"
        print self.config
        self.init_ui()

    def run(self):
        print "generator args"
        new_cfg = self.configwg.config_as_dict()
        logger.debug(str(new_cfg))
        self.config = new_cfg
        print self.config
        self.gen = cylinders.CylinderGenerator(**self.config)
        self.gen.run()

    def _show_stats(self):
        df = self.gen.getStats()
        from .. import tablewidget

        dfmerne = df[["length", "volume", "surface"]].sum() / self.gen.area_volume
        print "merne"
        print dfmerne
        dfmernef = dfmerne.to_frame()
        dfmernef.insert(0, "", dfmernef.index)
        # import ipdb; ipdb.set_trace()
        tw = tablewidget.TableWidget(self, dataframe=dfmernef)
        self.mainLayout.addWidget(tw)

        from matplotlib.backends.backend_qt4agg import FigureCanvasQTAgg as FigureCanvas
        # from matplotlib.backends.backend_qt4agg import NavigationToolbar2QTAgg as NavigationToolbar
        import matplotlib.pyplot as plt

        self.figure = plt.figure()
        self.canvas = FigureCanvas(self.figure)
        # self.toolbar = NavigationToolbar(self.canvas, self)
        self.mainLayout.addWidget(self.canvas)
        plt.subplot(141)
        df[["length"]].boxplot()
        plt.subplot(142)
        df[['radius']].boxplot()

        plt.subplot(143)
        df[["surface"]].boxplot()

        plt.subplot(144)
        df[["volume"]].boxplot()

        # TODO take care about redrawing
        dfdescribe = df.describe()
        dfdescribe.insert(0, "", dfdescribe.index)
        tw = tablewidget.TableWidget(self, dataframe=dfdescribe)
        tw.show()
        tw.raise_()

        self.mainLayout.addWidget(tw)
        self.resize(600,700)

    def complicated_to_yaml(self, cfg):
        import yaml
        # convert values to json
        isconverted = {}
        for key, value in cfg.iteritems():
            if type(value) in (str, int, float, bool):

                isconverted[key] = False
                if type(value) is str:
                    pass

            else:
                isconverted[key] = True
                cfg[key] = yaml.dump(value, default_flow_style=True)
        return cfg

    def init_ui(self):
        self.mainLayout = QGridLayout(self)
        hide_keys = ["build", "gtree"]

        self.configwg = dictwidgetqt.DictWidget(self.config, hide_keys=hide_keys)
        self.mainLayout.addWidget(self.configwg)

        # self.mainLayout.setColumnMinimumWidth(text_col, 500)

        btn_accept = QPushButton("Run", self)
        btn_accept.clicked.connect(self.btnAccept)
        self.mainLayout.addWidget(btn_accept) # , (gd_max_i / 2), text_col)

        btn_save = QPushButton("Save", self)
        btn_save.clicked.connect(self.btnSave)
        self.mainLayout.addWidget(btn_save) # , (gd_max_i / 2), text_col)
        # self.config.updated.connect(self.on_config_update)

    def btnAccept(self):

        logger.debug("btnAccept")
        logger.debug(str(self.config))
        self.run()
        self._show_stats()

    def btnSave(self):
        filename = "file{:05d}.jpg"
        try:
            filename = QtGui.QFileDialog.getSaveFileName(
                self,
                "Save file",
                "file{:05d}.jpg",
                ""
            )
        except:
            pass
        filename = str(filename)
        self.gen.saveVolumeToFile(filename=filename)

    def on_config_update(self):
        pass

    def config_as_dict(self):
        dictionary = self.config.as_dict()
        for key, value in dictionary.iteritems():
            from PyQt4.QtCore import pyqtRemoveInputHook
            pyqtRemoveInputHook()
            # import ipdb; ipdb.set_trace() #  noqa BREAKPOINT
            if type(value) == PyQt4.QtCore.QString:
                value = str(value)
            dictionary[key] = value

        return dictionary

def main():
    logger = logging.getLogger()

    logger.setLevel(logging.DEBUG)
    ch = logging.StreamHandler()
    logger.addHandler(ch)

    # create file handler which logs even debug messages
    # fh = logging.FileHandler('log.txt')
    # fh.setLevel(logging.DEBUG)
    # formatter = logging.Formatter(
    #     '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    # fh.setFormatter(formatter)
    # logger.addHandler(fh)
    # logger.debug('start')

    # input parser
    parser = argparse.ArgumentParser(
        description=__doc__
    )
    parser.add_argument(
        '-i', '--inputfile',
        default=None,
        # required=True,
        help='input file'
    )
    parser.add_argument(
        '-d', '--debug', action='store_true',
        help='Debug mode')
    args = parser.parse_args()

    if args.debug:
        ch.setLevel(logging.DEBUG)

    app = QApplication(sys.argv)
    cfg = {"bool": True, "int":1}
    cw = CylindersWidget(cfg)
    cw.show()
    app.exec_()

if __name__ == "__main__":
    main()<|endoftext|>