<commit_before># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from st2tests.fixturesloader import FixturesLoader
from tests import FunctionalTest

FIXTURES_PACK = 'traces'

TEST_MODELS = {
    'traces': [
        'trace_empty.yaml',
        'trace_one_each.yaml',
        'trace_multiple_components.yaml'
    ]
}

class TestTraces(FunctionalTest):

    models = None
    trace1 = None
    trace2 = None
    trace3 = None

    @classmethod
    def setUpClass(cls):
        super(TestTraces, cls).setUpClass()
        cls.models = FixturesLoader().save_fixtures_to_db(fixtures_pack=FIXTURES_PACK,
                                                          fixtures_dict=TEST_MODELS)
        cls.trace1 = cls.models['traces']['trace_empty.yaml']
        cls.trace2 = cls.models['traces']['trace_one_each.yaml']
        cls.trace3 = cls.models['traces']['trace_multiple_components.yaml']

    def test_get_all(self):
        resp = self.app.get('/v1/traces')
        self.assertEqual(resp.status_int, 200)
        self.assertEqual(len(resp.json), 3, '/v1/traces did not return all traces.')

        retrieved_trace_tags = [trace['trace_tag'] for trace in resp.json]

        self.assertEqual(retrieved_trace_tags,
                         [self.trace1.trace_tag, self.trace2.trace_tag, self.trace3.trace_tag],
                         'Incorrect traces retrieved.')

    def test_get_by_id(self):
        resp = self.app.get('/v1/traces/%s' % self.trace1.id)
        self.assertEqual(resp.status_int, 200)
        self.assertEqual(resp.json['id'], str(self.trace1.id),
                         'Incorrect trace retrieved.')

    def test_query_by_trace_tag(self):
        resp = self.app.get('/v1/traces/?trace_tag=test-trace-1')
        self.assertEqual(resp.status_int, 200)
        self.assertEqual(len(resp.json), 1, '/v1/traces?trace_tag=x did not return correct trace.')

        self.assertEqual(resp.json[0]['trace_tag'], self.trace1['trace_tag'],
                         'Correct trace not returned.')

    def test_query_by_action_execution(self):
        execution_id = self.trace3['action_executions'][0].object_id
        resp = self.app.get('/v1/traces/?execution=%s' % execution_id)

        self.assertEqual(resp.status_int, 200)
        self.assertEqual(len(resp.json), 1,
                         '/v1/traces?execution=x did not return correct trace.')
        self.assertEqual(resp.json[0]['trace_tag'], self.trace3['trace_tag'],
                         'Correct trace not returned.')

    def test_query_by_rule(self):
        rule_id = self.trace3['rules'][0].object_id
        resp = self.app.get('/v1/traces/?rule=%s' % rule_id)

        self.assertEqual(resp.status_int, 200)
        self.assertEqual(len(resp.json), 1, '/v1/traces?rule=x did not return correct trace.')
        self.assertEqual(resp.json[0]['trace_tag'], self.trace3['trace_tag'],
                         'Correct trace not returned.')

    def test_query_by_trigger_instance(self):
        trigger_instance_id = self.trace3['trigger_instances'][0].object_id
        resp = self.app.get('/v1/traces/?trigger_instance=%s' % trigger_instance_id)

        self.assertEqual(resp.status_int, 200)
        self.assertEqual(len(resp.json), 1,
                         '/v1/traces?trigger_instance=x did not return correct trace.')
        self.assertEqual(resp.json[0]['trace_tag'], self.trace3['trace_tag'],
                         'Correct trace not returned.')
<commit_msg>Update affected tests.<commit_after># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from st2tests.fixturesloader import FixturesLoader
from tests import FunctionalTest

FIXTURES_PACK = 'traces'

TEST_MODELS = {
    'traces': [
        'trace_empty.yaml',
        'trace_one_each.yaml',
        'trace_multiple_components.yaml'
    ]
}

class TestTraces(FunctionalTest):

    models = None
    trace1 = None
    trace2 = None
    trace3 = None

    @classmethod
    def setUpClass(cls):
        super(TestTraces, cls).setUpClass()
        cls.models = FixturesLoader().save_fixtures_to_db(fixtures_pack=FIXTURES_PACK,
                                                          fixtures_dict=TEST_MODELS)
        cls.trace1 = cls.models['traces']['trace_empty.yaml']
        cls.trace2 = cls.models['traces']['trace_one_each.yaml']
        cls.trace3 = cls.models['traces']['trace_multiple_components.yaml']

    def test_get_all(self):
        resp = self.app.get('/v1/traces')
        self.assertEqual(resp.status_int, 200)
        self.assertEqual(len(resp.json), 3, '/v1/traces did not return all traces.')

        # Note: traces are returned sorted by start_timestamp in descending order by default
        retrieved_trace_tags = [trace['trace_tag'] for trace in resp.json]

        self.assertEqual(retrieved_trace_tags,
                         [self.trace3.trace_tag, self.trace2.trace_tag, self.trace1.trace_tag],
                         'Incorrect traces retrieved.')

    def test_get_by_id(self):
        resp = self.app.get('/v1/traces/%s' % self.trace1.id)
        self.assertEqual(resp.status_int, 200)
        self.assertEqual(resp.json['id'], str(self.trace1.id),
                         'Incorrect trace retrieved.')

    def test_query_by_trace_tag(self):
        resp = self.app.get('/v1/traces/?trace_tag=test-trace-1')
        self.assertEqual(resp.status_int, 200)
        self.assertEqual(len(resp.json), 1, '/v1/traces?trace_tag=x did not return correct trace.')

        self.assertEqual(resp.json[0]['trace_tag'], self.trace1['trace_tag'],
                         'Correct trace not returned.')

    def test_query_by_action_execution(self):
        execution_id = self.trace3['action_executions'][0].object_id
        resp = self.app.get('/v1/traces/?execution=%s' % execution_id)

        self.assertEqual(resp.status_int, 200)
        self.assertEqual(len(resp.json), 1,
                         '/v1/traces?execution=x did not return correct trace.')
        self.assertEqual(resp.json[0]['trace_tag'], self.trace3['trace_tag'],
                         'Correct trace not returned.')

    def test_query_by_rule(self):
        rule_id = self.trace3['rules'][0].object_id
        resp = self.app.get('/v1/traces/?rule=%s' % rule_id)

        self.assertEqual(resp.status_int, 200)
        self.assertEqual(len(resp.json), 1, '/v1/traces?rule=x did not return correct trace.')
        self.assertEqual(resp.json[0]['trace_tag'], self.trace3['trace_tag'],
                         'Correct trace not returned.')

    def test_query_by_trigger_instance(self):
        trigger_instance_id = self.trace3['trigger_instances'][0].object_id
        resp = self.app.get('/v1/traces/?trigger_instance=%s' % trigger_instance_id)

        self.assertEqual(resp.status_int, 200)
        self.assertEqual(len(resp.json), 1,
                         '/v1/traces?trigger_instance=x did not return correct trace.')
        self.assertEqual(resp.json[0]['trace_tag'], self.trace3['trace_tag'],
                         'Correct trace not returned.')
<|endoftext|>


<commit_before># Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ===================================================================
"""Optional helper for gradient handling."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import collections

from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import variable_scope
from tensorflow.python.ops import variables
from tensorflow.python.platform import tf_logging as logging
from tensorflow.python.tpu.ops import tpu_ops

def get_gradients_through_compute_gradients(optimizer, loss, activations):
  """Compute gradients to send to TPU embedding.

  Args:
    optimizer: a subclass of optimizer.Optimizer, usually CrossShardOptimizer.
      Used to call compute_gradients().
    loss: a Tensor to call optimizer.compute_gradients() on.
    activations: an OrderedDict mapping feature_name to Tensors of activations.

  Returns:
    An OrderedDict mapping from feature name Strings to Tensors of gradients of
      the loss wrt the activations of the features.
  """
  activation_list = activations.values()
  grads_and_vars = optimizer.compute_gradients(loss, activation_list)
  grads = [grad for grad, _ in grads_and_vars]
  feature_to_gradient_dict = collections.OrderedDict(
      zip(activations.keys(), grads))
  return feature_to_gradient_dict

def create_dummy_table_variables(tpu_embedding):
  """Create dummy embedding table variables.

  The sole purpose of these dummy variables are to trigger gradient
  calculation wrt them so that the gradients wrt activation can be captured
  and later sent to TPU embedding.

  Args:
    tpu_embedding: TPUEmbedding, dummy table variables will be created for use
      with tpu_embedding.

  Returns:
    A tuple of dummy variables and their initializer.

  Raises:
    RuntimeError: if collection to store gradients already exists and is not
    empty.
  """
  dummy_table_variables = collections.OrderedDict()
  for table_id, table in enumerate(tpu_embedding.table_to_features_dict):
    dummy_table_variables[table] = (
        # Explicitly specifying collections prevents this variable from
        # being added to the GLOBAL_VARIABLES collection, so that Saver()
        # ignores it.
        # But Tensorflow optimizer creates slot variable for these dummy
        # variable, e.g. tpu_embedding_dummy_table_variable_mlp_user/Adam{_1},
        # which will be in GLOBAL_VARIABLES collection,
        variable_scope.get_variable(
            'tpu_embedding_dummy_table_variable_{}'.format(table),
            dtype=dtypes.float32,
            shape=[1],
            use_resource=True,
            trainable=True,
            collections=['tpu_embedding_dummy_table_variables']))

    g = ops.get_default_graph()
    table_gradients = g.get_collection_ref(
        'tpu_embedding_gradients_table_{}'.format(table_id))
    if table_gradients:
      raise RuntimeError(
          'tpu_embedding_gradients_table_{} is not empty.'.format(table_id))
    num_features = len(tpu_embedding.table_to_features_dict[table])
    table_gradients.extend([None for _ in range(num_features)])

  return (dummy_table_variables,
          variables.variables_initializer(
              dummy_table_variables.values(),
              name='tpu_embedding_dummy_table_variables_init'))

def hook_dummy_table_variables_to_activations(tpu_embedding, activations,
                                              dummy_table_variables):
  """Have activations depend on dummy table variables for gradient intercept.

  Args:
    tpu_embedding: TPUEmbedding, activations and dummy_table_variables are from
      tpu_embedding.
    activations: An OrderedDict of feature name String to activation tensors.
    dummy_table_variables: An OrderedDict of table name String to dummy table
      variables.

  Returns:
    An OrderedDict of feature name String to activation tensors, which can be
      used just as the activations input.
  """
  new_activations = collections.OrderedDict()
  for feature in activations:
    table = tpu_embedding.feature_to_config_dict[feature].table_id
    new_activations[feature] = tpu_ops.tpu_embedding_activations(
        dummy_table_variables[table],
        activations[feature],
        table_id=list(tpu_embedding.table_to_config_dict).index(table),
        lookup_id=tpu_embedding.table_to_features_dict[table].index(feature))
  return new_activations

def get_gradients_through_dummy_table_variables(tpu_embedding):
  """Get gradients wrt the activations of each feature.

  Args:
    tpu_embedding: TPUEmbedding, create dummy table variable to be used with
      tpu_embedding.

  Returns:
    An OrderedDict mapping feature name to gradient.

  Raises:
    ValueError: if some gradients are not defined.
  """
  g = ops.get_default_graph()
  gradients_found = False
  for table_id, table in enumerate(tpu_embedding.table_to_config_dict):
    table_gradients = g.get_collection(
        'tpu_embedding_gradients_table_{}'.format(table_id))
    if any(gradient is None for gradient in table_gradients):
      # TODO(bfontain): create a white-list for optimizers which are compatible
      # with `tf.stop_gradient`.
      logging.warn(
          'Table {} with id {} has undefined gradients: this is probably '
          'because the model asked TPUEmbedding to compute activations that '
          'were not used, or tf.stop_gradient() is applied. Gradients of zeros '
          'are sent back to TPUEmbedding instead. Gradients of zeros and no '
          'gradients are equivalent for SGD, AdaGrad, FTRL, etc, but '
          'might differ for other optimizers due to implementation of TPU '
          'embedding optimizers.'.format(table, table_id))
    gradients_found = gradients_found or any(
        gradient is not None for gradient in table_gradients)

  if not gradients_found:
    raise ValueError(
        'All tables have undefined gradients: this is probably because the '
        'model asked TPUEmbedding to compute activations that were not used. '
        'If all TPUEmbedding features have stop_gradients, consider using the '
        'INFERENCE mode instead.'
    )

  feature_to_gradient_dict = collections.OrderedDict()
  for table_id, table in enumerate(tpu_embedding.table_to_config_dict):
    table_gradients = g.get_collection(
        'tpu_embedding_gradients_table_{}'.format(table_id))
    for feature, gradient in zip(tpu_embedding.table_to_features_dict[table],
                                 table_gradients):
      if gradient is not None:
        feature_to_gradient_dict[feature] = gradient
      else:
        dimension = tpu_embedding.table_to_config_dict[table].dimension
        batch_size = tpu_embedding.batch_size_per_core
        max_sequence_length = (
            tpu_embedding.feature_to_config_dict[feature].max_sequence_length)
        if max_sequence_length:
          feature_to_gradient_dict[feature] = array_ops.zeros(
              [batch_size, max_sequence_length, dimension])
        else:
          feature_to_gradient_dict[feature] = array_ops.zeros(
              [batch_size, dimension])

  return feature_to_gradient_dict
<commit_msg>Add warning instead of raising an error if no gradients are found for TPUEmbedding. There is a valid use-case where all embeddings are frozen after initial training for which this support is needed.<commit_after># Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ===================================================================
"""Optional helper for gradient handling."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import collections

from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import variable_scope
from tensorflow.python.ops import variables
from tensorflow.python.platform import tf_logging as logging
from tensorflow.python.tpu.ops import tpu_ops

def get_gradients_through_compute_gradients(optimizer, loss, activations):
  """Compute gradients to send to TPU embedding.

  Args:
    optimizer: a subclass of optimizer.Optimizer, usually CrossShardOptimizer.
      Used to call compute_gradients().
    loss: a Tensor to call optimizer.compute_gradients() on.
    activations: an OrderedDict mapping feature_name to Tensors of activations.

  Returns:
    An OrderedDict mapping from feature name Strings to Tensors of gradients of
      the loss wrt the activations of the features.
  """
  activation_list = activations.values()
  grads_and_vars = optimizer.compute_gradients(loss, activation_list)
  grads = [grad for grad, _ in grads_and_vars]
  feature_to_gradient_dict = collections.OrderedDict(
      zip(activations.keys(), grads))
  return feature_to_gradient_dict

def create_dummy_table_variables(tpu_embedding):
  """Create dummy embedding table variables.

  The sole purpose of these dummy variables are to trigger gradient
  calculation wrt them so that the gradients wrt activation can be captured
  and later sent to TPU embedding.

  Args:
    tpu_embedding: TPUEmbedding, dummy table variables will be created for use
      with tpu_embedding.

  Returns:
    A tuple of dummy variables and their initializer.

  Raises:
    RuntimeError: if collection to store gradients already exists and is not
    empty.
  """
  dummy_table_variables = collections.OrderedDict()
  for table_id, table in enumerate(tpu_embedding.table_to_features_dict):
    dummy_table_variables[table] = (
        # Explicitly specifying collections prevents this variable from
        # being added to the GLOBAL_VARIABLES collection, so that Saver()
        # ignores it.
        # But Tensorflow optimizer creates slot variable for these dummy
        # variable, e.g. tpu_embedding_dummy_table_variable_mlp_user/Adam{_1},
        # which will be in GLOBAL_VARIABLES collection,
        variable_scope.get_variable(
            'tpu_embedding_dummy_table_variable_{}'.format(table),
            dtype=dtypes.float32,
            shape=[1],
            use_resource=True,
            trainable=True,
            collections=['tpu_embedding_dummy_table_variables']))

    g = ops.get_default_graph()
    table_gradients = g.get_collection_ref(
        'tpu_embedding_gradients_table_{}'.format(table_id))
    if table_gradients:
      raise RuntimeError(
          'tpu_embedding_gradients_table_{} is not empty.'.format(table_id))
    num_features = len(tpu_embedding.table_to_features_dict[table])
    table_gradients.extend([None for _ in range(num_features)])

  return (dummy_table_variables,
          variables.variables_initializer(
              dummy_table_variables.values(),
              name='tpu_embedding_dummy_table_variables_init'))

def hook_dummy_table_variables_to_activations(tpu_embedding, activations,
                                              dummy_table_variables):
  """Have activations depend on dummy table variables for gradient intercept.

  Args:
    tpu_embedding: TPUEmbedding, activations and dummy_table_variables are from
      tpu_embedding.
    activations: An OrderedDict of feature name String to activation tensors.
    dummy_table_variables: An OrderedDict of table name String to dummy table
      variables.

  Returns:
    An OrderedDict of feature name String to activation tensors, which can be
      used just as the activations input.
  """
  new_activations = collections.OrderedDict()
  for feature in activations:
    table = tpu_embedding.feature_to_config_dict[feature].table_id
    new_activations[feature] = tpu_ops.tpu_embedding_activations(
        dummy_table_variables[table],
        activations[feature],
        table_id=list(tpu_embedding.table_to_config_dict).index(table),
        lookup_id=tpu_embedding.table_to_features_dict[table].index(feature))
  return new_activations

def get_gradients_through_dummy_table_variables(tpu_embedding):
  """Get gradients wrt the activations of each feature.

  Args:
    tpu_embedding: TPUEmbedding, create dummy table variable to be used with
      tpu_embedding.

  Returns:
    An OrderedDict mapping feature name to gradient.

  Raises:
    ValueError: if some gradients are not defined.
  """
  g = ops.get_default_graph()
  gradients_found = False
  for table_id, table in enumerate(tpu_embedding.table_to_config_dict):
    table_gradients = g.get_collection(
        'tpu_embedding_gradients_table_{}'.format(table_id))
    if any(gradient is None for gradient in table_gradients):
      # TODO(bfontain): create a white-list for optimizers which are compatible
      # with `tf.stop_gradient`.
      logging.warn(
          'Table {} with id {} has undefined gradients: this is probably '
          'because the model asked TPUEmbedding to compute activations that '
          'were not used, or tf.stop_gradient() is applied. Gradients of zeros '
          'are sent back to TPUEmbedding instead. Gradients of zeros and no '
          'gradients are equivalent for SGD, AdaGrad, FTRL, etc, but '
          'might differ for other optimizers due to implementation of TPU '
          'embedding optimizers.'.format(table, table_id))
    gradients_found = gradients_found or any(
        gradient is not None for gradient in table_gradients)

  if not gradients_found:
    logging.warn(
        'All tables have undefined gradients: this is probably because the '
        'model asked TPUEmbedding to compute activations that were not used. '
        'If all TPUEmbedding features have stop_gradients, consider using the '
        'INFERENCE mode instead.')

  feature_to_gradient_dict = collections.OrderedDict()
  for table_id, table in enumerate(tpu_embedding.table_to_config_dict):
    table_gradients = g.get_collection(
        'tpu_embedding_gradients_table_{}'.format(table_id))
    for feature, gradient in zip(tpu_embedding.table_to_features_dict[table],
                                 table_gradients):
      if gradient is not None:
        feature_to_gradient_dict[feature] = gradient
      else:
        dimension = tpu_embedding.table_to_config_dict[table].dimension
        batch_size = tpu_embedding.batch_size_per_core
        max_sequence_length = (
            tpu_embedding.feature_to_config_dict[feature].max_sequence_length)
        if max_sequence_length:
          feature_to_gradient_dict[feature] = array_ops.zeros(
              [batch_size, max_sequence_length, dimension])
        else:
          feature_to_gradient_dict[feature] = array_ops.zeros(
              [batch_size, dimension])

  return feature_to_gradient_dict
<|endoftext|>


<commit_before>"""Dependency tracking for trackable objects."""
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import functools
import weakref

from tensorflow.python.eager import def_function
from tensorflow.python.eager import function as defun
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.training.tracking import base
from tensorflow.python.training.tracking import data_structures
from tensorflow.python.util import tf_contextlib

# global _RESOURCE_TRACKER_STACK
_RESOURCE_TRACKER_STACK = []

class NotTrackable(object):
  """Marks instances of child classes as unsaveable using an object-based API.

  Useful for marking objects which would otherwise look trackable because
  of inheritance (e.g. through `Layer`) as not trackable. Inheriting from
  `NotTrackable` does not prevent an object from being assigned to any
  attributes, but will throw an error on save/restore.
  """
  pass

class AutoTrackable(base.Trackable):
  """Manages dependencies on other objects.

  `Trackable` objects may have dependencies: other `Trackable` objects
  which should be saved if the object declaring the dependency is saved. A
  correctly saveable program has a dependency graph such that if changing a
  global variable affects an object (e.g. changes the behavior of any of its
  methods) then there is a chain of dependencies from the influenced object to
  the variable.

  Dependency edges have names, and are created implicitly when a
  `Trackable` object is assigned to an attribute of another
  `Trackable` object. For example:

  ```
  obj = Trackable()
  obj.v = ResourceVariable(0.)
  ```

  The `Trackable` object `obj` now has a dependency named "v" on a
  variable.

  `Trackable` objects may specify `Tensor`s to be saved and restored
  directly (e.g. a `Variable` indicating how to save itself) rather than through
  dependencies on other objects. See
  `Trackable._gather_saveables_for_checkpoint` for details.
  """

  def __setattr__(self, name, value):
    """Support self.foo = trackable syntax."""
    if getattr(self, "_self_setattr_tracking", True):
      value = data_structures.sticky_attribute_assignment(
          trackable=self, value=value, name=name)
    super(AutoTrackable, self).__setattr__(name, value)

  def __delattr__(self, name):
    self._maybe_initialize_trackable()
    delete_tracking(self, name)
    super(AutoTrackable, self).__delattr__(name)

  def _no_dependency(self, value):
    """Override to allow TrackableBase to disable dependency tracking."""
    return data_structures.NoDependency(value)

  def _list_functions_for_serialization(self, unused_serialization_cache):
    """Return a dict of `Function`s of a trackable."""
    functions = {}
    for attribute_name in dir(self):
      try:
        attribute_value = getattr(self, attribute_name, None)
      except Exception:  # pylint: disable=broad-except
        # We really don't want to throw an exception just because some object's
        # attribute accessor is broken.
        attribute_value = None
      if isinstance(attribute_value, (def_function.Function,
                                      defun.ConcreteFunction)):
        functions[attribute_name] = attribute_value
    return functions

def delete_tracking(obj, name):
  """Removes the tracking of name from object."""
  # pylint: disable=protected-access
  if name in obj._unconditional_dependency_names:
    del obj._unconditional_dependency_names[name]
    for index, (dep_name, _) in enumerate(
        obj._unconditional_checkpoint_dependencies):
      if dep_name == name:
        del obj._unconditional_checkpoint_dependencies[index]
        break
  # pylint: enable=protected-access

class ResourceTracker(object):
  """An object that tracks a list of resources."""

  def __init__(self):
    self._resources = []

  @property
  def resources(self):
    return self._resources

  def add_resource(self, resource):
    self._resources.append(resource)

@tf_contextlib.contextmanager
def resource_tracker_scope(resource_tracker):
  """A context to manage resource trackers.

  Use this in order to collect up all resources created within a block of code.
  Example usage:

  ```python
  resource_tracker = ResourceTracker()
  with resource_tracker_scope(resource_tracker):
    resource = TrackableResource()

  assert resource_tracker.resources == [resource]

  Args:
    resource_tracker: The passed in ResourceTracker object

  Yields:
    A scope in which the resource_tracker is active.
  """
  global _RESOURCE_TRACKER_STACK
  old = list(_RESOURCE_TRACKER_STACK)
  _RESOURCE_TRACKER_STACK.append(resource_tracker)
  try:
    yield
  finally:
    _RESOURCE_TRACKER_STACK = old

class CapturableResource(base.Trackable):
  """Holds a Tensor which a tf.function can capture.

  `CapturableResource`s are discovered by traversing the graph of object
  attributes, e.g. during `tf.saved_model.save`. They are excluded from the
  scope-based tracking of `TrackableResource`; generally things that require
  initialization should inherit from `TrackableResource` instead of
  `CapturableResource` directly.
  """

  def __init__(self, device=""):
    """Initialize the `CapturableResource`.

    Args:
      device: A string indicating a required placement for this resource,
        e.g. "CPU" if this resource must be created on a CPU device. A blank
        device allows the user to place resource creation, so generally this
        should be blank unless the resource only makes sense on one device.
    """
    self._resource_handle = None
    self._resource_device = device

  def _create_resource(self):
    """A function that creates a resource handle."""
    raise NotImplementedError("TrackableResource._create_resource not "
                              "implemented.")

  def _initialize(self):
    """A function that initializes the resource. Optional."""
    pass

  @property
  def resource_handle(self):
    """Returns the resource handle associated with this Resource."""
    if self._resource_handle is None:
      with ops.device(self._resource_device):
        self._resource_handle = self._create_resource()
    return self._resource_handle

  def _list_functions_for_serialization(self, unused_functions):
    @def_function.function(input_signature=[], autograph=False)
    def _creator():
      resource = self._create_resource()
      return resource

    @def_function.function(input_signature=[], autograph=False)
    def _initializer():
      self._initialize()
      return 1  # Dummy return

    return {
        "_create_resource": _creator,
        "_initialize": _initializer,
    }

class TrackableResource(CapturableResource):
  """Adds scope tracking to CapturableResource."""

  def __init__(self, device=""):
    """Initialize the `TrackableResource`.

    Args:
      device: A string indicating a required placement for this resource,
        e.g. "CPU" if this resource must be created on a CPU device. A blank
        device allows the user to place resource creation, so generally this
        should be blank unless the resource only makes sense on one device.
    """
    global _RESOURCE_TRACKER_STACK
    for resource_tracker in _RESOURCE_TRACKER_STACK:
      resource_tracker.add_resource(self)
    super(TrackableResource, self).__init__(device=device)

class TrackableAsset(base.Trackable):
  """Base class for asset files which need to be tracked."""

  def __init__(self, path):
    """Record the full path to the asset."""
    # The init_scope prevents functions from capturing `path` in an
    # initialization graph, since it is transient and should not end up in a
    # serialized function body.
    with ops.init_scope(), ops.device("CPU"):
      self._path = ops.internal_convert_to_tensor(path, dtype=dtypes.string,
                                                  name="asset_path")

  @property
  def asset_path(self):
    """Fetch the current asset path."""
    return self._path

def cached_per_instance(f):
  """Lightweight decorator for caching lazily constructed properties.

  When to use:
  This decorator provides simple caching with minimal overhead. It is designed
  for properties which are expensive to compute and static over the life of a
  class instance, and provides no mechanism for cache invalidation. Thus it is
  best suited for lazily exposing derived properties of other static data.

  For classes with custom getattr / setattr behavior (such as trackable
  objects), storing cache results as object attributes is not performant.
  Instead, a specialized cache can significantly reduce property lookup
  overhead. (While still allowing the decorated property to be lazily computed.)
  Consider the following class:

  ```
  class MyClass(object):
    def __setattr__(self, key, value):
      # Some expensive class specific code
      # ...
      # ...

      super(MyClass, self).__setattr__(key, value)

    @property
    def thing(self):
      # `thing` is expensive to compute (and may not even be requested), so we
      # want to lazily compute it and then cache it.
      output = getattr(self, '_thing', None)
      if output is None:
        self._thing = output = compute_thing(self)
      return output
  ```

  It's also worth noting that ANY overriding of __setattr__, even something as
  simple as:
  ```
    def __setattr__(self, key, value):
      super(MyClass, self).__setattr__(key, value)
  ```

  Slows down attribute assignment by nearly 10x.

  By contrast, replacing the definition of `thing` with the following sidesteps
  the expensive __setattr__ altogether:

  '''
  @property
  @tracking.cached_per_instance
  def thing(self):
    # `thing` is expensive to compute (and may not even be requested), so we
    # want to lazily compute it and then cache it.
    return compute_thing(self)
  '''

  Performance:
  The overhead for this decorator is ~0.4 us / call. A much lower overhead
  implementation (~0.085 us / call) can be achieved by using a custom dict type:

  ```
  def dict_based_cache(f):
    class Cache(dict):
      __slots__ = ()
      def __missing__(self, key):
        self[key] = output = f(key)
        return output

    return property(Cache().__getitem__)
  ```

  However, that implementation holds class instances as keys, and as a result
  blocks garbage collection. (And modifying it to use weakref's as keys raises
  the lookup overhead to ~0.4 us) As a result, the WeakKeyDictionary
  implementation below turns out to be more prudent.

  Args:
    f: The function to cache.

  Returns:
    f decorated with simple caching behavior.
  """

  cache = weakref.WeakKeyDictionary()

  @functools.wraps(f)
  def wrapped(item):
    output = cache.get(item)
    if output is None:
      cache[item] = output = f(item)
    return output
  return wrapped

ops.register_tensor_conversion_function(
    TrackableAsset,
    lambda asset, **kw: ops.internal_convert_to_tensor(asset.asset_path, **kw))
<commit_msg>Fast path `self.x = self.x` (or equivalent).<commit_after>"""Dependency tracking for trackable objects."""
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import functools
import weakref

from tensorflow.python.eager import def_function
from tensorflow.python.eager import function as defun
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.training.tracking import base
from tensorflow.python.training.tracking import data_structures
from tensorflow.python.util import tf_contextlib

# global _RESOURCE_TRACKER_STACK
_RESOURCE_TRACKER_STACK = []

class NotTrackable(object):
  """Marks instances of child classes as unsaveable using an object-based API.

  Useful for marking objects which would otherwise look trackable because
  of inheritance (e.g. through `Layer`) as not trackable. Inheriting from
  `NotTrackable` does not prevent an object from being assigned to any
  attributes, but will throw an error on save/restore.
  """
  pass

class AutoTrackable(base.Trackable):
  """Manages dependencies on other objects.

  `Trackable` objects may have dependencies: other `Trackable` objects
  which should be saved if the object declaring the dependency is saved. A
  correctly saveable program has a dependency graph such that if changing a
  global variable affects an object (e.g. changes the behavior of any of its
  methods) then there is a chain of dependencies from the influenced object to
  the variable.

  Dependency edges have names, and are created implicitly when a
  `Trackable` object is assigned to an attribute of another
  `Trackable` object. For example:

  ```
  obj = Trackable()
  obj.v = ResourceVariable(0.)
  ```

  The `Trackable` object `obj` now has a dependency named "v" on a
  variable.

  `Trackable` objects may specify `Tensor`s to be saved and restored
  directly (e.g. a `Variable` indicating how to save itself) rather than through
  dependencies on other objects. See
  `Trackable._gather_saveables_for_checkpoint` for details.
  """

  def __setattr__(self, name, value):
    """Support self.foo = trackable syntax."""
    try:
      if getattr(self, name) is value:
        # Short circuit for `self.$x = self.$x`.
        return
    except AttributeError:
      pass

    if getattr(self, "_self_setattr_tracking", True):
      value = data_structures.sticky_attribute_assignment(
          trackable=self, value=value, name=name)
    super(AutoTrackable, self).__setattr__(name, value)

  def __delattr__(self, name):
    self._maybe_initialize_trackable()
    delete_tracking(self, name)
    super(AutoTrackable, self).__delattr__(name)

  def _no_dependency(self, value):
    """Override to allow TrackableBase to disable dependency tracking."""
    return data_structures.NoDependency(value)

  def _list_functions_for_serialization(self, unused_serialization_cache):
    """Return a dict of `Function`s of a trackable."""
    functions = {}
    for attribute_name in dir(self):
      try:
        attribute_value = getattr(self, attribute_name, None)
      except Exception:  # pylint: disable=broad-except
        # We really don't want to throw an exception just because some object's
        # attribute accessor is broken.
        attribute_value = None
      if isinstance(attribute_value, (def_function.Function,
                                      defun.ConcreteFunction)):
        functions[attribute_name] = attribute_value
    return functions

def delete_tracking(obj, name):
  """Removes the tracking of name from object."""
  # pylint: disable=protected-access
  if name in obj._unconditional_dependency_names:
    del obj._unconditional_dependency_names[name]
    for index, (dep_name, _) in enumerate(
        obj._unconditional_checkpoint_dependencies):
      if dep_name == name:
        del obj._unconditional_checkpoint_dependencies[index]
        break
  # pylint: enable=protected-access

class ResourceTracker(object):
  """An object that tracks a list of resources."""

  def __init__(self):
    self._resources = []

  @property
  def resources(self):
    return self._resources

  def add_resource(self, resource):
    self._resources.append(resource)

@tf_contextlib.contextmanager
def resource_tracker_scope(resource_tracker):
  """A context to manage resource trackers.

  Use this in order to collect up all resources created within a block of code.
  Example usage:

  ```python
  resource_tracker = ResourceTracker()
  with resource_tracker_scope(resource_tracker):
    resource = TrackableResource()

  assert resource_tracker.resources == [resource]

  Args:
    resource_tracker: The passed in ResourceTracker object

  Yields:
    A scope in which the resource_tracker is active.
  """
  global _RESOURCE_TRACKER_STACK
  old = list(_RESOURCE_TRACKER_STACK)
  _RESOURCE_TRACKER_STACK.append(resource_tracker)
  try:
    yield
  finally:
    _RESOURCE_TRACKER_STACK = old

class CapturableResource(base.Trackable):
  """Holds a Tensor which a tf.function can capture.

  `CapturableResource`s are discovered by traversing the graph of object
  attributes, e.g. during `tf.saved_model.save`. They are excluded from the
  scope-based tracking of `TrackableResource`; generally things that require
  initialization should inherit from `TrackableResource` instead of
  `CapturableResource` directly.
  """

  def __init__(self, device=""):
    """Initialize the `CapturableResource`.

    Args:
      device: A string indicating a required placement for this resource,
        e.g. "CPU" if this resource must be created on a CPU device. A blank
        device allows the user to place resource creation, so generally this
        should be blank unless the resource only makes sense on one device.
    """
    self._resource_handle = None
    self._resource_device = device

  def _create_resource(self):
    """A function that creates a resource handle."""
    raise NotImplementedError("TrackableResource._create_resource not "
                              "implemented.")

  def _initialize(self):
    """A function that initializes the resource. Optional."""
    pass

  @property
  def resource_handle(self):
    """Returns the resource handle associated with this Resource."""
    if self._resource_handle is None:
      with ops.device(self._resource_device):
        self._resource_handle = self._create_resource()
    return self._resource_handle

  def _list_functions_for_serialization(self, unused_functions):
    @def_function.function(input_signature=[], autograph=False)
    def _creator():
      resource = self._create_resource()
      return resource

    @def_function.function(input_signature=[], autograph=False)
    def _initializer():
      self._initialize()
      return 1  # Dummy return

    return {
        "_create_resource": _creator,
        "_initialize": _initializer,
    }

class TrackableResource(CapturableResource):
  """Adds scope tracking to CapturableResource."""

  def __init__(self, device=""):
    """Initialize the `TrackableResource`.

    Args:
      device: A string indicating a required placement for this resource,
        e.g. "CPU" if this resource must be created on a CPU device. A blank
        device allows the user to place resource creation, so generally this
        should be blank unless the resource only makes sense on one device.
    """
    global _RESOURCE_TRACKER_STACK
    for resource_tracker in _RESOURCE_TRACKER_STACK:
      resource_tracker.add_resource(self)
    super(TrackableResource, self).__init__(device=device)

class TrackableAsset(base.Trackable):
  """Base class for asset files which need to be tracked."""

  def __init__(self, path):
    """Record the full path to the asset."""
    # The init_scope prevents functions from capturing `path` in an
    # initialization graph, since it is transient and should not end up in a
    # serialized function body.
    with ops.init_scope(), ops.device("CPU"):
      self._path = ops.internal_convert_to_tensor(path, dtype=dtypes.string,
                                                  name="asset_path")

  @property
  def asset_path(self):
    """Fetch the current asset path."""
    return self._path

def cached_per_instance(f):
  """Lightweight decorator for caching lazily constructed properties.

  When to use:
  This decorator provides simple caching with minimal overhead. It is designed
  for properties which are expensive to compute and static over the life of a
  class instance, and provides no mechanism for cache invalidation. Thus it is
  best suited for lazily exposing derived properties of other static data.

  For classes with custom getattr / setattr behavior (such as trackable
  objects), storing cache results as object attributes is not performant.
  Instead, a specialized cache can significantly reduce property lookup
  overhead. (While still allowing the decorated property to be lazily computed.)
  Consider the following class:

  ```
  class MyClass(object):
    def __setattr__(self, key, value):
      # Some expensive class specific code
      # ...
      # ...

      super(MyClass, self).__setattr__(key, value)

    @property
    def thing(self):
      # `thing` is expensive to compute (and may not even be requested), so we
      # want to lazily compute it and then cache it.
      output = getattr(self, '_thing', None)
      if output is None:
        self._thing = output = compute_thing(self)
      return output
  ```

  It's also worth noting that ANY overriding of __setattr__, even something as
  simple as:
  ```
    def __setattr__(self, key, value):
      super(MyClass, self).__setattr__(key, value)
  ```

  Slows down attribute assignment by nearly 10x.

  By contrast, replacing the definition of `thing` with the following sidesteps
  the expensive __setattr__ altogether:

  '''
  @property
  @tracking.cached_per_instance
  def thing(self):
    # `thing` is expensive to compute (and may not even be requested), so we
    # want to lazily compute it and then cache it.
    return compute_thing(self)
  '''

  Performance:
  The overhead for this decorator is ~0.4 us / call. A much lower overhead
  implementation (~0.085 us / call) can be achieved by using a custom dict type:

  ```
  def dict_based_cache(f):
    class Cache(dict):
      __slots__ = ()
      def __missing__(self, key):
        self[key] = output = f(key)
        return output

    return property(Cache().__getitem__)
  ```

  However, that implementation holds class instances as keys, and as a result
  blocks garbage collection. (And modifying it to use weakref's as keys raises
  the lookup overhead to ~0.4 us) As a result, the WeakKeyDictionary
  implementation below turns out to be more prudent.

  Args:
    f: The function to cache.

  Returns:
    f decorated with simple caching behavior.
  """

  cache = weakref.WeakKeyDictionary()

  @functools.wraps(f)
  def wrapped(item):
    output = cache.get(item)
    if output is None:
      cache[item] = output = f(item)
    return output
  return wrapped

ops.register_tensor_conversion_function(
    TrackableAsset,
    lambda asset, **kw: ops.internal_convert_to_tensor(asset.asset_path, **kw))
<|endoftext|>


<commit_before>'''
Script to visualize the "Noise vs. Exposure" data from DevWare

According to section 2.2.4.7 of the DevWareX help (http://is.gd/mt7FyF) we get
signal levels and different kinds of noise measurements by using the "Sensor
Control" > "Diagnostics" > "Noise vs. Exposure" tool.

The analyis report gives
- Signal
- RMS Dyn (temporal noise), Avg Dyn (temporal noise)
- FPN (fixed pattern noise), Col FPN (columnwise FPN), Row FPN (rowwise FPN)
- Col Dyn (columnwise temporal noise) and Row Dyn (rowwise temporal noise).

See the wiki page linkes above to see how the values are calulated.
'''

import os
import numpy
import matplotlib.pyplot as plt

Root = '/afs/psi.ch/project/EssentialMed/Images/NoiseVsExposure'
DataFiles = [
    'AR0130_Navitar_10_100_3_16.txt',
    'AR0130_Navitar_10_500_3_16.txt',
    'AR0130_Navitar_20_200_3_12.txt',
    'AR0130_Navitar_20_200_4_16.txt',
    'AR0130_Navitar_20_200_6_16.txt',
    'AR0130_Navitar_20_500_5_16.txt',
    'AR0130_Navitar_50_100_1_10.txt'
    ]

Labels = ['Exposure time [ms]',
    'Signal',
    'RMS Dyn (temporal noise)',
    'Avg Dyn (temporal noise)',
    'FPN (fixed pattern noise)',
    'columnwise FPN',
    'rowwise FPN',
    'columnwise temporal noise',
    'rowwise temporal noise']

# Which plot do we show?
whichone = 6

# Tell what we do
print 'Reading data file', DataFiles[whichone]
Sensor = DataFiles[whichone][:-4].split('_')[0]
Lens = DataFiles[whichone][:-4].split('_')[1]
FramesPerSample = DataFiles[whichone][:-4].split('_')[2]
MaximumExposure = DataFiles[whichone][:-4].split('_')[3]
Decades = DataFiles[whichone][:-4].split('_')[4]
SamplesPerDecade = DataFiles[whichone][:-4].split('_')[5]

print 'We are showing the data from the', Sensor, 'CMOS with the',  Lens, \
    'lens. The analysis was done with', FramesPerSample, \
    'frames per sample,', MaximumExposure, 'ms maximum exposure over', \
    Decades, 'decades with', SamplesPerDecade, 'samples per decade.'

print 'If the exposure has not been recorded in "log scale" (you will see',\
    'it in the plots), the "Decades" correspond to the "minimal exposure"',\
    'and the "samples per decade" correspond to the "numbers of samples".'

# Generate figure title, so we can distinguish the output
Title = Sensor, Lens, FramesPerSample, 'Frames per Sample', MaximumExposure, \
    'ms Maximum Exposure', Decades, 'Decades', SamplesPerDecade, \
    'Samples/Decade'

# Load the data from the file
File = os.path.join(Root, DataFiles[whichone])
Data = numpy.loadtxt(File, skiprows=3)
## First line gives the full range. Read it with the snippet based on
## http://stackoverflow.com/a/1904455
with open(File, 'r') as f:
    FullRange = int(f.readline().split('=')[1])

# Plot the data
## The title of the plot is split over all the suplots, otherwise it destroys
## the layout due to its long length
plt.figure(' '.join(Title), figsize=(16, 9))
## Signal
plt.subplot(131)
plt.plot(Data[:, 0], Data[:, 1], 'o-', label='1 ' + Labels[1])
plt.axhline(FullRange, linestyle='--', label='Full range')
plt.xlabel(Labels[0])
plt.ylabel(Labels[1])
plt.legend(loc='best')
plt.title(' '.join(Title[:2]))

## Fixed pattern noise
plt.subplot(132)
for i in range(4, 7):
    plt.plot(Data[:, 0], Data[:, i], 'o-', label=str(i) + ' ' + Labels[i])
plt.xlabel(Labels[0])
plt.ylabel('FPN')
plt.legend(loc='best')
plt.title(' '.join(Title[2:6]))

# Temporal noise
plt.subplot(133)
for i in [2, 3, 7, 8]:
    plt.plot(Data[:, 0], Data[:, i], 'o-', label=str(i) + ' ' + Labels[i])
plt.legend(loc='best')
plt.xlabel(Labels[0])
plt.ylabel('Dyn')
plt.title(' '.join(Title[6:]))
plt.tight_layout()
plt.savefig(os.path.join(Root, DataFiles[whichone][:-4] + '.png'),
            Transparent=True, bbox_inches='tight')
plt.draw()
plt.show()
<commit_msg>Ask user which file to plot at runtime.<commit_after>'''
Script to visualize the "Noise vs. Exposure" data from DevWare

According to section 2.2.4.7 of the DevWareX help (http://is.gd/mt7FyF) we get
signal levels and different kinds of noise measurements by using the "Sensor
Control" > "Diagnostics" > "Noise vs. Exposure" tool.

The analyis report gives
- Signal
- RMS Dyn (temporal noise), Avg Dyn (temporal noise)
- FPN (fixed pattern noise), Col FPN (columnwise FPN), Row FPN (rowwise FPN)
- Col Dyn (columnwise temporal noise) and Row Dyn (rowwise temporal noise).

See the wiki page linkes above to see how the values are calulated.
'''

import glob
import os
import numpy
import matplotlib.pyplot as plt

def AskUser(Blurb, Choices):
    """ Ask for input. Based on function in MasterThesisIvan.ini """
    print(Blurb)
    for Counter, Item in enumerate(sorted(Choices)):
        print '    * [' + str(Counter) + ']:', Item
    Selection = []
    while Selection not in range(len(Choices)):
        try:
            Selection = int(input(' '.join(['Please enter the choice you',
                                            'want [0-' +
                                            str(len(Choices) - 1) +
                                            ']:'])))
        except:
            print 'You actually have to select *something*'
        if Selection not in range(len(Choices)):
            print 'Try again with a valid choice'
    print 'You selected', sorted(Choices)[Selection]
    return sorted(Choices)[Selection]

Root = '/afs/psi.ch/project/EssentialMed/Images/NoiseVsExposure'
DataFiles = [os.path.basename(i) for
    i in glob.glob(os.path.join(Root, '*.txt'))]

# Which plot do we show?
whichone = DataFiles.index(AskUser('Which file should I show you?', DataFiles))

# Tell what we do
Sensor = DataFiles[whichone][:-4].split('_')[0]
Lens = DataFiles[whichone][:-4].split('_')[1]
FramesPerSample = DataFiles[whichone][:-4].split('_')[2]
MaximumExposure = DataFiles[whichone][:-4].split('_')[3]
Decades = DataFiles[whichone][:-4].split('_')[4]
SamplesPerDecade = DataFiles[whichone][:-4].split('_')[5]

print 'We are showing the data from the', Sensor, 'CMOS with the',  Lens, \
    'lens. The analysis was done with', FramesPerSample, \
    'frames per sample,', MaximumExposure, 'ms maximum exposure over', \
    Decades, 'decades with', SamplesPerDecade, 'samples per decade.'

print 'If the exposure has not been recorded in "log scale" (you will see',\
    'it in the plots), the "Decades" correspond to the "minimal exposure"',\
    'and the "samples per decade" correspond to the "numbers of samples".'

# Generate figure title, so we can distinguish the output
Title = Sensor, Lens, FramesPerSample, 'Frames per Sample', MaximumExposure, \
    'ms Maximum Exposure', Decades, 'Decades', SamplesPerDecade, \
    'Samples/Decade'

# Load the data from the file
File = os.path.join(Root, DataFiles[whichone])
Data = numpy.loadtxt(File, skiprows=3)
## First line gives the full range. Read it with the snippet based on
## http://stackoverflow.com/a/1904455
with open(File, 'r') as f:
    FullRange = int(f.readline().split('=')[1])

# Plot the data
Labels = ['Exposure time [ms]',
    'Signal',
    'RMS Dyn (temporal noise)',
    'Avg Dyn (temporal noise)',
    'FPN (fixed pattern noise)',
    'columnwise FPN',
    'rowwise FPN',
    'columnwise temporal noise',
    'rowwise temporal noise']
## The title of the plot is split over all the suplots, otherwise it destroys
## the layout due to its long length
plt.figure(' '.join(Title), figsize=(16, 9))
## Signal
plt.subplot(131)
plt.plot(Data[:, 0], Data[:, 1], 'o-', label='1 ' + Labels[1])
plt.axhline(FullRange, linestyle='--', label='Full range')
plt.xlabel(Labels[0])
plt.ylabel(Labels[1])
plt.legend(loc='best')
plt.title(' '.join(Title[:2]))

## Fixed pattern noise
plt.subplot(132)
for i in range(4, 7):
    plt.plot(Data[:, 0], Data[:, i], 'o-', label=str(i) + ' ' + Labels[i])
plt.xlabel(Labels[0])
plt.ylabel('FPN')
plt.legend(loc='best')
plt.title(' '.join(Title[2:6]))

# Temporal noise
plt.subplot(133)
for i in [2, 3, 7, 8]:
    plt.plot(Data[:, 0], Data[:, i], 'o-', label=str(i) + ' ' + Labels[i])
plt.legend(loc='best')
plt.xlabel(Labels[0])
plt.ylabel('Dyn')
plt.title(' '.join(Title[6:]))
plt.tight_layout()
plt.savefig(os.path.join(Root, DataFiles[whichone][:-4] + '.png'),
            Transparent=True, bbox_inches='tight')
plt.draw()
plt.show()
<|endoftext|>


<commit_before>from bsdploy import bsdploy_path
import os
import pytest

@pytest.fixture
def ctrl(ployconf, tempdir):
    from ploy import Controller
    import bsdploy
    import ploy_ezjail
    import ploy_ansible
    ployconf.fill([
        '[ez-master:jailhost]'])
    ctrl = Controller(configpath=ployconf.directory)
    ctrl.plugins = {
        'bsdploy': bsdploy.plugin,
        'ezjail': ploy_ezjail.plugin,
        'ansible': ploy_ansible.plugin}
    ctrl.configfile = ployconf.path
    return ctrl

def get_all_roles():
    roles_path = os.path.join(bsdploy_path, 'roles')
    roles = []
    for item in os.listdir(roles_path):
        if os.path.isdir(os.path.join(roles_path, item)):
            roles.append(item)
    return sorted(roles)

def iter_tasks(plays):
    for play in plays:
        for task in play.tasks():
            if task.meta:
                if task.meta == 'flush_handlers':  # pragma: nocover - branch coverage only on failure
                    continue
                raise ValueError  # pragma: nocover - only on failure
            yield play, task

def test_roles(ctrl, monkeypatch):
    instance = ctrl.instances['jailhost']
    pb = instance.get_playbook()
    plays = []
    monkeypatch.setattr('ansible.playbook.PlayBook._run_play', plays.append)
    pb.run()
    tasks = []
    for play, task in iter_tasks(plays):
        tasks.append(task.name)
    assert tasks == [
        'bind host sshd to primary ip',
        'Enable ntpd in rc.conf',
        'Disable public use of ntpd',
        'Check for old ipnat_rules setting',
        'Remove ipfilter from rc.conf',
        'Remove ipfilter_rules from rc.conf',
        'Remove ipmon from rc.conf',
        'Remove ipmon_flags from rc.conf',
        'Remove ipnat from rc.conf',
        'Remove ipnat_rules from rc.conf',
        'Enable pf in rc.conf',
        'Check for /etc/pf.conf',
        'Default pf.conf',
        'Stat of /dev/pf',
        'Checking pf',
        'Setup pf.conf',
        'Reload pf.conf',
        'Enable gateway in rc.conf',
        'Add lo1 to rc.conf',
        'Enable security.jail.allow_raw_sockets',
        'Enable security.jail.sysvipc_allowed',
        'Ensure helper packages are installed',
        'Set default jail interface',
        'Set default jail parameters',
        'Set default jail exec stop',
        'Enable ezjail in rc.conf',
        'Setup ezjail.conf',
        'Setup data zpool',
        'Set data zpool options',
        'Jails ZFS file system',
        'Initialize ezjail (using http proxy)',
        'Initialize ezjail (may take a while)',
        'Create pkg cache folder',
        'Download pkg.txz',
        'Directory for jail flavour "bsdploy_base"',
        'Pkg in bsdploy_base flavour',
        'The .ssh directory for root in bsdploy_base flavour',
        'The etc directory in bsdploy_base flavour',
        'The etc/ssh directory in bsdploy_base flavour',
        'copy src=make.conf dest=/usr/jails/flavours/bsdploy_base/etc/make.conf owner=root group=wheel',
        'file dest=/usr/jails/flavours/bsdploy_base/usr/local/etc/pkg/repos state=directory owner=root group=wheel',
        'template src=pkg.conf dest=/usr/jails/flavours/bsdploy_base/usr/local/etc/pkg.conf owner=root group=wheel',
        'template src=FreeBSD.conf dest=/usr/jails/flavours/bsdploy_base/usr/local/etc/pkg/repos/FreeBSD.conf owner=root group=wheel',
        'rc.conf for bsdploy_base flavour',
        'sshd_config for bsdploy_base flavour',
        'motd for bsdploy_base flavour',
        'copy some settings from host to bsdploy_base flavour']

def test_all_role_templates_tested(ctrl, monkeypatch, request):
    instance = ctrl.instances['jailhost']
    instance.config['roles'] = ' '.join(get_all_roles())
    pb = instance.get_playbook()
    plays = []
    monkeypatch.setattr('ansible.playbook.PlayBook._run_play', plays.append)
    pb.run()
    # import after running to avoid module import issues
    from ansible.utils import parse_kv, path_dwim_relative
    templates = []
    for play, task in iter_tasks(plays):
        if task.module_name != 'template':
            continue
        module_args_dict = task.args
        if not module_args_dict and task.module_args:
            module_args_dict = parse_kv(task.module_args)
        template_path = path_dwim_relative(
            task.module_vars['_original_file'], 'templates',
            module_args_dict['src'], play.basedir)
        if not os.path.exists(template_path):  # pragma: nocover - only on failure
            raise ValueError
        name = module_args_dict['src'].lower()
        for rep in ('-', '.'):
            name = name.replace(rep, '_')
        templates.append((
            name,
            dict(
                path=task.module_vars.get('_original_file'),
                role_name=task.role_name,
                name=module_args_dict['src'], task_name=task.name)))
    test_names = [x.name for x in request.session.items]
    for name, info in templates:
        test_name = 'test_%s_%s' % (info['role_name'], name)
        if not any(x for x in test_names if x.startswith(test_name)):  # pragma: nocover - only on failure
            pytest.fail("No test '{0}' for template '{name}' of task '{task_name}' in role '{role_name}' at '{path}'.".format(test_name, **info))
<commit_msg>Fix the roles test.<commit_after>from bsdploy import bsdploy_path
import os
import pytest

@pytest.fixture
def ctrl(ployconf, tempdir):
    from ploy import Controller
    import bsdploy
    import ploy_ezjail
    import ploy_ansible
    ployconf.fill([
        '[ez-master:jailhost]'])
    ctrl = Controller(configpath=ployconf.directory)
    ctrl.plugins = {
        'bsdploy': bsdploy.plugin,
        'ezjail': ploy_ezjail.plugin,
        'ansible': ploy_ansible.plugin}
    ctrl.configfile = ployconf.path
    return ctrl

def get_all_roles():
    roles_path = os.path.join(bsdploy_path, 'roles')
    roles = []
    for item in os.listdir(roles_path):
        if os.path.isdir(os.path.join(roles_path, item)):
            roles.append(item)
    return sorted(roles)

def iter_tasks(plays):
    for play in plays:
        for task in play.tasks():
            if task.meta:
                if task.meta == 'flush_handlers':  # pragma: nocover - branch coverage only on failure
                    continue
                raise ValueError  # pragma: nocover - only on failure
            yield play, task

def test_roles(ctrl, monkeypatch):
    instance = ctrl.instances['jailhost']
    pb = instance.get_playbook()
    plays = []
    monkeypatch.setattr('ansible.playbook.PlayBook._run_play', plays.append)
    pb.run()
    tasks = []
    for play, task in iter_tasks(plays):
        tasks.append(task.name)
    assert tasks == [
        'bind host sshd to primary ip',
        'Enable ntpd in rc.conf',
        'Disable public use of ntpd',
        'Check for old ipnat_rules setting',
        'Remove ipfilter from rc.conf',
        'Remove ipfilter_rules from rc.conf',
        'Remove ipmon from rc.conf',
        'Remove ipmon_flags from rc.conf',
        'Remove ipnat from rc.conf',
        'Remove ipnat_rules from rc.conf',
        'Enable pf in rc.conf',
        'Check for /etc/pf.conf',
        'Default pf.conf',
        'Stat of /dev/pf',
        'Checking pf',
        'Setup pf.conf',
        'Reload pf.conf',
        'Enable gateway in rc.conf',
        'Setup cloned interfaces',
        'Enable security.jail.allow_raw_sockets',
        'Enable security.jail.sysvipc_allowed',
        'Ensure helper packages are installed',
        'Set default jail interface',
        'Set default jail parameters',
        'Set default jail exec stop',
        'Enable ezjail in rc.conf',
        'Setup ezjail.conf',
        'Setup data zpool',
        'Set data zpool options',
        'Jails ZFS file system',
        'Initialize ezjail (using http proxy)',
        'Initialize ezjail (may take a while)',
        'Create pkg cache folder',
        'Download pkg.txz',
        'Directory for jail flavour "bsdploy_base"',
        'Pkg in bsdploy_base flavour',
        'The .ssh directory for root in bsdploy_base flavour',
        'The etc directory in bsdploy_base flavour',
        'The etc/ssh directory in bsdploy_base flavour',
        'copy src=make.conf dest=/usr/jails/flavours/bsdploy_base/etc/make.conf owner=root group=wheel',
        'file dest=/usr/jails/flavours/bsdploy_base/usr/local/etc/pkg/repos state=directory owner=root group=wheel',
        'template src=pkg.conf dest=/usr/jails/flavours/bsdploy_base/usr/local/etc/pkg.conf owner=root group=wheel',
        'template src=FreeBSD.conf dest=/usr/jails/flavours/bsdploy_base/usr/local/etc/pkg/repos/FreeBSD.conf owner=root group=wheel',
        'rc.conf for bsdploy_base flavour',
        'sshd_config for bsdploy_base flavour',
        'motd for bsdploy_base flavour',
        'copy some settings from host to bsdploy_base flavour']

def test_all_role_templates_tested(ctrl, monkeypatch, request):
    instance = ctrl.instances['jailhost']
    instance.config['roles'] = ' '.join(get_all_roles())
    pb = instance.get_playbook()
    plays = []
    monkeypatch.setattr('ansible.playbook.PlayBook._run_play', plays.append)
    pb.run()
    # import after running to avoid module import issues
    from ansible.utils import parse_kv, path_dwim_relative
    templates = []
    for play, task in iter_tasks(plays):
        if task.module_name != 'template':
            continue
        module_args_dict = task.args
        if not module_args_dict and task.module_args:
            module_args_dict = parse_kv(task.module_args)
        template_path = path_dwim_relative(
            task.module_vars['_original_file'], 'templates',
            module_args_dict['src'], play.basedir)
        if not os.path.exists(template_path):  # pragma: nocover - only on failure
            raise ValueError
        name = module_args_dict['src'].lower()
        for rep in ('-', '.'):
            name = name.replace(rep, '_')
        templates.append((
            name,
            dict(
                path=task.module_vars.get('_original_file'),
                role_name=task.role_name,
                name=module_args_dict['src'], task_name=task.name)))
    test_names = [x.name for x in request.session.items]
    for name, info in templates:
        test_name = 'test_%s_%s' % (info['role_name'], name)
        if not any(x for x in test_names if x.startswith(test_name)):  # pragma: nocover - only on failure
            pytest.fail("No test '{0}' for template '{name}' of task '{task_name}' in role '{role_name}' at '{path}'.".format(test_name, **info))
<|endoftext|>


<commit_before>"""Module that allows custom Sphinx parameters to be set on the notebook and
on the 'other' object passed into Jinja.  Called prior to Jinja conversion
process.
"""
#-----------------------------------------------------------------------------
# Copyright (c) 2013, the IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

from __future__ import print_function, absolute_import

# Stdlib imports
# Used to find Sphinx package location
import sphinx
import os.path

# Used to set the default date to today's date
from datetime import date

# Third-party imports
# Needed for Pygments latex definitions.
from pygments.formatters import LatexFormatter

# Our own imports
# Configurable traitlets
from IPython.utils.traitlets import Unicode, Bool

# Needed to override transformer
from .activatable import (ActivatableTransformer) #TODO

from IPython.nbconvert.utils import console

#-----------------------------------------------------------------------------
# Classes and functions
#-----------------------------------------------------------------------------

class SphinxTransformer(ActivatableTransformer):
    """
    Sphinx utility transformer.

    This transformer is used to set variables needed by the latex to build
    Sphinx stylized templates.
    """

    interactive = Bool(False, config=True, help="""
        Allows you to define whether or not the Sphinx exporter will prompt
        you for input during the conversion process.  If this is set to false,
        the author, version, release, date, and chapter_style traits should
        be set.
        """)

    author = Unicode("Unknown Author", config=True, help="Author name")

    version = Unicode("", config=True, help="""
        Version number
        You can leave this blank if you do not want to render a version number.
        Example: "1.0.0"
        """)

    release = Unicode("", config=True, help="""
        Release name
        You can leave this blank if you do not want to render a release name.
        Example: "Rough Draft"
        """)

    publish_date = Unicode("", config=True, help="""
        Publish date
        This is the date to render on the document as the publish date.
        Leave this blank to default to todays date.
        Example: "June 12, 1990"
        """)

    chapter_style = Unicode("Bjarne", config=True, help="""
        Sphinx chapter style
        This is the style to use for the chapter headers in the document.
        You may choose one of the following:
            "Bjarne"    (default)
            "Lenny"
            "Glenn"
            "Conny"
            "Rejne"
            "Sonny"    (used for international documents)
        """)

    output_style = Unicode("notebook", config=True, help="""
        Nbconvert Ipython
        notebook input/output formatting style.
        You may choose one of the following:
            "simple     (recommended for long code segments)"
            "notebook"  (default)
        """)

    center_output = Bool(False, config=True, help="""
        Optional attempt to center all output.  If this is false, no additional
        formatting is applied.
        """)

    use_headers = Bool(True, config=True, help="""
        Whether not a header should be added to the document.
        """)

    #Allow the user to override the title of the notebook (useful for
    #fancy document titles that the file system doesn't support.)
    overridetitle = Unicode("", config=True, help="")

    def call(self, nb, resources):
        """
        Sphinx transformation to apply on each notebook.

        Parameters
        ----------
        nb : NotebookNode
            Notebook being converted
        resources : dictionary
            Additional resources used in the conversion process.  Allows
            transformers to pass variables into the Jinja engine.
        """

        # TODO: Add versatile method of additional notebook metadata.  Include
        #       handling of multiple files.  For now use a temporay namespace,
        #       '_draft' to signify that this needs to change.
        if not "_draft" in nb.metadata:
            nb.metadata._draft = {}

        if not "sphinx" in resources:
            resources["sphinx"] = {}

        if self.interactive:

            # Prompt the user for additional meta data that doesn't exist currently
            # but would be usefull for Sphinx.
            nb.metadata._draft["author"] = self._prompt_author()
            nb.metadata._draft["version"] = self._prompt_version()
            nb.metadata._draft["release"] = self._prompt_release()
            nb.metadata._draft["date"] = self._prompt_date()

            # Prompt the user for the document style.
            resources["sphinx"]["chapterstyle"] = self._prompt_chapter_title_style()
            resources["sphinx"]["outputstyle"] = self._prompt_output_style()

            # Small options
            resources["sphinx"]["centeroutput"] = console.prompt_boolean("Do you want to center the output? (false)", False)
            resources["sphinx"]["header"] = console.prompt_boolean("Should a Sphinx document header be used? (true)", True)
        else:

            # Try to use the traitlets.
            nb.metadata._draft["author"] = self.author
            nb.metadata._draft["version"] = self.version
            nb.metadata._draft["release"] = self.release

            # Use todays date if none is provided.
            if len(self.publish_date.strip()) == 0:
                nb.metadata._draft["date"] = date.today().strftime("%B %-d, %Y")
            else:
                nb.metadata._draft["date"] = self.publish_date

            # Sphinx traitlets.
            resources["sphinx"]["chapterstyle"] = self.chapter_style
            resources["sphinx"]["outputstyle"] = self.output_style
            resources["sphinx"]["centeroutput"] = self.center_output
            resources["sphinx"]["header"] = self.use_headers

        # Find and pass in the path to the Sphinx dependencies.
        resources["sphinx"]["texinputs"] = os.path.realpath(os.path.join(sphinx.__file__, "..", "texinputs"))

        # Generate Pygments definitions for Latex
        resources["sphinx"]["pygment_definitions"] = self._generate_pygments_latex_def()

        if not (self.overridetitle == None or len(self.overridetitle.strip()) == 0):
            nb.metadata.name = self.overridetitle

        # End
        return nb, resources

    def _generate_pygments_latex_def(self):
        """
        Generate the pygments latex definitions that allows pygments
        to work in latex.
        """

        return LatexFormatter().get_style_defs()

    def _prompt_author(self):
        """
        Prompt the user to input an Author name
        """
        return  console.input("Author name: ")

    def _prompt_version(self):
        """
        prompt the user to enter a version number
        """
        return  console.input("Version (ie ""1.0.0""): ")

    def _prompt_release(self):
        """
        Prompt the user to input a release name
        """

        return  console.input("Release Name (ie ""Rough draft""): ")

    def _prompt_date(self):
        """
        Prompt the user to enter a date
        """

        default_date = date.today().strftime("%B %-d, %Y")
        user_date = console.input("Date (deafults to \"" + default_date + "\"): ")
        if len(user_date.strip()) == 0:
            user_date = default_date
        return user_date

    def _prompt_output_style(self):
        """
        Prompts the user to pick an IPython output style.
        """

        # Dictionary of available output styles
        styles = {1: "simple",
                  2: "notebook"}

        #Append comments to the menu when displaying it to the user.
        comments = {1: "(recommended for long code segments)",
                    2: "(default)"}

        return console.prompt_dictionary(styles, default_style=2, menu_comments=comments)

    def _prompt_chapter_title_style(self):
        """
        Prompts the user to pick a Sphinx chapter style
        """

        # Dictionary of available Sphinx styles
        styles = {1: "Bjarne",
                  2: "Lenny",
                  3: "Glenn",
                  4: "Conny",
                  5: "Rejne",
                  6: "Sonny"}

        #Append comments to the menu when displaying it to the user.
        comments = {1: "(default)",
                    6: "(for international documents)"}

        return console.prompt_dictionary(styles, menu_comments=comments)

<commit_msg>Corrected a typo in the format string for strftime<commit_after>"""Module that allows custom Sphinx parameters to be set on the notebook and
on the 'other' object passed into Jinja.  Called prior to Jinja conversion
process.
"""
#-----------------------------------------------------------------------------
# Copyright (c) 2013, the IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

from __future__ import print_function, absolute_import

# Stdlib imports
# Used to find Sphinx package location
import sphinx
import os.path

# Used to set the default date to today's date
from datetime import date

# Third-party imports
# Needed for Pygments latex definitions.
from pygments.formatters import LatexFormatter

# Our own imports
# Configurable traitlets
from IPython.utils.traitlets import Unicode, Bool

# Needed to override transformer
from .activatable import (ActivatableTransformer) #TODO

from IPython.nbconvert.utils import console

#-----------------------------------------------------------------------------
# Classes and functions
#-----------------------------------------------------------------------------

class SphinxTransformer(ActivatableTransformer):
    """
    Sphinx utility transformer.

    This transformer is used to set variables needed by the latex to build
    Sphinx stylized templates.
    """

    interactive = Bool(False, config=True, help="""
        Allows you to define whether or not the Sphinx exporter will prompt
        you for input during the conversion process.  If this is set to false,
        the author, version, release, date, and chapter_style traits should
        be set.
        """)

    author = Unicode("Unknown Author", config=True, help="Author name")

    version = Unicode("", config=True, help="""
        Version number
        You can leave this blank if you do not want to render a version number.
        Example: "1.0.0"
        """)

    release = Unicode("", config=True, help="""
        Release name
        You can leave this blank if you do not want to render a release name.
        Example: "Rough Draft"
        """)

    publish_date = Unicode("", config=True, help="""
        Publish date
        This is the date to render on the document as the publish date.
        Leave this blank to default to todays date.
        Example: "June 12, 1990"
        """)

    chapter_style = Unicode("Bjarne", config=True, help="""
        Sphinx chapter style
        This is the style to use for the chapter headers in the document.
        You may choose one of the following:
            "Bjarne"    (default)
            "Lenny"
            "Glenn"
            "Conny"
            "Rejne"
            "Sonny"    (used for international documents)
        """)

    output_style = Unicode("notebook", config=True, help="""
        Nbconvert Ipython
        notebook input/output formatting style.
        You may choose one of the following:
            "simple     (recommended for long code segments)"
            "notebook"  (default)
        """)

    center_output = Bool(False, config=True, help="""
        Optional attempt to center all output.  If this is false, no additional
        formatting is applied.
        """)

    use_headers = Bool(True, config=True, help="""
        Whether not a header should be added to the document.
        """)

    #Allow the user to override the title of the notebook (useful for
    #fancy document titles that the file system doesn't support.)
    overridetitle = Unicode("", config=True, help="")

    def call(self, nb, resources):
        """
        Sphinx transformation to apply on each notebook.

        Parameters
        ----------
        nb : NotebookNode
            Notebook being converted
        resources : dictionary
            Additional resources used in the conversion process.  Allows
            transformers to pass variables into the Jinja engine.
        """

        # TODO: Add versatile method of additional notebook metadata.  Include
        #       handling of multiple files.  For now use a temporay namespace,
        #       '_draft' to signify that this needs to change.
        if not "_draft" in nb.metadata:
            nb.metadata._draft = {}

        if not "sphinx" in resources:
            resources["sphinx"] = {}

        if self.interactive:

            # Prompt the user for additional meta data that doesn't exist currently
            # but would be usefull for Sphinx.
            nb.metadata._draft["author"] = self._prompt_author()
            nb.metadata._draft["version"] = self._prompt_version()
            nb.metadata._draft["release"] = self._prompt_release()
            nb.metadata._draft["date"] = self._prompt_date()

            # Prompt the user for the document style.
            resources["sphinx"]["chapterstyle"] = self._prompt_chapter_title_style()
            resources["sphinx"]["outputstyle"] = self._prompt_output_style()

            # Small options
            resources["sphinx"]["centeroutput"] = console.prompt_boolean("Do you want to center the output? (false)", False)
            resources["sphinx"]["header"] = console.prompt_boolean("Should a Sphinx document header be used? (true)", True)
        else:

            # Try to use the traitlets.
            nb.metadata._draft["author"] = self.author
            nb.metadata._draft["version"] = self.version
            nb.metadata._draft["release"] = self.release

            # Use todays date if none is provided.
            if len(self.publish_date.strip()) == 0:
                nb.metadata._draft["date"] = date.today().strftime("%B %d, %Y")
            else:
                nb.metadata._draft["date"] = self.publish_date

            # Sphinx traitlets.
            resources["sphinx"]["chapterstyle"] = self.chapter_style
            resources["sphinx"]["outputstyle"] = self.output_style
            resources["sphinx"]["centeroutput"] = self.center_output
            resources["sphinx"]["header"] = self.use_headers

        # Find and pass in the path to the Sphinx dependencies.
        resources["sphinx"]["texinputs"] = os.path.realpath(os.path.join(sphinx.__file__, "..", "texinputs"))

        # Generate Pygments definitions for Latex
        resources["sphinx"]["pygment_definitions"] = self._generate_pygments_latex_def()

        if not (self.overridetitle == None or len(self.overridetitle.strip()) == 0):
            nb.metadata.name = self.overridetitle

        # End
        return nb, resources

    def _generate_pygments_latex_def(self):
        """
        Generate the pygments latex definitions that allows pygments
        to work in latex.
        """

        return LatexFormatter().get_style_defs()

    def _prompt_author(self):
        """
        Prompt the user to input an Author name
        """
        return  console.input("Author name: ")

    def _prompt_version(self):
        """
        prompt the user to enter a version number
        """
        return  console.input("Version (ie ""1.0.0""): ")

    def _prompt_release(self):
        """
        Prompt the user to input a release name
        """

        return  console.input("Release Name (ie ""Rough draft""): ")

    def _prompt_date(self):
        """
        Prompt the user to enter a date
        """

        default_date = date.today().strftime("%B %d, %Y")
        user_date = console.input("Date (deafults to \"" + default_date + "\"): ")
        if len(user_date.strip()) == 0:
            user_date = default_date
        return user_date

    def _prompt_output_style(self):
        """
        Prompts the user to pick an IPython output style.
        """

        # Dictionary of available output styles
        styles = {1: "simple",
                  2: "notebook"}

        #Append comments to the menu when displaying it to the user.
        comments = {1: "(recommended for long code segments)",
                    2: "(default)"}

        return console.prompt_dictionary(styles, default_style=2, menu_comments=comments)

    def _prompt_chapter_title_style(self):
        """
        Prompts the user to pick a Sphinx chapter style
        """

        # Dictionary of available Sphinx styles
        styles = {1: "Bjarne",
                  2: "Lenny",
                  3: "Glenn",
                  4: "Conny",
                  5: "Rejne",
                  6: "Sonny"}

        #Append comments to the menu when displaying it to the user.
        comments = {1: "(default)",
                    6: "(for international documents)"}

        return console.prompt_dictionary(styles, menu_comments=comments)

<|endoftext|>