<commit_before># Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Common operations for RNN Estimators."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensorflow.contrib import metrics
from tensorflow.contrib import rnn as contrib_rnn
from tensorflow.contrib.learn.python.learn.estimators import constants
from tensorflow.contrib.learn.python.learn.estimators import prediction_key
from tensorflow.python.framework import ops
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import math_ops

# NOTE(jtbates): As of February 10, 2017, some of the `RNNKeys` have been
# removed and replaced with values from `prediction_key.PredictionKey`. The key
# `RNNKeys.PREDICTIONS_KEY` has been replaced by
# `prediction_key.PredictionKey.SCORES` for regression and
# `prediction_key.PredictionKey.CLASSES` for classification. The key
# `RNNKeys.PROBABILITIES_KEY` has been replaced by
# `prediction_key.PredictionKey.PROBABILITIES`.
class RNNKeys(object):
  FINAL_STATE_KEY = 'final_state'
  LABELS_KEY = '__labels__'
  SEQUENCE_LENGTH_KEY = 'sequence_length'
  STATE_PREFIX = 'rnn_cell_state'

class PredictionType(object):
  """Enum-like values for the type of prediction that the model makes.
  """
  SINGLE_VALUE = 1
  MULTIPLE_VALUE = 2

_CELL_TYPES = {'basic_rnn': contrib_rnn.BasicRNNCell,
               'lstm': contrib_rnn.LSTMCell,
               'gru': contrib_rnn.GRUCell,}

def _get_single_cell(cell_type, num_units):
  """Constructs and return a single `RNNCell`.

  Args:
    cell_type: Either a string identifying the `RNNCell` type or a subclass of
      `RNNCell`.
    num_units: The number of units in the `RNNCell`.
  Returns:
    An initialized `RNNCell`.
  Raises:
    ValueError: `cell_type` is an invalid `RNNCell` name.
    TypeError: `cell_type` is not a string or a subclass of `RNNCell`.
  """
  cell_type = _CELL_TYPES.get(cell_type, cell_type)
  if not cell_type or not issubclass(cell_type, contrib_rnn.RNNCell):
    raise ValueError('The supported cell types are {}; got {}'.format(
        list(_CELL_TYPES.keys()), cell_type))
  return cell_type(num_units=num_units)

def construct_rnn_cell(num_units, cell_type='basic_rnn',
                       dropout_keep_probabilities=None):
  """Constructs cells, applies dropout and assembles a `MultiRNNCell`.

  The cell type chosen by DynamicRNNEstimator.__init__() is the same as
  returned by this function when called with the same arguments.

  Args:
    num_units: A single `int` or a list/tuple of `int`s. The size of the
      `RNNCell`s.
    cell_type: A string identifying the `RNNCell` type or a subclass of
      `RNNCell`.
    dropout_keep_probabilities: a list of dropout probabilities or `None`. If a
      list is given, it must have length `len(cell_type) + 1`.

  Returns:
    An initialized `RNNCell`.
  """
  if not isinstance(num_units, (list, tuple)):
    num_units = (num_units,)

  cells = [_get_single_cell(cell_type, n) for n in num_units]
  if dropout_keep_probabilities:
    cells = apply_dropout(cells, dropout_keep_probabilities)
  if len(cells) == 1:
    return cells[0]
  return contrib_rnn.MultiRNNCell(cells)

def apply_dropout(cells, dropout_keep_probabilities, random_seed=None):
  """Applies dropout to the outputs and inputs of `cell`.

  Args:
    cells: A list of `RNNCell`s.
    dropout_keep_probabilities: a list whose elements are either floats in
    `[0.0, 1.0]` or `None`. It must have length one greater than `cells`.
    random_seed: Seed for random dropout.

  Returns:
    A list of `RNNCell`s, the result of applying the supplied dropouts.

  Raises:
    ValueError: If `len(dropout_keep_probabilities) != len(cells) + 1`.
  """
  if len(dropout_keep_probabilities) != len(cells) + 1:
    raise ValueError(
        'The number of dropout probabilities must be one greater than the '
        'number of cells. Got {} cells and {} dropout probabilities.'.format(
            len(cells), len(dropout_keep_probabilities)))
  wrapped_cells = [
      contrib_rnn.DropoutWrapper(cell, prob, 1.0, seed=random_seed)
      for cell, prob in zip(cells[:-1], dropout_keep_probabilities[:-2])
  ]
  wrapped_cells.append(
      contrib_rnn.DropoutWrapper(cells[-1], dropout_keep_probabilities[-2],
                                 dropout_keep_probabilities[-1]))
  return wrapped_cells

def get_eval_metric_ops(problem_type, prediction_type, sequence_length,
                        prediction_dict, labels):
  """Returns eval metric ops for given `problem_type` and `prediction_type`.

  Args:
    problem_type: `ProblemType.CLASSIFICATION` or
      `ProblemType.LINEAR_REGRESSION`.
    prediction_type: `PredictionType.SINGLE_VALUE` or
      `PredictionType.MULTIPLE_VALUE`.
    sequence_length: A `Tensor` with shape `[batch_size]` and dtype `int32`
      containing the length of each sequence in the batch. If `None`, sequences
      are assumed to be unpadded.
    prediction_dict: A dict of prediction tensors.
    labels: The label `Tensor`.

  Returns:
    A `dict` mapping strings to the result of calling the metric_fn.
  """
  eval_metric_ops = {}
  if problem_type == constants.ProblemType.CLASSIFICATION:
    # Multi value classification
    if prediction_type == PredictionType.MULTIPLE_VALUE:
      mask_predictions, mask_labels = mask_activations_and_labels(
          prediction_dict[prediction_key.PredictionKey.CLASSES], labels,
          sequence_length)
      eval_metric_ops['accuracy'] = metrics.streaming_accuracy(
          predictions=mask_predictions, labels=mask_labels)
    # Single value classification
    elif prediction_type == PredictionType.SINGLE_VALUE:
      eval_metric_ops['accuracy'] = metrics.streaming_accuracy(
          predictions=prediction_dict[prediction_key.PredictionKey.CLASSES],
          labels=labels)
  elif problem_type == constants.ProblemType.LINEAR_REGRESSION:
    # Multi value regression
    if prediction_type == PredictionType.MULTIPLE_VALUE:
      pass
    # Single value regression
    elif prediction_type == PredictionType.SINGLE_VALUE:
      pass
  return eval_metric_ops

def select_last_activations(activations, sequence_lengths):
  """Selects the nth set of activations for each n in `sequence_length`.

  Reuturns a `Tensor` of shape `[batch_size, k]`. If `sequence_length` is not
  `None`, then `output[i, :] = activations[i, sequence_length[i], :]`. If
  `sequence_length` is `None`, then `output[i, :] = activations[i, -1, :]`.

  Args:
    activations: A `Tensor` with shape `[batch_size, padded_length, k]`.
    sequence_lengths: A `Tensor` with shape `[batch_size]` or `None`.
  Returns:
    A `Tensor` of shape `[batch_size, k]`.
  """
  with ops.name_scope(
      'select_last_activations', values=[activations, sequence_lengths]):
    activations_shape = array_ops.shape(activations)
    batch_size = activations_shape[0]
    padded_length = activations_shape[1]
    num_label_columns = activations_shape[2]
    if sequence_lengths is None:
      sequence_lengths = padded_length
    reshaped_activations = array_ops.reshape(activations,
                                             [-1, num_label_columns])
    indices = math_ops.range(batch_size) * padded_length + sequence_lengths - 1
    last_activations = array_ops.gather(reshaped_activations, indices)
    last_activations.set_shape(
        [activations.get_shape()[0], activations.get_shape()[2]])
    return last_activations

def mask_activations_and_labels(activations, labels, sequence_lengths):
  """Remove entries outside `sequence_lengths` and returned flattened results.

  Args:
    activations: Output of the RNN, shape `[batch_size, padded_length, k]`.
    labels: Label values, shape `[batch_size, padded_length]`.
    sequence_lengths: A `Tensor` of shape `[batch_size]` with the unpadded
      length of each sequence. If `None`, then each sequence is unpadded.

  Returns:
    activations_masked: `logit` values with those beyond `sequence_lengths`
      removed for each batch. Batches are then concatenated. Shape
      `[tf.sum(sequence_lengths), k]` if `sequence_lengths` is not `None` and
      shape `[batch_size * padded_length, k]` otherwise.
    labels_masked: Label values after removing unneeded entries. Shape
      `[tf.sum(sequence_lengths)]` if `sequence_lengths` is not `None` and shape
      `[batch_size * padded_length]` otherwise.
  """
  with ops.name_scope(
      'mask_activations_and_labels',
      values=[activations, labels, sequence_lengths]):
    labels_shape = array_ops.shape(labels)
    batch_size = labels_shape[0]
    padded_length = labels_shape[1]
    if sequence_lengths is None:
      flattened_dimension = padded_length * batch_size
      activations_masked = array_ops.reshape(activations,
                                             [flattened_dimension, -1])
      labels_masked = array_ops.reshape(labels, [flattened_dimension])
    else:
      mask = array_ops.sequence_mask(sequence_lengths, padded_length)
      activations_masked = array_ops.boolean_mask(activations, mask)
      labels_masked = array_ops.boolean_mask(labels, mask)
    return activations_masked, labels_masked

def multi_value_predictions(activations, target_column, problem_type,
                            predict_probabilities):
  """Maps `activations` from the RNN to predictions for multi value models.

  If `predict_probabilities` is `False`, this function returns a `dict`
  containing single entry with key `prediction_key.PredictionKey.CLASSES` for
  `problem_type` `ProblemType.CLASSIFICATION` or
  `prediction_key.PredictionKey.SCORE` for `problem_type`
  `ProblemType.LINEAR_REGRESSION`.

  If `predict_probabilities` is `True`, it will contain a second entry with key
  `prediction_key.PredictionKey.PROBABILITIES`. The
  value of this entry is a `Tensor` of probabilities with shape
  `[batch_size, padded_length, num_classes]`.

  Note that variable length inputs will yield some predictions that don't have
  meaning. For example, if `sequence_length = [3, 2]`, then prediction `[1, 2]`
  has no meaningful interpretation.

  Args:
    activations: Output from an RNN. Should have dtype `float32` and shape
      `[batch_size, padded_length, ?]`.
    target_column: An initialized `TargetColumn`, calculate predictions.
    problem_type: Either `ProblemType.CLASSIFICATION` or
      `ProblemType.LINEAR_REGRESSION`.
    predict_probabilities: A Python boolean, indicating whether probabilities
      should be returned. Should only be set to `True` for
      classification/logistic regression problems.
  Returns:
    A `dict` mapping strings to `Tensors`.
  """
  with ops.name_scope('MultiValuePrediction'):
    activations_shape = array_ops.shape(activations)
    flattened_activations = array_ops.reshape(activations,
                                              [-1, activations_shape[2]])
    prediction_dict = {}
    if predict_probabilities:
      flat_probabilities = target_column.logits_to_predictions(
          flattened_activations, proba=True)
      flat_predictions = math_ops.argmax(flat_probabilities, 1)
      if target_column.num_label_columns == 1:
        probability_shape = array_ops.concat([activations_shape[:2], [2]], 0)
      else:
        probability_shape = activations_shape
      probabilities = array_ops.reshape(
          flat_probabilities,
          probability_shape,
          name=prediction_key.PredictionKey.PROBABILITIES)
      prediction_dict[
          prediction_key.PredictionKey.PROBABILITIES] = probabilities
    else:
      flat_predictions = target_column.logits_to_predictions(
          flattened_activations, proba=False)
    predictions_name = (prediction_key.PredictionKey.CLASSES
                        if problem_type == constants.ProblemType.CLASSIFICATION
                        else prediction_key.PredictionKey.SCORES)
    predictions = array_ops.reshape(
        flat_predictions, [activations_shape[0], activations_shape[1]],
        name=predictions_name)
    prediction_dict[predictions_name] = predictions
    return prediction_dict
<commit_msg>Fix off-by-one documentation error.<commit_after># Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Common operations for RNN Estimators."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensorflow.contrib import metrics
from tensorflow.contrib import rnn as contrib_rnn
from tensorflow.contrib.learn.python.learn.estimators import constants
from tensorflow.contrib.learn.python.learn.estimators import prediction_key
from tensorflow.python.framework import ops
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import math_ops

# NOTE(jtbates): As of February 10, 2017, some of the `RNNKeys` have been
# removed and replaced with values from `prediction_key.PredictionKey`. The key
# `RNNKeys.PREDICTIONS_KEY` has been replaced by
# `prediction_key.PredictionKey.SCORES` for regression and
# `prediction_key.PredictionKey.CLASSES` for classification. The key
# `RNNKeys.PROBABILITIES_KEY` has been replaced by
# `prediction_key.PredictionKey.PROBABILITIES`.
class RNNKeys(object):
  FINAL_STATE_KEY = 'final_state'
  LABELS_KEY = '__labels__'
  SEQUENCE_LENGTH_KEY = 'sequence_length'
  STATE_PREFIX = 'rnn_cell_state'

class PredictionType(object):
  """Enum-like values for the type of prediction that the model makes.
  """
  SINGLE_VALUE = 1
  MULTIPLE_VALUE = 2

_CELL_TYPES = {'basic_rnn': contrib_rnn.BasicRNNCell,
               'lstm': contrib_rnn.LSTMCell,
               'gru': contrib_rnn.GRUCell,}

def _get_single_cell(cell_type, num_units):
  """Constructs and return a single `RNNCell`.

  Args:
    cell_type: Either a string identifying the `RNNCell` type or a subclass of
      `RNNCell`.
    num_units: The number of units in the `RNNCell`.
  Returns:
    An initialized `RNNCell`.
  Raises:
    ValueError: `cell_type` is an invalid `RNNCell` name.
    TypeError: `cell_type` is not a string or a subclass of `RNNCell`.
  """
  cell_type = _CELL_TYPES.get(cell_type, cell_type)
  if not cell_type or not issubclass(cell_type, contrib_rnn.RNNCell):
    raise ValueError('The supported cell types are {}; got {}'.format(
        list(_CELL_TYPES.keys()), cell_type))
  return cell_type(num_units=num_units)

def construct_rnn_cell(num_units, cell_type='basic_rnn',
                       dropout_keep_probabilities=None):
  """Constructs cells, applies dropout and assembles a `MultiRNNCell`.

  The cell type chosen by DynamicRNNEstimator.__init__() is the same as
  returned by this function when called with the same arguments.

  Args:
    num_units: A single `int` or a list/tuple of `int`s. The size of the
      `RNNCell`s.
    cell_type: A string identifying the `RNNCell` type or a subclass of
      `RNNCell`.
    dropout_keep_probabilities: a list of dropout probabilities or `None`. If a
      list is given, it must have length `len(cell_type) + 1`.

  Returns:
    An initialized `RNNCell`.
  """
  if not isinstance(num_units, (list, tuple)):
    num_units = (num_units,)

  cells = [_get_single_cell(cell_type, n) for n in num_units]
  if dropout_keep_probabilities:
    cells = apply_dropout(cells, dropout_keep_probabilities)
  if len(cells) == 1:
    return cells[0]
  return contrib_rnn.MultiRNNCell(cells)

def apply_dropout(cells, dropout_keep_probabilities, random_seed=None):
  """Applies dropout to the outputs and inputs of `cell`.

  Args:
    cells: A list of `RNNCell`s.
    dropout_keep_probabilities: a list whose elements are either floats in
    `[0.0, 1.0]` or `None`. It must have length one greater than `cells`.
    random_seed: Seed for random dropout.

  Returns:
    A list of `RNNCell`s, the result of applying the supplied dropouts.

  Raises:
    ValueError: If `len(dropout_keep_probabilities) != len(cells) + 1`.
  """
  if len(dropout_keep_probabilities) != len(cells) + 1:
    raise ValueError(
        'The number of dropout probabilities must be one greater than the '
        'number of cells. Got {} cells and {} dropout probabilities.'.format(
            len(cells), len(dropout_keep_probabilities)))
  wrapped_cells = [
      contrib_rnn.DropoutWrapper(cell, prob, 1.0, seed=random_seed)
      for cell, prob in zip(cells[:-1], dropout_keep_probabilities[:-2])
  ]
  wrapped_cells.append(
      contrib_rnn.DropoutWrapper(cells[-1], dropout_keep_probabilities[-2],
                                 dropout_keep_probabilities[-1]))
  return wrapped_cells

def get_eval_metric_ops(problem_type, prediction_type, sequence_length,
                        prediction_dict, labels):
  """Returns eval metric ops for given `problem_type` and `prediction_type`.

  Args:
    problem_type: `ProblemType.CLASSIFICATION` or
      `ProblemType.LINEAR_REGRESSION`.
    prediction_type: `PredictionType.SINGLE_VALUE` or
      `PredictionType.MULTIPLE_VALUE`.
    sequence_length: A `Tensor` with shape `[batch_size]` and dtype `int32`
      containing the length of each sequence in the batch. If `None`, sequences
      are assumed to be unpadded.
    prediction_dict: A dict of prediction tensors.
    labels: The label `Tensor`.

  Returns:
    A `dict` mapping strings to the result of calling the metric_fn.
  """
  eval_metric_ops = {}
  if problem_type == constants.ProblemType.CLASSIFICATION:
    # Multi value classification
    if prediction_type == PredictionType.MULTIPLE_VALUE:
      mask_predictions, mask_labels = mask_activations_and_labels(
          prediction_dict[prediction_key.PredictionKey.CLASSES], labels,
          sequence_length)
      eval_metric_ops['accuracy'] = metrics.streaming_accuracy(
          predictions=mask_predictions, labels=mask_labels)
    # Single value classification
    elif prediction_type == PredictionType.SINGLE_VALUE:
      eval_metric_ops['accuracy'] = metrics.streaming_accuracy(
          predictions=prediction_dict[prediction_key.PredictionKey.CLASSES],
          labels=labels)
  elif problem_type == constants.ProblemType.LINEAR_REGRESSION:
    # Multi value regression
    if prediction_type == PredictionType.MULTIPLE_VALUE:
      pass
    # Single value regression
    elif prediction_type == PredictionType.SINGLE_VALUE:
      pass
  return eval_metric_ops

def select_last_activations(activations, sequence_lengths):
  """Selects the nth set of activations for each n in `sequence_length`.

  Reuturns a `Tensor` of shape `[batch_size, k]`. If `sequence_length` is not
  `None`, then `output[i, :] = activations[i, sequence_length[i] - 1, :]`. If
  `sequence_length` is `None`, then `output[i, :] = activations[i, -1, :]`.

  Args:
    activations: A `Tensor` with shape `[batch_size, padded_length, k]`.
    sequence_lengths: A `Tensor` with shape `[batch_size]` or `None`.
  Returns:
    A `Tensor` of shape `[batch_size, k]`.
  """
  with ops.name_scope(
      'select_last_activations', values=[activations, sequence_lengths]):
    activations_shape = array_ops.shape(activations)
    batch_size = activations_shape[0]
    padded_length = activations_shape[1]
    num_label_columns = activations_shape[2]
    if sequence_lengths is None:
      sequence_lengths = padded_length
    reshaped_activations = array_ops.reshape(activations,
                                             [-1, num_label_columns])
    indices = math_ops.range(batch_size) * padded_length + sequence_lengths - 1
    last_activations = array_ops.gather(reshaped_activations, indices)
    last_activations.set_shape(
        [activations.get_shape()[0], activations.get_shape()[2]])
    return last_activations

def mask_activations_and_labels(activations, labels, sequence_lengths):
  """Remove entries outside `sequence_lengths` and returned flattened results.

  Args:
    activations: Output of the RNN, shape `[batch_size, padded_length, k]`.
    labels: Label values, shape `[batch_size, padded_length]`.
    sequence_lengths: A `Tensor` of shape `[batch_size]` with the unpadded
      length of each sequence. If `None`, then each sequence is unpadded.

  Returns:
    activations_masked: `logit` values with those beyond `sequence_lengths`
      removed for each batch. Batches are then concatenated. Shape
      `[tf.sum(sequence_lengths), k]` if `sequence_lengths` is not `None` and
      shape `[batch_size * padded_length, k]` otherwise.
    labels_masked: Label values after removing unneeded entries. Shape
      `[tf.sum(sequence_lengths)]` if `sequence_lengths` is not `None` and shape
      `[batch_size * padded_length]` otherwise.
  """
  with ops.name_scope(
      'mask_activations_and_labels',
      values=[activations, labels, sequence_lengths]):
    labels_shape = array_ops.shape(labels)
    batch_size = labels_shape[0]
    padded_length = labels_shape[1]
    if sequence_lengths is None:
      flattened_dimension = padded_length * batch_size
      activations_masked = array_ops.reshape(activations,
                                             [flattened_dimension, -1])
      labels_masked = array_ops.reshape(labels, [flattened_dimension])
    else:
      mask = array_ops.sequence_mask(sequence_lengths, padded_length)
      activations_masked = array_ops.boolean_mask(activations, mask)
      labels_masked = array_ops.boolean_mask(labels, mask)
    return activations_masked, labels_masked

def multi_value_predictions(activations, target_column, problem_type,
                            predict_probabilities):
  """Maps `activations` from the RNN to predictions for multi value models.

  If `predict_probabilities` is `False`, this function returns a `dict`
  containing single entry with key `prediction_key.PredictionKey.CLASSES` for
  `problem_type` `ProblemType.CLASSIFICATION` or
  `prediction_key.PredictionKey.SCORE` for `problem_type`
  `ProblemType.LINEAR_REGRESSION`.

  If `predict_probabilities` is `True`, it will contain a second entry with key
  `prediction_key.PredictionKey.PROBABILITIES`. The
  value of this entry is a `Tensor` of probabilities with shape
  `[batch_size, padded_length, num_classes]`.

  Note that variable length inputs will yield some predictions that don't have
  meaning. For example, if `sequence_length = [3, 2]`, then prediction `[1, 2]`
  has no meaningful interpretation.

  Args:
    activations: Output from an RNN. Should have dtype `float32` and shape
      `[batch_size, padded_length, ?]`.
    target_column: An initialized `TargetColumn`, calculate predictions.
    problem_type: Either `ProblemType.CLASSIFICATION` or
      `ProblemType.LINEAR_REGRESSION`.
    predict_probabilities: A Python boolean, indicating whether probabilities
      should be returned. Should only be set to `True` for
      classification/logistic regression problems.
  Returns:
    A `dict` mapping strings to `Tensors`.
  """
  with ops.name_scope('MultiValuePrediction'):
    activations_shape = array_ops.shape(activations)
    flattened_activations = array_ops.reshape(activations,
                                              [-1, activations_shape[2]])
    prediction_dict = {}
    if predict_probabilities:
      flat_probabilities = target_column.logits_to_predictions(
          flattened_activations, proba=True)
      flat_predictions = math_ops.argmax(flat_probabilities, 1)
      if target_column.num_label_columns == 1:
        probability_shape = array_ops.concat([activations_shape[:2], [2]], 0)
      else:
        probability_shape = activations_shape
      probabilities = array_ops.reshape(
          flat_probabilities,
          probability_shape,
          name=prediction_key.PredictionKey.PROBABILITIES)
      prediction_dict[
          prediction_key.PredictionKey.PROBABILITIES] = probabilities
    else:
      flat_predictions = target_column.logits_to_predictions(
          flattened_activations, proba=False)
    predictions_name = (prediction_key.PredictionKey.CLASSES
                        if problem_type == constants.ProblemType.CLASSIFICATION
                        else prediction_key.PredictionKey.SCORES)
    predictions = array_ops.reshape(
        flat_predictions, [activations_shape[0], activations_shape[1]],
        name=predictions_name)
    prediction_dict[predictions_name] = predictions
    return prediction_dict
<|endoftext|>


<commit_before>import matplotlib.pyplot as plt

###################################################################################################
# Adjust Spines (Dickinson style, thanks to Andrew Straw)
###################################################################################################

# NOTE: smart_bounds is disabled (commented out) in this function. It only works in matplotlib v >1.
# to fix this issue, try manually setting your tick marks (see example below)
def adjust_spines(ax,spines, spine_locations={}, smart_bounds=True, xticks=None, yticks=None, linewidth=1):
    if type(spines) is not list:
        spines = [spines]

    # get ticks
    if xticks is None:
        xticks = ax.get_xticks()
    if yticks is None:
        yticks = ax.get_yticks()

    spine_locations_dict = {'top': 10, 'right': 10, 'left': 10, 'bottom': 10}
    for key in spine_locations.keys():
        spine_locations_dict[key] = spine_locations[key]

    if 'none' in spines:
        for loc, spine in ax.spines.iteritems():
            spine.set_color('none') # don't draw spine
        ax.yaxis.set_ticks([])
        ax.xaxis.set_ticks([])
        return

    for loc, spine in ax.spines.iteritems():
        if loc in spines:
            spine.set_position(('outward',spine_locations_dict[loc])) # outward by x points
            spine.set_linewidth(linewidth)
            spine.set_color('black')
        else:
            spine.set_color('none') # don't draw spine

    # smart bounds, if possible
    if int(matplotlib.__version__[0]) > 0 and smart_bounds:
        for loc, spine in ax.spines.items():
            if loc in ['left', 'right']:
                ticks = yticks
            if loc in ['top', 'bottom']:
                ticks = xticks
            spine.set_bounds(ticks[0], ticks[-1])

    # turn off ticks where there is no spine
    if 'left' in spines:
        ax.yaxis.set_ticks_position('left')
    elif 'right' in spines:
        ax.yaxis.set_ticks_position('right')
    else:
        # no yaxis ticks
        ax.yaxis.set_ticks([])

    if 'bottom' in spines:
        ax.xaxis.set_ticks_position('bottom')
    if 'top' in spines:
        ax.xaxis.set_ticks_position('top')
    else:
        # no xaxis ticks
        ax.xaxis.set_ticks([])

    if 'left' in spines or 'right' in spines:
        ax.set_yticks(yticks)
    if 'top' in spines or 'bottom' in spines:
        ax.set_xticks(xticks)

    for line in ax.get_xticklines() + ax.get_yticklines():
        #line.set_markersize(6)
        line.set_markeredgewidth(linewidth)
<commit_msg>import matplotlib<commit_after>import matplotlib.pyplot as plt
import matplotlib

###################################################################################################
# Adjust Spines (Dickinson style, thanks to Andrew Straw)
###################################################################################################

# NOTE: smart_bounds is disabled (commented out) in this function. It only works in matplotlib v >1.
# to fix this issue, try manually setting your tick marks (see example below)
def adjust_spines(ax,spines, spine_locations={}, smart_bounds=True, xticks=None, yticks=None, linewidth=1):
    if type(spines) is not list:
        spines = [spines]

    # get ticks
    if xticks is None:
        xticks = ax.get_xticks()
    if yticks is None:
        yticks = ax.get_yticks()

    spine_locations_dict = {'top': 10, 'right': 10, 'left': 10, 'bottom': 10}
    for key in spine_locations.keys():
        spine_locations_dict[key] = spine_locations[key]

    if 'none' in spines:
        for loc, spine in ax.spines.iteritems():
            spine.set_color('none') # don't draw spine
        ax.yaxis.set_ticks([])
        ax.xaxis.set_ticks([])
        return

    for loc, spine in ax.spines.iteritems():
        if loc in spines:
            spine.set_position(('outward',spine_locations_dict[loc])) # outward by x points
            spine.set_linewidth(linewidth)
            spine.set_color('black')
        else:
            spine.set_color('none') # don't draw spine

    # smart bounds, if possible
    if int(matplotlib.__version__[0]) > 0 and smart_bounds:
        for loc, spine in ax.spines.items():
            if loc in ['left', 'right']:
                ticks = yticks
            if loc in ['top', 'bottom']:
                ticks = xticks
            spine.set_bounds(ticks[0], ticks[-1])

    # turn off ticks where there is no spine
    if 'left' in spines:
        ax.yaxis.set_ticks_position('left')
    elif 'right' in spines:
        ax.yaxis.set_ticks_position('right')
    else:
        # no yaxis ticks
        ax.yaxis.set_ticks([])

    if 'bottom' in spines:
        ax.xaxis.set_ticks_position('bottom')
    if 'top' in spines:
        ax.xaxis.set_ticks_position('top')
    else:
        # no xaxis ticks
        ax.xaxis.set_ticks([])

    if 'left' in spines or 'right' in spines:
        ax.set_yticks(yticks)
    if 'top' in spines or 'bottom' in spines:
        ax.set_xticks(xticks)

    for line in ax.get_xticklines() + ax.get_yticklines():
        #line.set_markersize(6)
        line.set_markeredgewidth(linewidth)
<|endoftext|>


<commit_before>'''
DCS Filter
Version 2.0
By Brendan Kohrn and Scott Kennedy(1)
(1) Department of Pathology, University of Washington School of Medicine, Seattle, WA 98195
Based on work by Scott Kennedy, Mike Schmitt and Joe Hiatt
October 23, 2013

Written for Python 2.7.3
Required modules: Pysam, Samtools, BioPython

Inputs:
    A position-sorted paired-end BAM file containing SSCSs

Outputs:
    1: A paired-end BAM file containing DCSs
    2: A single-end BAM file containing unpaired DCSs
    3: A pair of fastq files containing DCSs for use in realligning.

    Note: Quality scores and cigar strings in these files are meaningless.

This program goes through the input file by position, making DCSs as it goes and writing them to file.  At the end of the run, any unpaired DCSs are written to a file ending in _UP.bam.

usage: DuplexMaker2.2.py [-h] [--infile INFILE] [--outfile OUTFILE]
                         [--Ncutoff NCUTOFF] [--readlength READ_LENGTH]

arguments:
  -h, --help            show this help message and exit
  --infile INFILE       input BAM file
  --outfile OUTFILE     output BAM file
  --Ncutoff NCUTOFF     Maximum percentage of Ns allowed in a consensus [1]
  --readlength READ_LENGTH
                        Length of the input read that is being used.  [80]

'''

import sys
import pysam
import re
import jellyfish
from Bio.Seq import Seq
from Bio.Alphabet import IUPAC
from collections import defaultdict
from argparse import ArgumentParser

def printRead(readIn):
    sys.stderr.write("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" % (readIn.qname, readIn.flag, readIn.tid, readIn.pos, readIn.mapq, readIn.cigar, readIn.mrnm, readIn.mpos, readIn.isize, readIn.seq, readIn.qual, readIn.tags))

def DSCMaker (groupedReadsList,  readLength) :
    #The consensus maker uses a simple "majority rules" algorithm to qmake a consensus at each base position.  If no nucleotide majority reaches above the minimum theshold (--cutoff), the position is considered undefined and an 'N' is placed at that position in the read.
    consensusRead = ''
    for i in xrange(readLength) :#rebuild consensus read taking into account the cutoff percentage
        if groupedReadsList[0][i]==groupedReadsList[1][i]:
            consensusRead += groupedReadsList[0][i]
        else:
            consensusRead += "N"

    return consensusRead

def main():
    ##########################################################################################################################
    #Parameters to be input.                                                 #
    ##########################################################################################################################
    parser=ArgumentParser()
    parser.add_argument("--infile", action="store", dest="infile", help="input BAM file", default='sys.stdin')
    parser.add_argument("--outfile",  action="store", dest="outfile", help="output BAM file",  default='sys.stdout')
    parser.add_argument('--Ncutoff', type=float, default=1.0, dest='Ncutoff', help="Maximum percentage of Ns allowed in a consensus [1.0]")
    #parser.add_argument('-p', action='store_true', dest='pipe', help="Output consensus reads to stdout"  )
    parser.add_argument('--readlength', type=int, default=80, dest='read_length', help="Length of the input read that is being used. [80]")
    parser.add_argument('--read_out', type = int, default = 1000000, dest = 'rOut', help = 'How often you want to be told what the program is doing. [1000000]')
    parser.add_argument('--hammingfilt', '-f', action = 'store_true', dest = 'hammingfilt', help = 'Optional: Filter reads based on hamming distance for derived families.  ')
    o = parser.parse_args()

    ##########################################################################################################################
    #DCS Maker module.                                          f            #
    ##########################################################################################################################

    ##########################################################################################################################
    #Initialization of all global variables, main input/output files, and main iterator and dictionaries.            #
    ##########################################################################################################################

    inBam = pysam.Samfile(o.infile, "rb") #open the input BAM file
    outBam = pysam.Samfile(o.outfile, "wb", template = inBam) #open the output BAM file
    if o.hammingfilt:
        hammingBam = pysam.Samfile(o.outfile.replace('.bam', '_DT.bam'), 'wb', template = inBam)
    fastqFile1 = open(o.outfile.replace('.bam','')+".r1.fq",'w')
    fastqFile2 = open(o.outfile.replace('.bam','')+".r2.fq",'w')
    #outStd = pysam.Samfile('-', 'wb', template = inBam ) #open the stdOut writer

    #if o.pipe==False:
    #   outStd.close()
    readNum = 0
    duplexMade = 0
    uP = 0
    nC = 0

    fileDone=False #initialize end of file bool
    finished=False
    readOne=True

    bamEntry = inBam.fetch( until_eof = True ) #initialize the iterator
    firstRead = bamEntry.next() #get the first read
    readDict = {} #initialize the read dictionary
    firstTag=firstRead.qname.split(":")[0]
    qualScore = firstRead.qual #set a dummy quality score

    ##########################################################################################################################
    #Find the first good read to serve as a start point for analysis.                                #
    ##########################################################################################################################

    consensusDict={}

    cigDum = firstRead.cigar #set a dummy cigar score

    ##########################################################################################################################
    #Start going through the input BAM file, one position at a time.                                 #
    ##########################################################################################################################

    for line in bamEntry:
        #reinitialize first line
        readNum += 1
        if readOne==True:
            readDict[firstTag] = [firstRead.flag, firstRead.rname, firstRead.pos, firstRead.mrnm, firstRead.mpos, firstRead.isize, firstRead.seq]
            readOne=False

        while line.pos == firstRead.pos and fileDone==False:
            tag = line.qname.split(":")[0] #extract the barcode
            #add the sequence to the read dictionary

            readDict[tag] = [line.flag, line.rname, line.pos, line.mrnm, line.mpos, line.isize, line.seq]
            #if fileDone==False:
            try: #keep StopIteration error from happening
                line = bamEntry.next() #iterate the line
                readNum += 1
            except:
                fileDone = True #tell the program that it has reached the end of the fil
                readNum += 1

            if readNum % o.rOut == 0:
                sys.stderr.write("%s reads processed\n" % (readNum))
        else:

    ##########################################################################################################################
    #Send reads to DCSMaker                                                #
    ##########################################################################################################################
            if o.hammingfilt:
                myDist = {}
                myKeys=[]
                for elmt in readDict.keys():
                    myDist[elmt] = 0
                    myKeys.append(elmt)
                for elmt in xrange(len(myKeys)):
                    for elmt2 in xrange(len(myKeys) - 1):
                        myHD = jellyfish.hamming_distance(myKeys[elmt], myKeys[elmt2 + 1])
                        if myHD != 0 and myHD <= 2:
                            myDist[myKeys[elmt]] = 1
                            myDist[myKeys[elmt2 + 1]] = 1

            firstRead = line #store the present line for the next group of lines
            firstTag = firstRead.qname
            readOne=True
            dictKeys = readDict.keys()

            for dictTag in readDict.keys(): #extract sequences to send to the DCSmaker
                switchtag = dictTag[12:]+dictTag[:12]

                try:
                    consensus = DSCMaker( [readDict[dictTag][6], readDict[switchtag][6]],  o.read_length )
                    duplexMade += 1
                    #Filter out consensuses with too many Ns in them
                    if consensus.count("N")/ len(consensus) > o.Ncutoff:
                        nC += 1
                    else:
                        #write a line to the consensusDictionary
                        a = pysam.AlignedRead()
                        a.qname = dictTag
                        a.flag = readDict[dictTag][0]

                        if a.is_reverse == True:
                            tmpSeq=Seq(consensus,IUPAC.unambiguous_dna)
                            a.seq=str(tmpSeq.reverse_complement())
                        else:
                            a.seq = consensus

                        a.rname = readDict[dictTag][1]
                        a.pos = readDict[dictTag][2]
                        a.mapq = 255
                        a.cigar = cigDum
                        a.mrnm = readDict[dictTag][3]
                        a.mpos=readDict[dictTag][4]
                        a.isize = readDict[dictTag][5]
                        a.qual = qualScore

                        if o.hammingfilt:
                            if myDist[dictTag] != 0:
                                hammingBam.write(a)
                        else:
            ##########################################################################################################################
            #Write SSCSs to output BAM file in read pairs.                                           #
            ##########################################################################################################################
                            if dictTag in consensusDict:

                                if a.is_read1 == True:
                                    #if o.pipe==True:
                                    #   outStd.write(a)
                                    #   outStd.write(consensusDict[switchtag])UG
                                    fastqFile1.write('@:%s\n%s\n+\n%s\n' %(a.qname, a.seq, a.qual))
                                    outBam.write(a)
                                    fastqFile2.write('@:%s\n%s\n+\n%s\n' %(consensusDict[dictTag].qname, consensusDict[dictTag].seq, consensusDict[dictTag].qual))
                                    outBam.write(consensusDict.pop(dictTag))
                                else:
                                    #if o.pipe==True:
                                    #        outStd.write(consensusDict[switchtag])
                                    #        outStd.write(a)
                                    fastqFile1.write('@:%s\n%s\n+\n%s\n' %(consensusDict[dictTag].qname, consensusDict[dictTag].seq, consensusDict[dictTag].qual))
                                    outBam.write(consensusDict.pop(dictTag))
                                    fastqFile2.write('@:%s\n%s\n+\n%s\n' %(a.qname, a.seq, a.qual))
                                    outBam.write(a)
                            else:
                                consensusDict[dictTag]=a

                    del readDict[dictTag]
                    del readDict[switchtag]

                except:
                    pass

        readDict={} #reset the read dictionary

    ##########################################################################################################################
    #Write unpaired SSCSs to extraConsensus.bam                                          #
    ##########################################################################################################################

    extraBam=pysam.Samfile(o.outfile.replace(".bam","_UP.bam"), "wb", template = inBam)
    #close BAM files
    inBam.close()

    for consTag in consensusDict.keys():
        a = pysam.AlignedRead()
        a.qname = consTag
        a.flag = 0
        a.seq = '.' * o.read_length
        a.rname = consensusDict[consTag].rname
        a.pos = consensusDict[consTag].pos
        a.mapq = 255
        a.cigar = cigDum
        a.mrnm = consensusDict[consTag].mrnm
        a.mpos=consensusDict[consTag].pos
        a.isize = consensusDict[consTag].isize
        a.qual = qualScore
        if consensusDict[consTag].is_read1 == False:
            fastqFile1.write('@:%s\n%s\n+\n%s\n' %(a.qname, a.seq, a.qual))
            outBam.write(a)
            fastqFile2.write('@:%s\n%s\n+\n%s\n' %(consensusDict[consTag].qname, consensusDict[consTag].seq, consensusDict[consTag].qual))
            outBam.write(consensusDict.pop(consTag))
        else:
            fastqFile1.write('@:%s\n%s\n+\n%s\n' %(consensusDict[consTag].qname, consensusDict[consTag].seq, consensusDict[consTag].qual))
            outBam.write(consensusDict.pop(consTag))
            fastqFile2.write('@:%s\n%s\n+\n%s\n' %(a.qname, a.seq, a.qual))
            outBam.write(a)
        uP += 1
    extraBam.close()
    fastqFile1.close()
    fastqFile2.close()
    outBam.close()
    #outStd.close()

    sys.stderr.write("Summary Statistics: \n")
    sys.stderr.write("Reads Processed: %s\n" % readNum)
    sys.stderr.write("Duplexes Made: %s\n" % duplexMade)
    sys.stderr.write("Unpaired Duplexes: %s\n" % uP)
    sys.stderr.write("N-clipped Duplexes: %s\n\n" % nC)

if __name__ == "__main__":
    main()
<commit_msg>fix derivitive family removal and add counter<commit_after>'''
DCS Filter
Version 2.0
By Brendan Kohrn and Scott Kennedy(1)
(1) Department of Pathology, University of Washington School of Medicine, Seattle, WA 98195
Based on work by Scott Kennedy, Mike Schmitt and Joe Hiatt
October 23, 2013

Written for Python 2.7.3
Required modules: Pysam, Samtools, BioPython

Inputs:
    A position-sorted paired-end BAM file containing SSCSs

Outputs:
    1: A paired-end BAM file containing DCSs
    2: A single-end BAM file containing unpaired DCSs
    3: A pair of fastq files containing DCSs for use in realligning.

    Note: Quality scores and cigar strings in these files are meaningless.

This program goes through the input file by position, making DCSs as it goes and writing them to file.  At the end of the run, any unpaired DCSs are written to a file ending in _UP.bam.

usage: DuplexMaker2.2.py [-h] [--infile INFILE] [--outfile OUTFILE]
                         [--Ncutoff NCUTOFF] [--readlength READ_LENGTH]

arguments:
  -h, --help            show this help message and exit
  --infile INFILE       input BAM file
  --outfile OUTFILE     output BAM file
  --Ncutoff NCUTOFF     Maximum percentage of Ns allowed in a consensus [1]
  --readlength READ_LENGTH
                        Length of the input read that is being used.  [80]

'''

import sys
import pysam
import re
import jellyfish
from Bio.Seq import Seq
from Bio.Alphabet import IUPAC
from collections import defaultdict
from argparse import ArgumentParser

def printRead(readIn):
    sys.stderr.write("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" % (readIn.qname, readIn.flag, readIn.tid, readIn.pos, readIn.mapq, readIn.cigar, readIn.mrnm, readIn.mpos, readIn.isize, readIn.seq, readIn.qual, readIn.tags))

def DSCMaker (groupedReadsList,  readLength) :
    #The consensus maker uses a simple "majority rules" algorithm to qmake a consensus at each base position.  If no nucleotide majority reaches above the minimum theshold (--cutoff), the position is considered undefined and an 'N' is placed at that position in the read.
    consensusRead = ''
    for i in xrange(readLength) :#rebuild consensus read taking into account the cutoff percentage
        if groupedReadsList[0][i]==groupedReadsList[1][i]:
            consensusRead += groupedReadsList[0][i]
        else:
            consensusRead += "N"

    return consensusRead

def main():
    ##########################################################################################################################
    #Parameters to be input.                                                 #
    ##########################################################################################################################
    parser=ArgumentParser()
    parser.add_argument("--infile", action="store", dest="infile", help="input BAM file", default='sys.stdin')
    parser.add_argument("--outfile",  action="store", dest="outfile", help="output BAM file",  default='sys.stdout')
    parser.add_argument('--Ncutoff', type=float, default=1.0, dest='Ncutoff', help="Maximum percentage of Ns allowed in a consensus [1.0]")
    #parser.add_argument('-p', action='store_true', dest='pipe', help="Output consensus reads to stdout"  )
    parser.add_argument('--readlength', type=int, default=80, dest='read_length', help="Length of the input read that is being used. [80]")
    parser.add_argument('--read_out', type = int, default = 1000000, dest = 'rOut', help = 'How often you want to be told what the program is doing. [1000000]')
    parser.add_argument('--hammingfilt', '-f', action = 'store_true', dest = 'hammingfilt', help = 'Optional: Filter reads based on hamming distance for derived families.  ')
    o = parser.parse_args()

    ##########################################################################################################################
    #DCS Maker module.                                          f            #
    ##########################################################################################################################

    ##########################################################################################################################
    #Initialization of all global variables, main input/output files, and main iterator and dictionaries.            #
    ##########################################################################################################################

    inBam = pysam.Samfile(o.infile, "rb") #open the input BAM file
    outBam = pysam.Samfile(o.outfile, "wb", template = inBam) #open the output BAM file
    if o.hammingfilt:
        hammingBam = pysam.Samfile(o.outfile.replace('.bam', '_DT.bam'), 'wb', template = inBam)
    fastqFile1 = open(o.outfile.replace('.bam','')+".r1.fq",'w')
    fastqFile2 = open(o.outfile.replace('.bam','')+".r2.fq",'w')
    #outStd = pysam.Samfile('-', 'wb', template = inBam ) #open the stdOut writer

    #if o.pipe==False:
    #   outStd.close()
    readNum = 0
    duplexMade = 0
    uP = 0
    nC = 0
    dT = 0

    fileDone=False #initialize end of file bool
    finished=False
    readOne=True

    bamEntry = inBam.fetch( until_eof = True ) #initialize the iterator
    firstRead = bamEntry.next() #get the first read
    readDict = {} #initialize the read dictionary
    firstTag=firstRead.qname.split(":")[0]
    qualScore = firstRead.qual #set a dummy quality score

    ##########################################################################################################################
    #Find the first good read to serve as a start point for analysis.                                #
    ##########################################################################################################################

    consensusDict={}

    cigDum = firstRead.cigar #set a dummy cigar score

    ##########################################################################################################################
    #Start going through the input BAM file, one position at a time.                                 #
    ##########################################################################################################################

    for line in bamEntry:
        #reinitialize first line
        readNum += 1
        if readOne==True:
            readDict[firstTag] = [firstRead.flag, firstRead.rname, firstRead.pos, firstRead.mrnm, firstRead.mpos, firstRead.isize, firstRead.seq]
            readOne=False

        while line.pos == firstRead.pos and fileDone==False:
            tag = line.qname.split(":")[0] #extract the barcode
            #add the sequence to the read dictionary

            readDict[tag] = [line.flag, line.rname, line.pos, line.mrnm, line.mpos, line.isize, line.seq]
            #if fileDone==False:
            try: #keep StopIteration error from happening
                line = bamEntry.next() #iterate the line
                readNum += 1
            except:
                fileDone = True #tell the program that it has reached the end of the fil
                readNum += 1

            if readNum % o.rOut == 0:
                sys.stderr.write("%s reads processed\n" % (readNum))
        else:

    ##########################################################################################################################
    #Send reads to DCSMaker                                                #
    ##########################################################################################################################
            if o.hammingfilt:
                myDist = {}
                myKeys=[]
                for elmt in readDict.keys():
                    myDist[elmt] = 0
                    myKeys.append(elmt)
                for elmt in xrange(len(myKeys)):
                    for elmt2 in xrange(len(myKeys) - 1):
                        myHD = jellyfish.hamming_distance(myKeys[elmt], myKeys[elmt2 + 1])
                        if myHD != 0 and myHD <= 2:
                            myDist[myKeys[elmt]] = 1
                            myDist[myKeys[elmt2 + 1]] = 1

            firstRead = line #store the present line for the next group of lines
            firstTag = firstRead.qname
            readOne=True
            dictKeys = readDict.keys()

            for dictTag in readDict.keys(): #extract sequences to send to the DCSmaker
                switchtag = dictTag[12:]+dictTag[:12]

                try:
                    consensus = DSCMaker( [readDict[dictTag][6], readDict[switchtag][6]],  o.read_length )
                    duplexMade += 1
                    #Filter out consensuses with too many Ns in them
                    if consensus.count("N")/ len(consensus) > o.Ncutoff:
                        nC += 1
                    else:
                        #write a line to the consensusDictionary
                        a = pysam.AlignedRead()
                        a.qname = dictTag
                        a.flag = readDict[dictTag][0]

                        if a.is_reverse == True:
                            tmpSeq=Seq(consensus,IUPAC.unambiguous_dna)
                            a.seq=str(tmpSeq.reverse_complement())
                        else:
                            a.seq = consensus

                        a.rname = readDict[dictTag][1]
                        a.pos = readDict[dictTag][2]
                        a.mapq = 255
                        a.cigar = cigDum
                        a.mrnm = readDict[dictTag][3]
                        a.mpos=readDict[dictTag][4]
                        a.isize = readDict[dictTag][5]
                        a.qual = qualScore

                        if o.hammingfilt and myDist[dictTag] != 0:
                                hammingBam.write(a)
                                dT += 1
                        else:
            ##########################################################################################################################
            #Write SSCSs to output BAM file in read pairs.                                           #
            ##########################################################################################################################
                            if dictTag in consensusDict:

                                if a.is_read1 == True:
                                    #if o.pipe==True:
                                    #   outStd.write(a)
                                    #   outStd.write(consensusDict[switchtag])UG
                                    fastqFile1.write('@:%s\n%s\n+\n%s\n' %(a.qname, a.seq, a.qual))
                                    outBam.write(a)
                                    fastqFile2.write('@:%s\n%s\n+\n%s\n' %(consensusDict[dictTag].qname, consensusDict[dictTag].seq, consensusDict[dictTag].qual))
                                    outBam.write(consensusDict.pop(dictTag))
                                else:
                                    #if o.pipe==True:
                                    #        outStd.write(consensusDict[switchtag])
                                    #        outStd.write(a)
                                    fastqFile1.write('@:%s\n%s\n+\n%s\n' %(consensusDict[dictTag].qname, consensusDict[dictTag].seq, consensusDict[dictTag].qual))
                                    outBam.write(consensusDict.pop(dictTag))
                                    fastqFile2.write('@:%s\n%s\n+\n%s\n' %(a.qname, a.seq, a.qual))
                                    outBam.write(a)
                            else:
                                consensusDict[dictTag]=a

                    del readDict[dictTag]
                    del readDict[switchtag]

                except:
                    pass

        readDict={} #reset the read dictionary

    ##########################################################################################################################
    #Write unpaired SSCSs to extraConsensus.bam                                          #
    ##########################################################################################################################

    extraBam=pysam.Samfile(o.outfile.replace(".bam","_UP.bam"), "wb", template = inBam)
    #close BAM files
    inBam.close()

    for consTag in consensusDict.keys():
        a = pysam.AlignedRead()
        a.qname = consTag
        a.flag = 0
        a.seq = '.' * o.read_length
        a.rname = consensusDict[consTag].rname
        a.pos = consensusDict[consTag].pos
        a.mapq = 255
        a.cigar = cigDum
        a.mrnm = consensusDict[consTag].mrnm
        a.mpos=consensusDict[consTag].pos
        a.isize = consensusDict[consTag].isize
        a.qual = qualScore
        if consensusDict[consTag].is_read1 == False:
            fastqFile1.write('@:%s\n%s\n+\n%s\n' %(a.qname, a.seq, a.qual))
            outBam.write(a)
            fastqFile2.write('@:%s\n%s\n+\n%s\n' %(consensusDict[consTag].qname, consensusDict[consTag].seq, consensusDict[consTag].qual))
            outBam.write(consensusDict.pop(consTag))
        else:
            fastqFile1.write('@:%s\n%s\n+\n%s\n' %(consensusDict[consTag].qname, consensusDict[consTag].seq, consensusDict[consTag].qual))
            outBam.write(consensusDict.pop(consTag))
            fastqFile2.write('@:%s\n%s\n+\n%s\n' %(a.qname, a.seq, a.qual))
            outBam.write(a)
        uP += 1
    extraBam.close()
    fastqFile1.close()
    fastqFile2.close()
    outBam.close()
    #outStd.close()

    sys.stderr.write("Summary Statistics: \n")
    sys.stderr.write("Reads Processed: %s\n" % readNum)
    sys.stderr.write("Duplexes Made: %s\n" % duplexMade)
    sys.stderr.write("Unpaired Duplexes: %s\n" % uP)
    sys.stderr.write("N-clipped Duplexes: %s\n" % nC)
    sys.stderr.write('Derivitive Families Removed: %s\n\n' % dT)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: UTF-8 -*-

from flask import jsonify
from flask import current_app

from influxdb import InfluxDBClient

from app.base import ApiResource

class InfluxResource(ApiResource):
    endpoint = 'influx'
    url_prefix = '/influx'
    url_rules = {
        'index': {
            'rule': '/',
            'defaults': { 'query': None  }
        },
        'show': {
            'rule': '/<query>'
        },
    }

    def get(self, query):
        client = InfluxDBClient.from_DSN(current_app.config['INFLUX_DATABASE_URI'], timeout=5)
        result = client.query(query)

        fmtResults = []
        if result:
            for series in result.raw['series']:
                fmtSeries= {}
                fmtSeries['name'] = series['name']
                if 'tags' in series:
                    fmtSeries['tags'] = series['tags']

                columns = series['columns']
                values = series['values']

                fmtSeries['data'] = [ dict(zip(columns, v)) for v in values ]

                fmtResults.append(fmtSeries)

            return jsonify(result=fmtResults)
        else:
            return jsonify(result=[])

<commit_msg>completed influxdb endpoint<commit_after>#!/usr/bin/env python
# -*- coding: UTF-8 -*-

from flask import jsonify
from flask import current_app

from influxdb import InfluxDBClient

from app.base import ApiResource

class InfluxResource(ApiResource):
    endpoint = 'influx'
    url_prefix = '/influx'
    url_rules = {
        'index': {
            'rule': '/',
        }
    }

    def get(self):
        config = current_app.config['USER_CONFIG'].get('influxdb', {})
        #TODO: define default colors in js
        default_colors = [ "#62c462", "#f89406", "#ee5f5b", "#5bc0de" ]
        client = InfluxDBClient.from_DSN(config['uri'], timeout=5)
        results = []

        for metric in config.get('metrics', []):
            for query in metric.get('queries', []):
                result = client.query(query, epoch='ms')

                collection = []
                if result:
                    for index, dataset in enumerate(result.raw['series']):
                        series= {}
                        series['data'] = dataset['values']
                        series['label'] = metric['labels'][index] if 'labels' in metric else None
                        series['lines'] = dict(fill=True)
                        series['mode'] = metric['mode'] if 'mode' in metric else None
                        series['color'] = metric['colors'][index] if 'colors' in metric else default_colors[index]

                        collection.append(series)

            results.append(collection)

        return jsonify(result=results)
<|endoftext|>


<commit_before>"""
extractor.base
==============

Base & default implmentation class of extractor.

"""
import io
import inspect
import collections

from flask import Flask
from werkzeug.routing import parse_rule, parse_converter_args

from ..core import PathItem, Operation, Parameter, Response
from ..utils import get_type_base, TYPE_MAP, parse_endpoint, \
    merge, normalize_indent

_MISSING = object()

CONVERTER_TYPES = {
    'float': float,
    'path': str,
    'any': str,
    'default': str,
    'uuid': str,
    'int': int,
    'string': str,
}

WerkzeugConverter = collections.namedtuple(
    'WerkzeugConverter', ['converter', 'args', 'kwargs'])
PathAndParams = collections.namedtuple('PathAndParams', ['path', 'params'])
PathAndPathItem = collections.namedtuple('PathAndPathItem', ['path', 'item'])

class Extractor(object):
    """
    Base class that extract swagger spec from flask application.

    You can extract path items from app by using :meth:`extract_paths`
    and customize converting method by overriding them.

    """
    def convert_werkzeug_converter(self, name: str,
                                   converter: WerkzeugConverter, ctx: dict):
        """Convert werkzeug converter to swagger parameter object."""
        python_type = CONVERTER_TYPES.get(converter.converter, None)
        type_base = get_type_base(python_type)
        if type_base is None:
            return None
        return Parameter(name=name, in_="path", required=True, **type_base)

    def convert_annotation(self, name, annotation, ctx: dict):
        """Convert function annotation to swagger parameter object."""
        if annotation is None:
            return None
        if not isinstance(annotation, type):
            return None
        type_base = None
        for available_type in TYPE_MAP:
            if issubclass(annotation, available_type):
                type_base = get_type_base(available_type)
                break
        if type_base is None:
            return None
        return Parameter(name=name, in_='path', **type_base)

    def parse_werkzeug_rule(self, rule: str, ctx: dict) -> PathAndParams:
        """
        Convert werkzeug rule to swagger path format and
        extract parameter info.
        """
        params = {}
        with io.StringIO() as buf:
            for converter, arguments, variable in parse_rule(rule):
                if converter:
                    if arguments is not None:
                        args, kwargs = parse_converter_args(arguments)
                    else:
                        args = ()
                        kwargs = {}
                    params[variable] = WerkzeugConverter(
                        converter=converter,
                        args=args,
                        kwargs=kwargs,
                    )
                    buf.write('{')
                    buf.write(variable)
                    buf.write('}')
                else:
                    buf.write(variable)
            return PathAndParams(buf.getvalue(), params)

    def extract_description(self, view, ctx: dict) -> str:
        """Extract description info from view function."""
        doc = getattr(view, '__doc__', None) or None
        if not doc:
            return None
        return normalize_indent(doc)

    def extract_summary(self, view, ctx) -> str:
        """Extract brief description from view function."""
        description = self.extract_description(view, ctx)
        if not description:
            return None
        return description.strip().split('\n', 1)[0][:120].strip()

    def extract_param(self, view, name, ctx: dict):
        """Extract path parameters info from view function."""
        signature = inspect.signature(view)
        if name not in signature.parameters:
            return None
        parameter = signature.parameters.get(name)
        annotation = parameter.annotation
        return self.convert_annotation(name, annotation, ctx)

    def default_path_param(self, name, ctx: dict):
        return Parameter(
            name=name,
            in_='path',
            **get_type_base(str)
        )

    def extract_responses(self, view, ctx: dict):
        return {
            'default': Response(
                description=''
            ),
        }

    def build_parameters(self, view, param_info, ctx: dict) -> list:
        """
        Build parameters from path params and view params.
        path params have higher order.
        """
        parameters = []
        for name, converter in param_info.items():
            parameter = self.convert_werkzeug_converter(name, converter, ctx)
            if parameter is None:
                parameter = self.extract_param(view, name, ctx)
            if parameter is None:
                parameter = self.default_path_param(name, ctx)
            parameters.append(parameter)
        return parameters

    def extract_others(self, view, ctx: dict):
        """Extract other fields from view & context."""
        return {}

    def make_operation(self, view, params: dict, ctx: dict):
        """Convert view to swagger opration object."""
        description = self.extract_description(view, ctx)
        summary = self.extract_summary(view, ctx)
        parameters = self.build_parameters(view, params, ctx)
        responses = self.extract_responses(view, ctx)
        others = self.extract_others(view, merge(ctx, {
            'params': params,
        }))
        kwargs = dict(
            description=description,
            summary=summary,
            parameters=parameters,
            responses=responses,
        )
        kwargs.update(others)

        return Operation(**kwargs)

    def make_path_item(self, app: Flask, rule: str, endpoints: dict,
                       ctx: dict) -> PathAndPathItem:
        """Make path item from rule and endpoints collected by HTTP methods."""
        path, params = self.parse_werkzeug_rule(rule, ctx)
        operations = {}
        for method, endpoint in endpoints.items():
            view = app.view_functions[endpoint]
            operations[method.lower()] = self.make_operation(
                view, params, merge(ctx, {
                    'endpoint': endpoint,
                    'path': path,
                    'method': method,
                }))
        return PathAndPathItem(
            path=path,
            item=PathItem(**operations),
        )

    def collect_endpoints(self, app: Flask, blueprint=_MISSING, endpoint=None,
                          exclude_blueprint=_MISSING, exclude_endpoint=None) \
            -> dict:
        """Collect endpoints in rules.

        :param blueprint: name of blueprints to be collected. :const:`None`
                          means non-blueprint endpoints. It cat either be list
                          or string.
        :param endpoint: endpoints to be collected. It cat either be list or
                         string.

        :param exclude_blueprint: blueprints not to be collected.

        :param exclude_endpoint: endpoint not to be collected.

        """
        if blueprint is not _MISSING:
            if blueprint is None or isinstance(blueprint, str):
                blueprint = (blueprint,)
        if isinstance(endpoint, str):
            endpoint = (endpoint,)

        if exclude_blueprint is not _MISSING:
            if exclude_blueprint is None or isinstance(exclude_blueprint, str):
                exclude_blueprint = (exclude_blueprint,)
        if isinstance(exclude_endpoint, str):
            exclude_endpoint = (exclude_endpoint,)

        endpoints = {}
        for rule in app.url_map.iter_rules():
            if blueprint is not _MISSING:
                rule_blueprint, rule_endpoint = parse_endpoint(rule.endpoint)
                if rule_blueprint not in blueprint:
                    continue
                if endpoint and rule_endpoint not in endpoint:
                    continue
            elif endpoint and rule.endpoint not in endpoint:
                continue
            if exclude_blueprint is not _MISSING:
                rule_blueprint, rule_endpoint = parse_endpoint(rule.endpoint)
                if rule_blueprint in exclude_blueprint:
                    continue
                if exclude_endpoint and rule_endpoint in exclude_endpoint:
                    continue
            elif exclude_endpoint and rule.endpoint in exclude_endpoint:
                continue
            methods = rule.methods.difference({'HEAD', 'OPTIONS'})
            method_collection = endpoints.setdefault(rule.rule, {})
            for method in methods:
                method_collection[method] = rule.endpoint
        return endpoints

    def extract_paths(self, app: Flask, blueprint=_MISSING, endpoint=None,
                      exclude_blueprint=_MISSING, exclude_endpoint=None):
        """Extract path items from flask app.

        :param blueprint: name of blueprints to be collected. :const:`None`
                          means non-blueprint endpoints. It cat either be list
                          or string.
        :param endpoint: endpoints to be collected. It cat either be list or
                         string.

        :param exclude_blueprint: blueprints not to be collected.

        :param exclude_endpoint: endpoint not to be collected.

        """
        endpoints = self.collect_endpoints(app, blueprint, endpoint,
                                           exclude_blueprint, exclude_endpoint)

        paths = {}
        for rule, methods in endpoints.items():
            ctx = {
                'rule': rule,
                'methods': methods,
                'app': app,
            }
            path, path_item = self.make_path_item(app, rule, methods, ctx)
            paths[path] = path_item
        return paths
<commit_msg>Set required=True for default path param<commit_after>"""
extractor.base
==============

Base & default implmentation class of extractor.

"""
import io
import inspect
import collections

from flask import Flask
from werkzeug.routing import parse_rule, parse_converter_args

from ..core import PathItem, Operation, Parameter, Response
from ..utils import get_type_base, TYPE_MAP, parse_endpoint, \
    merge, normalize_indent

_MISSING = object()

CONVERTER_TYPES = {
    'float': float,
    'path': str,
    'any': str,
    'default': str,
    'uuid': str,
    'int': int,
    'string': str,
}

WerkzeugConverter = collections.namedtuple(
    'WerkzeugConverter', ['converter', 'args', 'kwargs'])
PathAndParams = collections.namedtuple('PathAndParams', ['path', 'params'])
PathAndPathItem = collections.namedtuple('PathAndPathItem', ['path', 'item'])

class Extractor(object):
    """
    Base class that extract swagger spec from flask application.

    You can extract path items from app by using :meth:`extract_paths`
    and customize converting method by overriding them.

    """
    def convert_werkzeug_converter(self, name: str,
                                   converter: WerkzeugConverter, ctx: dict):
        """Convert werkzeug converter to swagger parameter object."""
        python_type = CONVERTER_TYPES.get(converter.converter, None)
        type_base = get_type_base(python_type)
        if type_base is None:
            return None
        return Parameter(name=name, in_="path", required=True, **type_base)

    def convert_annotation(self, name, annotation, ctx: dict):
        """Convert function annotation to swagger parameter object."""
        if annotation is None:
            return None
        if not isinstance(annotation, type):
            return None
        type_base = None
        for available_type in TYPE_MAP:
            if issubclass(annotation, available_type):
                type_base = get_type_base(available_type)
                break
        if type_base is None:
            return None
        return Parameter(name=name, in_='path', **type_base)

    def parse_werkzeug_rule(self, rule: str, ctx: dict) -> PathAndParams:
        """
        Convert werkzeug rule to swagger path format and
        extract parameter info.
        """
        params = {}
        with io.StringIO() as buf:
            for converter, arguments, variable in parse_rule(rule):
                if converter:
                    if arguments is not None:
                        args, kwargs = parse_converter_args(arguments)
                    else:
                        args = ()
                        kwargs = {}
                    params[variable] = WerkzeugConverter(
                        converter=converter,
                        args=args,
                        kwargs=kwargs,
                    )
                    buf.write('{')
                    buf.write(variable)
                    buf.write('}')
                else:
                    buf.write(variable)
            return PathAndParams(buf.getvalue(), params)

    def extract_description(self, view, ctx: dict) -> str:
        """Extract description info from view function."""
        doc = getattr(view, '__doc__', None) or None
        if not doc:
            return None
        return normalize_indent(doc)

    def extract_summary(self, view, ctx) -> str:
        """Extract brief description from view function."""
        description = self.extract_description(view, ctx)
        if not description:
            return None
        return description.strip().split('\n', 1)[0][:120].strip()

    def extract_param(self, view, name, ctx: dict):
        """Extract path parameters info from view function."""
        signature = inspect.signature(view)
        if name not in signature.parameters:
            return None
        parameter = signature.parameters.get(name)
        annotation = parameter.annotation
        return self.convert_annotation(name, annotation, ctx)

    def default_path_param(self, name, ctx: dict):
        return Parameter(
            name=name,
            in_='path',
            required=True,
            **get_type_base(str)
        )

    def extract_responses(self, view, ctx: dict):
        return {
            'default': Response(
                description=''
            ),
        }

    def build_parameters(self, view, param_info, ctx: dict) -> list:
        """
        Build parameters from path params and view params.
        path params have higher order.
        """
        parameters = []
        for name, converter in param_info.items():
            parameter = self.convert_werkzeug_converter(name, converter, ctx)
            if parameter is None:
                parameter = self.extract_param(view, name, ctx)
            if parameter is None:
                parameter = self.default_path_param(name, ctx)
            parameters.append(parameter)
        return parameters

    def extract_others(self, view, ctx: dict):
        """Extract other fields from view & context."""
        return {}

    def make_operation(self, view, params: dict, ctx: dict):
        """Convert view to swagger opration object."""
        description = self.extract_description(view, ctx)
        summary = self.extract_summary(view, ctx)
        parameters = self.build_parameters(view, params, ctx)
        responses = self.extract_responses(view, ctx)
        others = self.extract_others(view, merge(ctx, {
            'params': params,
        }))
        kwargs = dict(
            description=description,
            summary=summary,
            parameters=parameters,
            responses=responses,
        )
        kwargs.update(others)

        return Operation(**kwargs)

    def make_path_item(self, app: Flask, rule: str, endpoints: dict,
                       ctx: dict) -> PathAndPathItem:
        """Make path item from rule and endpoints collected by HTTP methods."""
        path, params = self.parse_werkzeug_rule(rule, ctx)
        operations = {}
        for method, endpoint in endpoints.items():
            view = app.view_functions[endpoint]
            operations[method.lower()] = self.make_operation(
                view, params, merge(ctx, {
                    'endpoint': endpoint,
                    'path': path,
                    'method': method,
                }))
        return PathAndPathItem(
            path=path,
            item=PathItem(**operations),
        )

    def collect_endpoints(self, app: Flask, blueprint=_MISSING, endpoint=None,
                          exclude_blueprint=_MISSING, exclude_endpoint=None) \
            -> dict:
        """Collect endpoints in rules.

        :param blueprint: name of blueprints to be collected. :const:`None`
                          means non-blueprint endpoints. It cat either be list
                          or string.
        :param endpoint: endpoints to be collected. It cat either be list or
                         string.

        :param exclude_blueprint: blueprints not to be collected.

        :param exclude_endpoint: endpoint not to be collected.

        """
        if blueprint is not _MISSING:
            if blueprint is None or isinstance(blueprint, str):
                blueprint = (blueprint,)
        if isinstance(endpoint, str):
            endpoint = (endpoint,)

        if exclude_blueprint is not _MISSING:
            if exclude_blueprint is None or isinstance(exclude_blueprint, str):
                exclude_blueprint = (exclude_blueprint,)
        if isinstance(exclude_endpoint, str):
            exclude_endpoint = (exclude_endpoint,)

        endpoints = {}
        for rule in app.url_map.iter_rules():
            if blueprint is not _MISSING:
                rule_blueprint, rule_endpoint = parse_endpoint(rule.endpoint)
                if rule_blueprint not in blueprint:
                    continue
                if endpoint and rule_endpoint not in endpoint:
                    continue
            elif endpoint and rule.endpoint not in endpoint:
                continue
            if exclude_blueprint is not _MISSING:
                rule_blueprint, rule_endpoint = parse_endpoint(rule.endpoint)
                if rule_blueprint in exclude_blueprint:
                    continue
                if exclude_endpoint and rule_endpoint in exclude_endpoint:
                    continue
            elif exclude_endpoint and rule.endpoint in exclude_endpoint:
                continue
            methods = rule.methods.difference({'HEAD', 'OPTIONS'})
            method_collection = endpoints.setdefault(rule.rule, {})
            for method in methods:
                method_collection[method] = rule.endpoint
        return endpoints

    def extract_paths(self, app: Flask, blueprint=_MISSING, endpoint=None,
                      exclude_blueprint=_MISSING, exclude_endpoint=None):
        """Extract path items from flask app.

        :param blueprint: name of blueprints to be collected. :const:`None`
                          means non-blueprint endpoints. It cat either be list
                          or string.
        :param endpoint: endpoints to be collected. It cat either be list or
                         string.

        :param exclude_blueprint: blueprints not to be collected.

        :param exclude_endpoint: endpoint not to be collected.

        """
        endpoints = self.collect_endpoints(app, blueprint, endpoint,
                                           exclude_blueprint, exclude_endpoint)

        paths = {}
        for rule, methods in endpoints.items():
            ctx = {
                'rule': rule,
                'methods': methods,
                'app': app,
            }
            path, path_item = self.make_path_item(app, rule, methods, ctx)
            paths[path] = path_item
        return paths
<|endoftext|>


<commit_before>'''
   Copyright 2012 Konrad Jopek

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
'''

from apel.db.records import Record

class GroupAttributeRecord(Record):
    '''
    GroupAttributeRecord represents single attribute record
    associated with StarRecord.

    Single StarRecord can have multiple GroupAttributeRecords.
    '''

    DB_FIELDS = ["StarRecordID", "AttributeType", "AttributeValue"]
    MANDATORY_FIELDS = ["StarRecordID"]

    ALL_FIELDS = DB_FIELDS

    def __init__(self):
        '''
        Initializer for GroupAttributeRecord
        '''
        Record.__init__(self)

        self._db_fields = self.DB_FIELDS
        self._mandatory_fields = self.MANDATORY_FIELDS
        self._all_fields = self._db_fields

    def get_db_tuple(self, source=None):
        """
        Return record contents as tuple ignoring the 'source' keyword argument.

        The source (DN of the sender) isn't used in this record type currently.
        """
        return Record.get_db_tuple(self)
<commit_msg>Remove trailing whitespace from apel/db/records/group_attribute.py<commit_after>'''
   Copyright 2012 Konrad Jopek

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
'''

from apel.db.records import Record

class GroupAttributeRecord(Record):
    '''
    GroupAttributeRecord represents single attribute record
    associated with StarRecord.

    Single StarRecord can have multiple GroupAttributeRecords.
    '''

    DB_FIELDS = ["StarRecordID", "AttributeType", "AttributeValue"]
    MANDATORY_FIELDS = ["StarRecordID"]

    ALL_FIELDS = DB_FIELDS

    def __init__(self):
        '''
        Initializer for GroupAttributeRecord
        '''
        Record.__init__(self)

        self._db_fields = self.DB_FIELDS
        self._mandatory_fields = self.MANDATORY_FIELDS
        self._all_fields = self._db_fields

    def get_db_tuple(self, source=None):
        """
        Return record contents as tuple ignoring the 'source' keyword argument.

        The source (DN of the sender) isn't used in this record type currently.
        """
        return Record.get_db_tuple(self)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from collections.abc import Iterable
from wtforms.widgets import html_params, HTMLString

class FileDisplayWidget(object):
    html_params = staticmethod(html_params)

    def __init__(self, input_type='string', text=''):
        self.input_type = input_type
        self.text = text

    def __call__(self, field, **kwargs):
        kwargs.setdefault('id', field.id)
        html = [u'
' % html_params(id=field.id)]
        data = field.data
        if data:
            if not isinstance(data, Iterable):
                params = dict(href=data)
                html.append(u'
%s
' % (html_params(**params), data))
            else:
                for link in data:
                    if link:
                        params = dict(href=link)
                        html.append(u'
%s
' % (html_params(**params), link))
        html.append(u'
')
        return HTMLString(u''.join(html))
<commit_msg>[FIX] add missed close tag<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from collections.abc import Iterable
from wtforms.widgets import html_params, HTMLString

class FileDisplayWidget(object):

    def __init__(self, input_type='string', text=''):
        self.input_type = input_type
        self.text = text

    def __call__(self, field, **kwargs):
        kwargs.setdefault('id', field.id)
        html = [u'
' % html_params(id=field.id)]
        data = field.data
        if data:
            if not isinstance(data, Iterable):
                params = dict(href=data)
                html.append(u'
%s
' % (html_params(**params), data))
            else:
                for link in data:
                    if link:
                        params = dict(href=link)
                        html.append(u'
%s
' % (html_params(**params), link))
        html.append(u'
')
        return HTMLString(u''.join(html))
<|endoftext|>