<commit_before># Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Adam for TensorFlow."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensorflow.python.eager import context
from tensorflow.python.framework import ops
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import control_flow_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import resource_variable_ops
from tensorflow.python.ops import state_ops
from tensorflow.python.training import optimizer
from tensorflow.python.training import training_ops
from tensorflow.python.util.tf_export import tf_export

@tf_export("train.AdamOptimizer")
class AdamOptimizer(optimizer.Optimizer):
  """Optimizer that implements the Adam algorithm.

  See [Kingma et al., 2014](http://arxiv.org/abs/1412.6980)
  ([pdf](http://arxiv.org/pdf/1412.6980.pdf)).
  """

  def __init__(self, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-8,
               use_locking=False, name="Adam"):
    """Construct a new Adam optimizer.

    Initialization:

    $$m_0 := 0 \text{(Initialize initial 1st moment vector)}$$
    $$v_0 := 0 \text{(Initialize initial 2nd moment vector)}$$
    $$t := 0 \text{(Initialize timestep)}$$

    The update rule for `variable` with gradient `g` uses an optimization
    described at the end of section2 of the paper:

    $$t := t + 1$$
    $$lr_t := \text{learning\_rate} * \sqrt{1 - beta_2^t} / (1 - beta_1^t)$$

    $$m_t := beta_1 * m_{t-1} + (1 - beta_1) * g$$
    $$v_t := beta_2 * v_{t-1} + (1 - beta_2) * g * g$$
    $$variable := variable - lr_t * m_t / (\sqrt{v_t} + \epsilon)$$

    The default value of 1e-8 for epsilon might not be a good default in
    general. For example, when training an Inception network on ImageNet a
    current good choice is 1.0 or 0.1. Note that since AdamOptimizer uses the
    formulation just before Section 2.1 of the Kingma and Ba paper rather than
    the formulation in Algorithm 1, the "epsilon" referred to here is "epsilon
    hat" in the paper.

    The sparse implementation of this algorithm (used when the gradient is an
    IndexedSlices object, typically because of `tf.gather` or an embedding
    lookup in the forward pass) does apply momentum to variable slices even if
    they were not used in the forward pass (meaning they have a gradient equal
    to zero). Momentum decay (beta1) is also applied to the entire momentum
    accumulator. This means that the sparse behavior is equivalent to the dense
    behavior (in contrast to some momentum implementations which ignore momentum
    unless a variable slice was actually used).

    Args:
      learning_rate: A Tensor or a floating point value.  The learning rate.
      beta1: A float value or a constant float tensor.
        The exponential decay rate for the 1st moment estimates.
      beta2: A float value or a constant float tensor.
        The exponential decay rate for the 2nd moment estimates.
      epsilon: A small constant for numerical stability. This epsilon is
        "epsilon hat" in the Kingma and Ba paper (in the formula just before
        Section 2.1), not the epsilon in Algorithm 1 of the paper.
      use_locking: If True use locks for update operations.
      name: Optional name for the operations created when applying gradients.
        Defaults to "Adam".

    @compatibility(eager)
    When eager execution is enabled, `learning_rate`, `beta1`, `beta2`, and
    `epsilon` can each be a callable that takes no arguments and returns the
    actual value to use. This can be useful for changing these values across
    different invocations of optimizer functions.
    @end_compatibility
    """
    super(AdamOptimizer, self).__init__(use_locking, name)
    self._lr = learning_rate
    self._beta1 = beta1
    self._beta2 = beta2
    self._epsilon = epsilon

    # Tensor versions of the constructor arguments, created in _prepare().
    self._lr_t = None
    self._beta1_t = None
    self._beta2_t = None
    self._epsilon_t = None

    # Created in SparseApply if needed.
    self._updated_lr = None

  def _get_beta_accumulators(self):
    with ops.init_scope():
      if context.executing_eagerly():
        graph = None
      else:
        graph = ops.get_default_graph()
      return (self._get_non_slot_variable("beta1_power", graph=graph),
              self._get_non_slot_variable("beta2_power", graph=graph))

  def _create_slots(self, var_list):
    # Create the beta1 and beta2 accumulators on the same device as the first
    # variable. Sort the var_list to make sure this device is consistent across
    # workers (these need to go on the same PS, otherwise some updates are
    # silently ignored).
    first_var = min(var_list, key=lambda x: x.name)
    self._create_non_slot_variable(initial_value=self._beta1,
                                   name="beta1_power",
                                   colocate_with=first_var)
    self._create_non_slot_variable(initial_value=self._beta2,
                                   name="beta2_power",
                                   colocate_with=first_var)

    # Create slots for the first and second moments.
    for v in var_list:
      self._zeros_slot(v, "m", self._name)
      self._zeros_slot(v, "v", self._name)

  def _prepare(self):
    lr = self._call_if_callable(self._lr)
    beta1 = self._call_if_callable(self._beta1)
    beta2 = self._call_if_callable(self._beta2)
    epsilon = self._call_if_callable(self._epsilon)

    self._lr_t = ops.convert_to_tensor(lr, name="learning_rate")
    self._beta1_t = ops.convert_to_tensor(beta1, name="beta1")
    self._beta2_t = ops.convert_to_tensor(beta2, name="beta2")
    self._epsilon_t = ops.convert_to_tensor(epsilon, name="epsilon")

  def _apply_dense(self, grad, var):
    m = self.get_slot(var, "m")
    v = self.get_slot(var, "v")
    beta1_power, beta2_power = self._get_beta_accumulators()
    return training_ops.apply_adam(
        var, m, v,
        math_ops.cast(beta1_power, var.dtype.base_dtype),
        math_ops.cast(beta2_power, var.dtype.base_dtype),
        math_ops.cast(self._lr_t, var.dtype.base_dtype),
        math_ops.cast(self._beta1_t, var.dtype.base_dtype),
        math_ops.cast(self._beta2_t, var.dtype.base_dtype),
        math_ops.cast(self._epsilon_t, var.dtype.base_dtype),
        grad, use_locking=self._use_locking).op

  def _resource_apply_dense(self, grad, var):
    m = self.get_slot(var, "m")
    v = self.get_slot(var, "v")
    beta1_power, beta2_power = self._get_beta_accumulators()
    return training_ops.resource_apply_adam(
        var.handle, m.handle, v.handle,
        math_ops.cast(beta1_power, grad.dtype.base_dtype),
        math_ops.cast(beta2_power, grad.dtype.base_dtype),
        math_ops.cast(self._lr_t, grad.dtype.base_dtype),
        math_ops.cast(self._beta1_t, grad.dtype.base_dtype),
        math_ops.cast(self._beta2_t, grad.dtype.base_dtype),
        math_ops.cast(self._epsilon_t, grad.dtype.base_dtype),
        grad, use_locking=self._use_locking)

  class ScatterOpWrapper(object):
    """Wraps necessary scatter ops for sparse tensors."""

    def __init__(self, use_locking=False):
      self._use_locking = use_locking

    def add(self, sparse_tensor, index, delta):
      return state_ops.scatter_add(sparse_tensor, index, delta,
                                   use_locking = self._use_locking)

    def sub(self, sparse_tensor, index, delta):
      return state_ops.scatter_sub(sparse_tensor, index, delta,
                                   use_locking = self._use_locking)

    def update(self, sparse_tensor, index, value):
      return state_ops.scatter_update(sparse_tensor, index, value,
                                      use_locking = self._use_locking)

  class ResourceScatterOpWrapper(ScatterOpWrapper):
    """Wraps necessay scatter ops for sparse resource variables."""

    def __init__(self, use_locking=False):
      super(AdamOptimizer.ResourceScatterOpWrapper, self).__init__(use_locking)

    def add(self, sparse_tensor, index, delta):
      with ops.control_dependencies(
          [resource_variable_ops.resource_scatter_add(
              sparse_tensor.handle, index, delta)]):
        return sparse_tensor.value()

    def sub(self, sparse_tensor, index, delta):
      with ops.control_dependencies(
          [resource_variable_ops.resource_scatter_sub(
              sparse_tensor.handle, index, delta)]):
        return sparse_tensor.value()

    def update(self, sparse_tensor, index, value):
      with ops.control_dependencies(
          [resource_variable_ops.resource_scatter_update(
              sparse_tensor.handle, index, value)]):
        return sparse_tensor.value()

  def _apply_sparse_shared(self, grad, var, indices, scatter_op_wrapper):
    beta1_power, beta2_power = self._get_beta_accumulators()
    beta1_power = math_ops.cast(beta1_power, var.dtype.base_dtype)
    beta2_power = math_ops.cast(beta2_power, var.dtype.base_dtype)
    lr_t = math_ops.cast(self._lr_t, var.dtype.base_dtype)
    beta1_t = math_ops.cast(self._beta1_t, var.dtype.base_dtype)
    beta2_t = math_ops.cast(self._beta2_t, var.dtype.base_dtype)
    epsilon_t = math_ops.cast(self._epsilon_t, var.dtype.base_dtype)
    lr = (lr_t * math_ops.sqrt(1 - beta2_power) / (1 - beta1_power))
    # m_t = beta1 * m + (1 - beta1) * g_t
    m = self.get_slot(var, "m")
    m_scaled_g_values = grad * (1 - beta1_t)
    m_gathered = array_ops.gather(m, indices)
    m_t_gathered = m_gathered * beta1_t + m_scaled_g_values
    m_t = scatter_op_wrapper.update(m, indices, m_t_gathered)
    # v_t = beta2 * v + (1 - beta2) * (g_t * g_t)
    v = self.get_slot(var, "v")
    v_scaled_g_values = (grad * grad) * (1 - beta2_t)
    v_gathered = array_ops.gather(v, indices)
    v_t_gathered = v_gathered * beta2_t + v_scaled_g_values
    v_t = scatter_op_wrapper.update(v, indices, v_t_gathered)

    v_sqrt_gathered = math_ops.sqrt(v_t_gathered)
    var_update = scatter_op_wrapper.sub(
        var, indices, lr * m_t_gathered / (v_sqrt_gathered + epsilon_t))
    return control_flow_ops.group(*[var_update, m_t, v_t])

  def _apply_sparse(self, grad, var):
    return self._apply_sparse_shared(
        grad.values, var, grad.indices, AdamOptimizer.ScatterOpWrapper(self._use_locking))

  def _resource_apply_sparse(self, grad, var, indices):
    return self._apply_sparse_shared(
       grad, var, indices, AdamOptimizer.ResourceScatterOpWrapper(self._use_locking))

  def _finish(self, update_ops, name_scope):
    # Update the power accumulators.
    with ops.control_dependencies(update_ops):
      beta1_power, beta2_power = self._get_beta_accumulators()
      with ops.colocate_with(beta1_power):
        update_beta1 = beta1_power.assign(
            beta1_power * self._beta1_t, use_locking=self._use_locking)
        update_beta2 = beta2_power.assign(
            beta2_power * self._beta2_t, use_locking=self._use_locking)
    return control_flow_ops.group(*update_ops + [update_beta1, update_beta2],
                                  name=name_scope)
<commit_msg>Fix lint<commit_after># Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Adam for TensorFlow."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensorflow.python.eager import context
from tensorflow.python.framework import ops
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import control_flow_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import resource_variable_ops
from tensorflow.python.ops import state_ops
from tensorflow.python.training import optimizer
from tensorflow.python.training import training_ops
from tensorflow.python.util.tf_export import tf_export

@tf_export("train.AdamOptimizer")
class AdamOptimizer(optimizer.Optimizer):
  """Optimizer that implements the Adam algorithm.

  See [Kingma et al., 2014](http://arxiv.org/abs/1412.6980)
  ([pdf](http://arxiv.org/pdf/1412.6980.pdf)).
  """

  def __init__(self, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-8,
               use_locking=False, name="Adam"):
    """Construct a new Adam optimizer.

    Initialization:

    $$m_0 := 0 \text{(Initialize initial 1st moment vector)}$$
    $$v_0 := 0 \text{(Initialize initial 2nd moment vector)}$$
    $$t := 0 \text{(Initialize timestep)}$$

    The update rule for `variable` with gradient `g` uses an optimization
    described at the end of section2 of the paper:

    $$t := t + 1$$
    $$lr_t := \text{learning\_rate} * \sqrt{1 - beta_2^t} / (1 - beta_1^t)$$

    $$m_t := beta_1 * m_{t-1} + (1 - beta_1) * g$$
    $$v_t := beta_2 * v_{t-1} + (1 - beta_2) * g * g$$
    $$variable := variable - lr_t * m_t / (\sqrt{v_t} + \epsilon)$$

    The default value of 1e-8 for epsilon might not be a good default in
    general. For example, when training an Inception network on ImageNet a
    current good choice is 1.0 or 0.1. Note that since AdamOptimizer uses the
    formulation just before Section 2.1 of the Kingma and Ba paper rather than
    the formulation in Algorithm 1, the "epsilon" referred to here is "epsilon
    hat" in the paper.

    The sparse implementation of this algorithm (used when the gradient is an
    IndexedSlices object, typically because of `tf.gather` or an embedding
    lookup in the forward pass) does apply momentum to variable slices even if
    they were not used in the forward pass (meaning they have a gradient equal
    to zero). Momentum decay (beta1) is also applied to the entire momentum
    accumulator. This means that the sparse behavior is equivalent to the dense
    behavior (in contrast to some momentum implementations which ignore momentum
    unless a variable slice was actually used).

    Args:
      learning_rate: A Tensor or a floating point value.  The learning rate.
      beta1: A float value or a constant float tensor.
        The exponential decay rate for the 1st moment estimates.
      beta2: A float value or a constant float tensor.
        The exponential decay rate for the 2nd moment estimates.
      epsilon: A small constant for numerical stability. This epsilon is
        "epsilon hat" in the Kingma and Ba paper (in the formula just before
        Section 2.1), not the epsilon in Algorithm 1 of the paper.
      use_locking: If True use locks for update operations.
      name: Optional name for the operations created when applying gradients.
        Defaults to "Adam".

    @compatibility(eager)
    When eager execution is enabled, `learning_rate`, `beta1`, `beta2`, and
    `epsilon` can each be a callable that takes no arguments and returns the
    actual value to use. This can be useful for changing these values across
    different invocations of optimizer functions.
    @end_compatibility
    """
    super(AdamOptimizer, self).__init__(use_locking, name)
    self._lr = learning_rate
    self._beta1 = beta1
    self._beta2 = beta2
    self._epsilon = epsilon

    # Tensor versions of the constructor arguments, created in _prepare().
    self._lr_t = None
    self._beta1_t = None
    self._beta2_t = None
    self._epsilon_t = None

    # Created in SparseApply if needed.
    self._updated_lr = None

  def _get_beta_accumulators(self):
    with ops.init_scope():
      if context.executing_eagerly():
        graph = None
      else:
        graph = ops.get_default_graph()
      return (self._get_non_slot_variable("beta1_power", graph=graph),
              self._get_non_slot_variable("beta2_power", graph=graph))

  def _create_slots(self, var_list):
    # Create the beta1 and beta2 accumulators on the same device as the first
    # variable. Sort the var_list to make sure this device is consistent across
    # workers (these need to go on the same PS, otherwise some updates are
    # silently ignored).
    first_var = min(var_list, key=lambda x: x.name)
    self._create_non_slot_variable(initial_value=self._beta1,
                                   name="beta1_power",
                                   colocate_with=first_var)
    self._create_non_slot_variable(initial_value=self._beta2,
                                   name="beta2_power",
                                   colocate_with=first_var)

    # Create slots for the first and second moments.
    for v in var_list:
      self._zeros_slot(v, "m", self._name)
      self._zeros_slot(v, "v", self._name)

  def _prepare(self):
    lr = self._call_if_callable(self._lr)
    beta1 = self._call_if_callable(self._beta1)
    beta2 = self._call_if_callable(self._beta2)
    epsilon = self._call_if_callable(self._epsilon)

    self._lr_t = ops.convert_to_tensor(lr, name="learning_rate")
    self._beta1_t = ops.convert_to_tensor(beta1, name="beta1")
    self._beta2_t = ops.convert_to_tensor(beta2, name="beta2")
    self._epsilon_t = ops.convert_to_tensor(epsilon, name="epsilon")

  def _apply_dense(self, grad, var):
    m = self.get_slot(var, "m")
    v = self.get_slot(var, "v")
    beta1_power, beta2_power = self._get_beta_accumulators()
    return training_ops.apply_adam(
        var, m, v,
        math_ops.cast(beta1_power, var.dtype.base_dtype),
        math_ops.cast(beta2_power, var.dtype.base_dtype),
        math_ops.cast(self._lr_t, var.dtype.base_dtype),
        math_ops.cast(self._beta1_t, var.dtype.base_dtype),
        math_ops.cast(self._beta2_t, var.dtype.base_dtype),
        math_ops.cast(self._epsilon_t, var.dtype.base_dtype),
        grad, use_locking=self._use_locking).op

  def _resource_apply_dense(self, grad, var):
    m = self.get_slot(var, "m")
    v = self.get_slot(var, "v")
    beta1_power, beta2_power = self._get_beta_accumulators()
    return training_ops.resource_apply_adam(
        var.handle, m.handle, v.handle,
        math_ops.cast(beta1_power, grad.dtype.base_dtype),
        math_ops.cast(beta2_power, grad.dtype.base_dtype),
        math_ops.cast(self._lr_t, grad.dtype.base_dtype),
        math_ops.cast(self._beta1_t, grad.dtype.base_dtype),
        math_ops.cast(self._beta2_t, grad.dtype.base_dtype),
        math_ops.cast(self._epsilon_t, grad.dtype.base_dtype),
        grad, use_locking=self._use_locking)

  class ScatterOpWrapper(object):
    """Wraps necessary scatter ops for sparse tensors."""

    def __init__(self, use_locking=False):
      self._use_locking = use_locking

    def add(self, sparse_tensor, index, delta):
      return state_ops.scatter_add(sparse_tensor, index, delta,
                                   use_locking=self._use_locking)

    def sub(self, sparse_tensor, index, delta):
      return state_ops.scatter_sub(sparse_tensor, index, delta,
                                   use_locking=self._use_locking)

    def update(self, sparse_tensor, index, value):
      return state_ops.scatter_update(sparse_tensor, index, value,
                                      use_locking=self._use_locking)

  class ResourceScatterOpWrapper(ScatterOpWrapper):
    """Wraps necessay scatter ops for sparse resource variables."""

    def __init__(self, use_locking=False):
      super(AdamOptimizer.ResourceScatterOpWrapper, self).__init__(use_locking)

    def add(self, sparse_tensor, index, delta):
      with ops.control_dependencies(
          [resource_variable_ops.resource_scatter_add(
              sparse_tensor.handle, index, delta)]):
        return sparse_tensor.value()

    def sub(self, sparse_tensor, index, delta):
      with ops.control_dependencies(
          [resource_variable_ops.resource_scatter_sub(
              sparse_tensor.handle, index, delta)]):
        return sparse_tensor.value()

    def update(self, sparse_tensor, index, value):
      with ops.control_dependencies(
          [resource_variable_ops.resource_scatter_update(
              sparse_tensor.handle, index, value)]):
        return sparse_tensor.value()

  def _apply_sparse_shared(self, grad, var, indices, scatter_op_wrapper):
    beta1_power, beta2_power = self._get_beta_accumulators()
    beta1_power = math_ops.cast(beta1_power, var.dtype.base_dtype)
    beta2_power = math_ops.cast(beta2_power, var.dtype.base_dtype)
    lr_t = math_ops.cast(self._lr_t, var.dtype.base_dtype)
    beta1_t = math_ops.cast(self._beta1_t, var.dtype.base_dtype)
    beta2_t = math_ops.cast(self._beta2_t, var.dtype.base_dtype)
    epsilon_t = math_ops.cast(self._epsilon_t, var.dtype.base_dtype)
    lr = (lr_t * math_ops.sqrt(1 - beta2_power) / (1 - beta1_power))
    # m_t = beta1 * m + (1 - beta1) * g_t
    m = self.get_slot(var, "m")
    m_scaled_g_values = grad * (1 - beta1_t)
    m_gathered = array_ops.gather(m, indices)
    m_t_gathered = m_gathered * beta1_t + m_scaled_g_values
    m_t = scatter_op_wrapper.update(m, indices, m_t_gathered)
    # v_t = beta2 * v + (1 - beta2) * (g_t * g_t)
    v = self.get_slot(var, "v")
    v_scaled_g_values = (grad * grad) * (1 - beta2_t)
    v_gathered = array_ops.gather(v, indices)
    v_t_gathered = v_gathered * beta2_t + v_scaled_g_values
    v_t = scatter_op_wrapper.update(v, indices, v_t_gathered)

    v_sqrt_gathered = math_ops.sqrt(v_t_gathered)
    var_update = scatter_op_wrapper.sub(
        var, indices, lr * m_t_gathered / (v_sqrt_gathered + epsilon_t))
    return control_flow_ops.group(*[var_update, m_t, v_t])

  def _apply_sparse(self, grad, var):
    return self._apply_sparse_shared(
        grad.values, var, grad.indices,
        AdamOptimizer.ScatterOpWrapper(self._use_locking))

  def _resource_apply_sparse(self, grad, var, indices):
    return self._apply_sparse_shared(
        grad, var, indices,
        AdamOptimizer.ResourceScatterOpWrapper(self._use_locking))

  def _finish(self, update_ops, name_scope):
    # Update the power accumulators.
    with ops.control_dependencies(update_ops):
      beta1_power, beta2_power = self._get_beta_accumulators()
      with ops.colocate_with(beta1_power):
        update_beta1 = beta1_power.assign(
            beta1_power * self._beta1_t, use_locking=self._use_locking)
        update_beta2 = beta2_power.assign(
            beta2_power * self._beta2_t, use_locking=self._use_locking)
    return control_flow_ops.group(*update_ops + [update_beta1, update_beta2],
                                  name=name_scope)
<|endoftext|>


<commit_before>#  Copyright 2008-2011 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

from output import Output
from logger import LOGGER
from monitor import CommandLineMonitor
from xmllogger import XmlLogger
from loggerhelper import LEVELS, Message

#TODO: Fix Public API
def TestSuite(outpath):
    """Factory method for getting test suite from an xml output file.

    If you want statistics get suite first and say Statistics(suite).
    """
    suite, errors = process_output(outpath)

    def write_to_file(path=None):
        """Write processed suite (incl. statistics and errors) back to xml.

        If path is not given the suite is written into the same file as it
        originally was read from.
        """
        if path is None:
            path = outpath
        suite.set_status()
        testoutput = ResultFromXml(suite, errors)
        testoutput.serialize_output(path)

    suite.write_to_file = write_to_file
    return suite

<commit_msg>output: removed old, broken TestSuite API<commit_after>#  Copyright 2008-2011 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

from .output import Output
from .logger import LOGGER
from .xmllogger import XmlLogger
from .loggerhelper import LEVELS, Message
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import shutil
import subprocess
import sys
#import tarfile

from inspectShell import archOfBinary, testDbgOrOpt, testJsShellOrXpcshell

p0 = os.path.dirname(os.path.abspath(__file__))
lithiumpy = os.path.abspath(os.path.join(p0, os.pardir, 'lithium', 'lithium.py'))
autobisectpy = os.path.abspath(os.path.join(p0, os.pardir, 'autoBisectJs', 'autoBisect.py'))
shellBeautificationpy = os.path.join(p0, 'shellBeautification.py')

path2 = os.path.abspath(os.path.join(p0, os.pardir, 'util'))
sys.path.append(path2)
from subprocesses import captureStdout, shellify

def tempdir(path):
    os.mkdir(path)
    return "--tempdir=" + path

def pinpoint(itest, logPrefix, jsEngine, engineFlags, infilename, bisectRepo, targetTime, alsoRunChar=True, alsoReduceEntireFile=False):
    """
       Run Lithium and autobisect.

       itest must be an array of the form [module, ...] where module is an interestingness module.
       The module's "interesting" function must accept [...] + [jsEngine] + engineFlags + infilename
       (If it's not prepared to accept engineFlags, engineFlags must be empty.)
    """

    valgrindSupport = "--valgrind" in itest
    valgrindX = ["--valgrind"] if valgrindSupport else []

    lithArgs = itest + [jsEngine] + engineFlags + [infilename]
    print shellify([lithiumpy] + lithArgs)
    subprocess.call([sys.executable, lithiumpy, tempdir(logPrefix + "-lith1-tmp")] + lithArgs, stdout=open(logPrefix + "-lith1-out", "w"))

    if alsoRunChar:
        lith2Args = ["--char"] + lithArgs
        print shellify([lithiumpy] + lith2Args)
        subprocess.call([sys.executable, lithiumpy, tempdir(logPrefix + "-lith2-tmp")] + lith2Args, stdout=open(logPrefix + "-lith2-out", "w"))

    print
    print "Done running Lithium on the part in between DDBEGIN and DDEND. To reproduce, run:"
    print shellify([lithiumpy, "--strategy=check-only"] + lithArgs)
    print

    unbeautifiedOutput = captureStdout([sys.executable, lithiumpy, "--strategy=check-only", tempdir(logPrefix + "-lith-b-tmp")] + lithArgs)[0]
    # Check that the testcase is interesting.
    if False and alsoReduceEntireFile and 'not interesting' not in unbeautifiedOutput:
        assert 'interesting' in unbeautifiedOutput
        # Beautify the output. This will remove DDBEGIN and DDEND as they are comments.
        # This will output a file with the '-beautified' suffix.
        # Reduce once using toString decompile method.
        subprocess.call([sys.executable, shellBeautificationpy, '--shell=' + jsEngine, "--decompilationType=uneval", infilename])

        print 'Operating on the beautified testcase for the n-th time where n =',
        # iterNum starts from 3 because lith1 and lith2 are already used above.
        iterNumStart = 3 if alsoRunChar is True else 2
        iterNum = iterNumStart
        # Run Lithium on the testcase 10 more times, but run it using char only for 3 tries of toString and uneval reduction each.
        # Generally, lines don't get significantly reduced after the 3rd try of line reduction.
        MAX_BEAUTIFIED_LITHIUM_RUNS = iterNumStart + 10
        while(iterNum < MAX_BEAUTIFIED_LITHIUM_RUNS):
            print iterNum - 2,
            # Operate on the beautified version first.
            lithArgs = lithArgs[0:-1]
            beautifiedFilename = infilename + '-beautified'
            lithArgs = lithArgs + [beautifiedFilename]
            # We must still be operating on the beautified version.
            assert beautifiedFilename in lithArgs
            # Check that the testcase is still interesting.
            beautifiedOutput = captureStdout([sys.executable, lithiumpy, "--strategy=check-only", tempdir(logPrefix + "-lith" + str(iterNum) + "-b-tmp")] + lithArgs)[0]
            if 'not interesting' not in beautifiedOutput:
                assert 'interesting' in beautifiedOutput
                # Overwrite the original -reduced file with the beautified version since it is interesting.
                shutil.move(beautifiedFilename, infilename)
                # Operate on the original -reduced file.
                lithArgs = lithArgs[0:-1]
                lithArgs = lithArgs + [infilename]
                assert beautifiedFilename not in lithArgs
                #print shellify([lithiumpy] + lithArgs)
                subprocess.call([sys.executable, lithiumpy, tempdir(logPrefix + '-lith' + str(iterNum) + '-tmp')] + lithArgs, stdout=open(logPrefix + "-lith" + str(iterNum) + "-out", "w"))

                # Run it using char only after 3 tries of toString and uneval reduction each.
                if alsoRunChar and (iterNum - 2) > ((MAX_BEAUTIFIED_LITHIUM_RUNS - iterNumStart) // 2):
                    iterNum += 1
                    print iterNum - 2,
                    print '(operating on chars..)',
                    assert iterNum in (9, 11, 13)  # Refer to reduction method below
                    lith2Args = ["--char"] + lithArgs
                    assert beautifiedFilename not in lith2Args
                    #print shellify([lithiumpy] + lith2Args)
                    subprocess.call([sys.executable, lithiumpy, tempdir(logPrefix + '-lith' + str(iterNum) + '-tmp')] + lith2Args, stdout=open(logPrefix + "-lith" + str(iterNum) + "-out", "w"))
            else:
                # Beautified testcase is no longer interesting.
                print 'Beautified testcase is no longer interesting!'
                break
            iterNum += 1
            # We want the following reduction method:
            # iterNum 3: uneval
            # iterNum 4: toString
            # iterNum 5: uneval
            # iterNum 6: toString
            # iterNum 7: uneval
            # iterNum 8: toString
            # iterNum 9: char
            # iterNum 10: uneval
            # iterNum 11: char
            # iterNum 12: toString
            # iterNum 13: char
            if iterNum < MAX_BEAUTIFIED_LITHIUM_RUNS:
                # This will output a file with the '-beautified' suffix.
                # Rotate between reducing using the toString and uneval decompile method
                if alsoRunChar:
                    if iterNum % 2 == 0 and iterNum != 10:
                        # toString
                        assert iterNum in (4, 6, 8, 12)
                        subprocess.call([sys.executable, shellBeautificationpy, '--shell=' + jsEngine, "--decompilationType=toString", infilename])
                    else:
                        # uneval
                        # iterNum 3 has already occurred prior to the increment of iterNum above.
                        assert iterNum in (5, 7, 10)
                        subprocess.call([sys.executable, shellBeautificationpy, '--shell=' + jsEngine, "--decompilationType=uneval", infilename])
                else:
                    # If alsoRunChar is false, which occurs when jsInteresting.py is operating at JS_DID_NOT_FINISH and below. iterNumStart also starts from 2.
                    if iterNum % 2 != 0:
                        # toString
                        assert iterNum in (3, 5, 7, 9, 11, 13)
                        subprocess.call([sys.executable, shellBeautificationpy, '--shell=' + jsEngine, "--decompilationType=toString", infilename])
                    else:
                        # uneval
                        # iterNum 3 has already occurred prior to the increment of iterNum above.
                        assert iterNum in (4, 6, 8, 10, 12)
                        subprocess.call([sys.executable, shellBeautificationpy, '--shell=' + jsEngine, "--decompilationType=uneval", infilename])
            else:
                print

        # Operate on the original -reduced file.
        lithArgs = lithArgs[0:-1]
        lithArgs = lithArgs + [infilename]
        assert beautifiedFilename not in lithArgs
        # Check that the testcase is still interesting after the extra beautified lithium reductions.
        finalBeautifiedOutput = captureStdout([sys.executable, lithiumpy, "--strategy=check-only", tempdir(logPrefix + "-lith-final-tmp")] + lithArgs)[0]
        if 'not interesting' not in finalBeautifiedOutput:
            assert 'interesting' in finalBeautifiedOutput

        # Archive all wXX-lith*-tmp directories in a tarball.
        #lithTmpDirTarball = tarfile.open('tempName.tar.bz2', 'w:bz2')
        #for loop from 1 to iterNum
            #lithTmpDirTarball.add(logPrefix + '-lith' + str(iterNum) + '-tmp')
        #lithTmpDirTarball.close()

    # FIXME: We should do the build slave detection logic later, and maybe somewhere else.
    # i.e. fix the logic of isBuildSlave and remove os.path.join('build', 'dist', 'js') not in jsEngine hack.
    isBuildSlave = False
    jsEngineName = os.path.basename(jsEngine)
    if bisectRepo is not "none":
        # We cannot test that it is not xpcshell with Python 2.5 since testJsShellOrXpcshell relies
        # on 'delete' being a keyword argument in NamedTemporaryFile(). The testing functions in
        # inspectShell in general need at least Python 2.6 because of this.
        if not isBuildSlave and sys.version_info >= (2, 6) and testJsShellOrXpcshell(jsEngine) != "xpcshell" and os.path.join('build', 'dist', 'js') not in jsEngine:
            autobisectCmd = [sys.executable, autobisectpy] + valgrindX + ["-d", bisectRepo, "-i", "-p", "-a", archOfBinary(jsEngine), "-c", testDbgOrOpt(jsEngine), ("--flags=" + ','.join(engineFlags)) if engineFlags else ""] + [infilename] + itest
            print shellify(autobisectCmd)
            subprocess.call(autobisectCmd, stdout=open(logPrefix + "-autobisect", "w"), stderr=subprocess.STDOUT)
            print "Done running autobisect. Log: " + logPrefix + "-autobisect"
        elif not isBuildSlave and sys.version_info < (2, 6):
            print 'Not pinpointing to exact changeset, please use a Python version >= 2.6.'
<commit_msg>Don't run autoBisect when there is a targetTime<commit_after>#!/usr/bin/env python

import os
import shutil
import subprocess
import sys
#import tarfile

from inspectShell import archOfBinary, testDbgOrOpt, testJsShellOrXpcshell

p0 = os.path.dirname(os.path.abspath(__file__))
lithiumpy = os.path.abspath(os.path.join(p0, os.pardir, 'lithium', 'lithium.py'))
autobisectpy = os.path.abspath(os.path.join(p0, os.pardir, 'autoBisectJs', 'autoBisect.py'))
shellBeautificationpy = os.path.join(p0, 'shellBeautification.py')

path2 = os.path.abspath(os.path.join(p0, os.pardir, 'util'))
sys.path.append(path2)
from subprocesses import captureStdout, shellify

def tempdir(path):
    os.mkdir(path)
    return "--tempdir=" + path

def pinpoint(itest, logPrefix, jsEngine, engineFlags, infilename, bisectRepo, targetTime, alsoRunChar=True, alsoReduceEntireFile=False):
    """
       Run Lithium and autobisect.

       itest must be an array of the form [module, ...] where module is an interestingness module.
       The module's "interesting" function must accept [...] + [jsEngine] + engineFlags + infilename
       (If it's not prepared to accept engineFlags, engineFlags must be empty.)
    """

    valgrindSupport = "--valgrind" in itest
    valgrindX = ["--valgrind"] if valgrindSupport else []

    lithArgs = itest + [jsEngine] + engineFlags + [infilename]
    print shellify([lithiumpy] + lithArgs)
    subprocess.call([sys.executable, lithiumpy, tempdir(logPrefix + "-lith1-tmp")] + lithArgs, stdout=open(logPrefix + "-lith1-out", "w"))

    if alsoRunChar:
        lith2Args = ["--char"] + lithArgs
        print shellify([lithiumpy] + lith2Args)
        subprocess.call([sys.executable, lithiumpy, tempdir(logPrefix + "-lith2-tmp")] + lith2Args, stdout=open(logPrefix + "-lith2-out", "w"))

    print
    print "Done running Lithium on the part in between DDBEGIN and DDEND. To reproduce, run:"
    print shellify([lithiumpy, "--strategy=check-only"] + lithArgs)
    print

    unbeautifiedOutput = captureStdout([sys.executable, lithiumpy, "--strategy=check-only", tempdir(logPrefix + "-lith-b-tmp")] + lithArgs)[0]
    # Check that the testcase is interesting.
    if False and alsoReduceEntireFile and 'not interesting' not in unbeautifiedOutput:
        assert 'interesting' in unbeautifiedOutput
        # Beautify the output. This will remove DDBEGIN and DDEND as they are comments.
        # This will output a file with the '-beautified' suffix.
        # Reduce once using toString decompile method.
        subprocess.call([sys.executable, shellBeautificationpy, '--shell=' + jsEngine, "--decompilationType=uneval", infilename])

        print 'Operating on the beautified testcase for the n-th time where n =',
        # iterNum starts from 3 because lith1 and lith2 are already used above.
        iterNumStart = 3 if alsoRunChar is True else 2
        iterNum = iterNumStart
        # Run Lithium on the testcase 10 more times, but run it using char only for 3 tries of toString and uneval reduction each.
        # Generally, lines don't get significantly reduced after the 3rd try of line reduction.
        MAX_BEAUTIFIED_LITHIUM_RUNS = iterNumStart + 10
        while(iterNum < MAX_BEAUTIFIED_LITHIUM_RUNS):
            print iterNum - 2,
            # Operate on the beautified version first.
            lithArgs = lithArgs[0:-1]
            beautifiedFilename = infilename + '-beautified'
            lithArgs = lithArgs + [beautifiedFilename]
            # We must still be operating on the beautified version.
            assert beautifiedFilename in lithArgs
            # Check that the testcase is still interesting.
            beautifiedOutput = captureStdout([sys.executable, lithiumpy, "--strategy=check-only", tempdir(logPrefix + "-lith" + str(iterNum) + "-b-tmp")] + lithArgs)[0]
            if 'not interesting' not in beautifiedOutput:
                assert 'interesting' in beautifiedOutput
                # Overwrite the original -reduced file with the beautified version since it is interesting.
                shutil.move(beautifiedFilename, infilename)
                # Operate on the original -reduced file.
                lithArgs = lithArgs[0:-1]
                lithArgs = lithArgs + [infilename]
                assert beautifiedFilename not in lithArgs
                #print shellify([lithiumpy] + lithArgs)
                subprocess.call([sys.executable, lithiumpy, tempdir(logPrefix + '-lith' + str(iterNum) + '-tmp')] + lithArgs, stdout=open(logPrefix + "-lith" + str(iterNum) + "-out", "w"))

                # Run it using char only after 3 tries of toString and uneval reduction each.
                if alsoRunChar and (iterNum - 2) > ((MAX_BEAUTIFIED_LITHIUM_RUNS - iterNumStart) // 2):
                    iterNum += 1
                    print iterNum - 2,
                    print '(operating on chars..)',
                    assert iterNum in (9, 11, 13)  # Refer to reduction method below
                    lith2Args = ["--char"] + lithArgs
                    assert beautifiedFilename not in lith2Args
                    #print shellify([lithiumpy] + lith2Args)
                    subprocess.call([sys.executable, lithiumpy, tempdir(logPrefix + '-lith' + str(iterNum) + '-tmp')] + lith2Args, stdout=open(logPrefix + "-lith" + str(iterNum) + "-out", "w"))
            else:
                # Beautified testcase is no longer interesting.
                print 'Beautified testcase is no longer interesting!'
                break
            iterNum += 1
            # We want the following reduction method:
            # iterNum 3: uneval
            # iterNum 4: toString
            # iterNum 5: uneval
            # iterNum 6: toString
            # iterNum 7: uneval
            # iterNum 8: toString
            # iterNum 9: char
            # iterNum 10: uneval
            # iterNum 11: char
            # iterNum 12: toString
            # iterNum 13: char
            if iterNum < MAX_BEAUTIFIED_LITHIUM_RUNS:
                # This will output a file with the '-beautified' suffix.
                # Rotate between reducing using the toString and uneval decompile method
                if alsoRunChar:
                    if iterNum % 2 == 0 and iterNum != 10:
                        # toString
                        assert iterNum in (4, 6, 8, 12)
                        subprocess.call([sys.executable, shellBeautificationpy, '--shell=' + jsEngine, "--decompilationType=toString", infilename])
                    else:
                        # uneval
                        # iterNum 3 has already occurred prior to the increment of iterNum above.
                        assert iterNum in (5, 7, 10)
                        subprocess.call([sys.executable, shellBeautificationpy, '--shell=' + jsEngine, "--decompilationType=uneval", infilename])
                else:
                    # If alsoRunChar is false, which occurs when jsInteresting.py is operating at JS_DID_NOT_FINISH and below. iterNumStart also starts from 2.
                    if iterNum % 2 != 0:
                        # toString
                        assert iterNum in (3, 5, 7, 9, 11, 13)
                        subprocess.call([sys.executable, shellBeautificationpy, '--shell=' + jsEngine, "--decompilationType=toString", infilename])
                    else:
                        # uneval
                        # iterNum 3 has already occurred prior to the increment of iterNum above.
                        assert iterNum in (4, 6, 8, 10, 12)
                        subprocess.call([sys.executable, shellBeautificationpy, '--shell=' + jsEngine, "--decompilationType=uneval", infilename])
            else:
                print

        # Operate on the original -reduced file.
        lithArgs = lithArgs[0:-1]
        lithArgs = lithArgs + [infilename]
        assert beautifiedFilename not in lithArgs
        # Check that the testcase is still interesting after the extra beautified lithium reductions.
        finalBeautifiedOutput = captureStdout([sys.executable, lithiumpy, "--strategy=check-only", tempdir(logPrefix + "-lith-final-tmp")] + lithArgs)[0]
        if 'not interesting' not in finalBeautifiedOutput:
            assert 'interesting' in finalBeautifiedOutput

        # Archive all wXX-lith*-tmp directories in a tarball.
        #lithTmpDirTarball = tarfile.open('tempName.tar.bz2', 'w:bz2')
        #for loop from 1 to iterNum
            #lithTmpDirTarball.add(logPrefix + '-lith' + str(iterNum) + '-tmp')
        #lithTmpDirTarball.close()

    if bisectRepo is not "none" and targetTime is None:
        # We cannot test that it is not xpcshell with Python 2.5 since testJsShellOrXpcshell relies
        # on 'delete' being a keyword argument in NamedTemporaryFile(). The testing functions in
        # inspectShell in general need at least Python 2.6 because of this.
        if sys.version_info >= (2, 6) and testJsShellOrXpcshell(jsEngine) != "xpcshell":
            autobisectCmd = [sys.executable, autobisectpy] + valgrindX + ["-d", bisectRepo, "-i", "-p", "-a", archOfBinary(jsEngine), "-c", testDbgOrOpt(jsEngine), ("--flags=" + ','.join(engineFlags)) if engineFlags else ""] + [infilename] + itest
            print shellify(autobisectCmd)
            subprocess.call(autobisectCmd, stdout=open(logPrefix + "-autobisect", "w"), stderr=subprocess.STDOUT)
            print "Done running autobisect. Log: " + logPrefix + "-autobisect"
        elif sys.version_info < (2, 6):
            print 'Not pinpointing to exact changeset, please use a Python version >= 2.6.'
<|endoftext|>


<commit_before>#!/usr/bin/env python

import requests
import urllib
import json
import os
import re

from flask import Flask
from flask import request
from flask import make_response
from datetime import datetime as DateTime, timedelta as TimeDelta

# Flask app should start in global layout
app = Flask(__name__)

@app.route('/webhook', methods=['POST'])
def webhook():
    req = request.get_json(silent=True, force=True)

    print("Request:")
    print(json.dumps(req, indent=4))

    res = makeWebhookResult(req)

    res = json.dumps(res, indent=4)
    print(res)
    r = make_response(res)
    r.headers['Content-Type'] = 'application/json'
    return r

def makeWebhookResult(req):
    if req.get("result").get("action") == "browse.search.products":
        result = req.get("result")
        parameters = result.get("parameters")
        color = parameters.get("color")
        cat = parameters.get("catalog-category")
        if (((color is None) or (color is "")) and (req.get("originalRequest") is not None) and (req.get("originalRequest").get("source") == "facebook")):
            return {
                "data": {
                    "facebook": {
                        "text": "Pick a color:",
                        "quick_replies": [
                            {
                                "content_type": "text",
                                "title": "Red",
                                "payload": "red"
                            },
                            {
                                "content_type": "text",
                                "title": "Green",
                                "payload": "green"
                            }
                        ]
                    }
                }
            }
        else:
            rq = requests.get("http://www.lanebryant.com/lanebryant/search?Ntt=" + color + " " + cat + "&format=JSON")
            jdata = json.loads(rq.text)
            speech = "I found " + str(jdata["contents"][0]["MainContent"][0]["MainContent"][0]["contents"][0]["totalNumRecs"]) + " " + color + " " + cat + " products."
    elif req.get("result").get("action") == "promos":
        result = req.get("result")
        headers = {'HOST': 'sit.catherines.com'}
        rq = requests.get("https://23.34.4.174/static/promo_01?format=json", headers=headers, verify=False)
        jdata = json.loads(rq.text)

        if ((req.get("originalRequest") is not None) and (req.get("originalRequest").get("source") == "facebook")):
            temp = "\"image_url\": \"https://www.lanebryant.com/assets/images/lanebryant-logo.png\",\"default_action\":{\"type\": \"web_url\",\"url\": \"http://www.lanebryant.com/\"}}"
            elements = ""
            count = len(jdata["MainContent"])

            for mc in jdata["MainContent"]:
                element = "{\"title\": " + "\"" + str(mc["freeFormContent"]) + "\"," + temp
                if(count != 1):
                    element = element + ","
                    count = count - 1
                elements = elements + element

            json_elements = json.loads("["+elements+"]")
            return{
                "data": {
                    "facebook": {
                        "attachment": {
                            "type": "template",
                            "payload": {
                                "template_type": "list",
                                "elements": json_elements
                            }
                        }
                    }
                }
            }
        else:
            speech = "Promos are "
            for mc in jdata["MainContent"]:
                speech = speech + str(mc["freeFormContent"]) + "... "
    elif ((req.get("result").get("action") == "Order_Status_yes") or (req.get("result").get("action") == "checkout.order.status")):
        result = req.get("result")
        parameters = result.get("parameters")
        zipcode = parameters.get("zipcode")
        ordernum = parameters.get("order-number")

        rq = requests.post("https://www.shopjustice.com/justice/homepage/includes/order-response-html.jsp", data={'orderNum': ordernum, 'billingZip': zipcode, 'Action': 'fetchODDetails'})
        #print rq.text
        matchObj = rq.text[rq.text.find("mar-status")+12:rq.text.find("<", rq.text.find("mar-status"))]
        matchDate = rq.text[rq.text.find("mar-date")+10:rq.text.find("<", rq.text.find("mar-date"))]
        date = DateTime.now()
        present = DateTime.now()

        if len(matchObj) < 50:
            print ("matchObj : ", matchObj)
            print ("matchDate : ", matchDate)
            status = matchObj
            date = DateTime.strptime(matchDate, '%m/%d/%Y') + TimeDelta(days=5)
        else:
            status = "I couldn't find that order. Either the number or the zipcode is not correct."
            print ("No match!!")

        if date >= present:
            if status == 'Shipped':
                speech = "Order status is " + status + ". You should receive the package by " + date.strftime('%m/%d/%Y') + "."
            elif status == 'Partially Shipped':
                speech = "Order status is " + status + ". You should receive the package by " + date.strftime('%m/%d/%Y') + "."
            elif status == 'Canceled':
                speech = "Order status is " + status + ". Please reach out to the customer support for more details about the order."
            elif status == 'Processing':
                speech = "Order status is " + status + ". You should receive the package by " + date.strftime('%m/%d/%Y') + "."
            else:
                speech = status
        else:
            if status == 'Shipped':
                speech = "Order status is " + status + ". You should have received the package by " + date.strftime('%m/%d/%Y') + "."
            elif status == 'Partially Shipped':
                speech = "Order status is " + status + ". You should have received the package by " + date.strftime('%m/%d/%Y') + "."
            elif status == 'Canceled':
                speech = "Order status is " + status + ". Please reach out to the customer support for more details about the order."
            elif status == 'Processing':
                speech = "Order status is " + status + ". You should have received the package by " + date.strftime('%m/%d/%Y') + "."
            else:
                speech = status
    elif req.get("result").get("action") == "Order_Status_no":
        result = req.get("result")
        parameters = result.get("parameters")
        zipcode = parameters.get("zipcode")
        email = parameters.get("email")
        ordertime = parameters.get("order-time")

        #TODO - Need to put here the API to get order number using email, zipcode and the timeframe
        if zipcode == '20166':
            ordernum = 'OJTW022967052'
        elif zipcode == '37122':
            ordernum = 'OJTW027567667'
        elif zipcode == '19148':
            ordernum = 'OJTW027678055'
        else:
            ordernum = 'OJTW022967052'

        rq = requests.post("https://www.shopjustice.com/justice/homepage/includes/order-response-html.jsp", data={'orderNum': ordernum, 'billingZip': zipcode, 'Action': 'fetchODDetails'})
        #print rq.text
        matchObj = rq.text[rq.text.find("mar-status")+12:rq.text.find("<", rq.text.find("mar-status"))]
        matchDate = rq.text[rq.text.find("mar-date")+10:rq.text.find("<", rq.text.find("mar-date"))]
        date = DateTime.now()
        present = DateTime.now()

        if len(matchObj) < 50:
            print ("matchObj : ", matchObj)
            print ("matchDate : ", matchDate)
            status = matchObj
            date = DateTime.strptime(matchDate, '%m/%d/%Y') + TimeDelta(days=5)
        else:
            status = "I couldn't find that order. Either the order number or the zipcode is not correct."
            print ("No match!!")

        if date >= present:
            if status == 'Shipped':
                speech = "Order status is " + status + ". You should receive the package by " + date.strftime('%m/%d/%Y') + "."
            elif status == 'Partially Shipped':
                speech = "Order status is " + status + ". You should receive the package by " + date.strftime('%m/%d/%Y') + "."
            elif status == 'Canceled':
                speech = "Order status is " + status + ". Please reach out to the customer support for more details about the order."
            elif status == 'Processing':
                speech = "Order status is " + status + ". You should receive the package by " + date.strftime('%m/%d/%Y') + "."
            else:
                speech = status
        else:
            if status == 'Shipped':
                speech = "Order status is " + status + ". You should have received the package by " + date.strftime('%m/%d/%Y') + "."
            elif status == 'Partially Shipped':
                speech = "Order status is " + status + ". You should have received the package by " + date.strftime('%m/%d/%Y') + "."
            elif status == 'Canceled':
                speech = "Order status is " + status + ". Please reach out to the customer support for more details about the order."
            elif status == 'Processing':
                speech = "Order status is " + status + ". You should have received the package by " + date.strftime('%m/%d/%Y') + "."
            else:
                speech = status
    else:
        return{}
    print("Response:")
    print(speech)
    return {
        "speech": speech,
        "displayText": speech,
        #"data": {},
        # "contextOut": [],
        "source": "apiai-onlinestore-search"
    }

if __name__ == '__main__':
    port = int(os.getenv('PORT', 5000))

    print ("Starting app on port %d" % port)

    app.run(debug=True, port=port, host='0.0.0.0')
<commit_msg>Adding new order logic<commit_after>#!/usr/bin/env python

import requests
import urllib
import json
import os
import re

from flask import Flask
from flask import request
from flask import make_response
from datetime import datetime as DateTime, timedelta as TimeDelta

# Flask app should start in global layout
app = Flask(__name__)

@app.route('/webhook', methods=['POST'])
def webhook():
    req = request.get_json(silent=True, force=True)

    print("Request:")
    print(json.dumps(req, indent=4))

    res = makeWebhookResult(req)

    res = json.dumps(res, indent=4)
    print(res)
    r = make_response(res)
    r.headers['Content-Type'] = 'application/json'
    return r

def makeWebhookResult(req):
    if req.get("result").get("action") == "browse.search.products":
        result = req.get("result")
        parameters = result.get("parameters")
        color = parameters.get("color")
        cat = parameters.get("catalog-category")
        if (((color is None) or (color is "")) and (req.get("originalRequest") is not None) and (req.get("originalRequest").get("source") == "facebook")):
            return {
                "data": {
                    "facebook": {
                        "text": "Pick a color:",
                        "quick_replies": [
                            {
                                "content_type": "text",
                                "title": "Red",
                                "payload": "red"
                            },
                            {
                                "content_type": "text",
                                "title": "Green",
                                "payload": "green"
                            }
                        ]
                    }
                }
            }
        else:
            rq = requests.get("http://www.lanebryant.com/lanebryant/search?Ntt=" + color + " " + cat + "&format=JSON")
            jdata = json.loads(rq.text)
            speech = "I found " + str(jdata["contents"][0]["MainContent"][0]["MainContent"][0]["contents"][0]["totalNumRecs"]) + " " + color + " " + cat + " products."
    elif req.get("result").get("action") == "promos":
        result = req.get("result")
        headers = {'HOST': 'sit.catherines.com'}
        rq = requests.get("https://23.34.4.174/static/promo_01?format=json", headers=headers, verify=False)
        jdata = json.loads(rq.text)

        if ((req.get("originalRequest") is not None) and (req.get("originalRequest").get("source") == "facebook")):
            temp = "\"image_url\": \"https://www.lanebryant.com/assets/images/lanebryant-logo.png\",\"default_action\":{\"type\": \"web_url\",\"url\": \"http://www.lanebryant.com/\"}}"
            elements = ""
            count = len(jdata["MainContent"])

            for mc in jdata["MainContent"]:
                element = "{\"title\": " + "\"" + str(mc["freeFormContent"]) + "\"," + temp
                if(count != 1):
                    element = element + ","
                    count = count - 1
                elements = elements + element

            json_elements = json.loads("["+elements+"]")
            return{
                "data": {
                    "facebook": {
                        "attachment": {
                            "type": "template",
                            "payload": {
                                "template_type": "list",
                                "elements": json_elements
                            }
                        }
                    }
                }
            }
        else:
            speech = "Promos are "
            for mc in jdata["MainContent"]:
                speech = speech + str(mc["freeFormContent"]) + "... "
    elif ((req.get("result").get("action") == "Order_Status_yes") or (req.get("result").get("action") == "checkout.order.status")):
        result = req.get("result")
        parameters = result.get("parameters")
        zipcode = parameters.get("zipcode")
        ordernum = parameters.get("order-number")

        rq = requests.post("https://www.shopjustice.com/justice/homepage/includes/order-response-html.jsp", data={'orderNum': ordernum, 'billingZip': zipcode, 'Action': 'fetchODDetails'})
        #print rq.text
        order_json = rq.text[rq.text.find("cart-json")+12:rq.text.find("<", rq.text.find("cart-json"))]
        print (str(order_json))
        matchObj = rq.text[rq.text.find("mar-status")+12:rq.text.find("<", rq.text.find("mar-status"))]
        matchDate = rq.text[rq.text.find("mar-date")+10:rq.text.find("<", rq.text.find("mar-date"))]
        date = DateTime.now()
        present = DateTime.now()

        if len(matchObj) < 50:
            print ("matchObj : ", matchObj)
            print ("matchDate : ", matchDate)
            status = matchObj
            date = DateTime.strptime(matchDate, '%m/%d/%Y') + TimeDelta(days=5)
        else:
            status = "I couldn't find that order. Either the number or the zipcode is not correct."
            print ("No match!!")

        if date >= present:
            if status == 'Shipped':
                speech = "Order status is " + status + ". You should receive the package by " + date.strftime('%m/%d/%Y') + "."
            elif status == 'Partially Shipped':
                speech = "Order status is " + status + ". You should receive the package by " + date.strftime('%m/%d/%Y') + "."
            elif status == 'Canceled':
                speech = "Order status is " + status + ". Please reach out to the customer support for more details about the order."
            elif status == 'Processing':
                speech = "Order status is " + status + ". You should receive the package by " + date.strftime('%m/%d/%Y') + "."
            else:
                speech = status
        else:
            if status == 'Shipped':
                speech = "Order status is " + status + ". You should have received the package by " + date.strftime('%m/%d/%Y') + "."
            elif status == 'Partially Shipped':
                speech = "Order status is " + status + ". You should have received the package by " + date.strftime('%m/%d/%Y') + "."
            elif status == 'Canceled':
                speech = "Order status is " + status + ". Please reach out to the customer support for more details about the order."
            elif status == 'Processing':
                speech = "Order status is " + status + ". You should have received the package by " + date.strftime('%m/%d/%Y') + "."
            else:
                speech = status
    elif req.get("result").get("action") == "Order_Status_no":
        result = req.get("result")
        parameters = result.get("parameters")
        zipcode = parameters.get("zipcode")
        email = parameters.get("email")
        ordertime = parameters.get("order-time")

        #TODO - Need to put here the API to get order number using email, zipcode and the timeframe
        if zipcode == '20166':
            ordernum = 'OJTW022967052'
        elif zipcode == '37122':
            ordernum = 'OJTW027567667'
        elif zipcode == '19148':
            ordernum = 'OJTW027678055'
        else:
            ordernum = 'OJTW022967052'

        rq = requests.post("https://www.shopjustice.com/justice/homepage/includes/order-response-html.jsp", data={'orderNum': ordernum, 'billingZip': zipcode, 'Action': 'fetchODDetails'})
        #print rq.text
        matchObj = rq.text[rq.text.find("mar-status")+12:rq.text.find("<", rq.text.find("mar-status"))]
        matchDate = rq.text[rq.text.find("mar-date")+10:rq.text.find("<", rq.text.find("mar-date"))]
        date = DateTime.now()
        present = DateTime.now()

        if len(matchObj) < 50:
            print ("matchObj : ", matchObj)
            print ("matchDate : ", matchDate)
            status = matchObj
            date = DateTime.strptime(matchDate, '%m/%d/%Y') + TimeDelta(days=5)
        else:
            status = "I couldn't find that order. Either the order number or the zipcode is not correct."
            print ("No match!!")

        if date >= present:
            if status == 'Shipped':
                speech = "Order status is " + status + ". You should receive the package by " + date.strftime('%m/%d/%Y') + "."
            elif status == 'Partially Shipped':
                speech = "Order status is " + status + ". You should receive the package by " + date.strftime('%m/%d/%Y') + "."
            elif status == 'Canceled':
                speech = "Order status is " + status + ". Please reach out to the customer support for more details about the order."
            elif status == 'Processing':
                speech = "Order status is " + status + ". You should receive the package by " + date.strftime('%m/%d/%Y') + "."
            else:
                speech = status
        else:
            if status == 'Shipped':
                speech = "Order status is " + status + ". You should have received the package by " + date.strftime('%m/%d/%Y') + "."
            elif status == 'Partially Shipped':
                speech = "Order status is " + status + ". You should have received the package by " + date.strftime('%m/%d/%Y') + "."
            elif status == 'Canceled':
                speech = "Order status is " + status + ". Please reach out to the customer support for more details about the order."
            elif status == 'Processing':
                speech = "Order status is " + status + ". You should have received the package by " + date.strftime('%m/%d/%Y') + "."
            else:
                speech = status
    else:
        return{}
    print("Response:")
    print(speech)
    return {
        "speech": speech,
        "displayText": speech,
        #"data": {},
        # "contextOut": [],
        "source": "apiai-onlinestore-search"
    }

if __name__ == '__main__':
    port = int(os.getenv('PORT', 5000))

    print ("Starting app on port %d" % port)

    app.run(debug=True, port=port, host='0.0.0.0')
<|endoftext|>


<commit_before>from devilry.utils import OrderedDict
from devilry.apps.core.models import AssignmentGroup

class GroupList(list):
    """
    Represents a list of :class:`devilry.apps.core.models.AssignmentGroup` objects,
    with utility functions for commonly needed actions. The list is ment to hold
    groups where the same student in candidate on a single assignment, and the
    utilities is ment to make it easier to work with the added complexity of
    supporting the same user in multiple groups on a single assignment.
    """
    def get_feedback_with_most_points(self):
        """
        Get the :class:`devilry.apps.core.models.StaticFeedback` with the most points in the list.
        """
        best = None
        for group in self:
            if best:
                if group.feedback and group.feedback.points > best.points:
                    best = group
            else:
                best = group.feedback
        return best

    def get_best_gradestring(self):
        """
        Uses :meth:`.get_feedback_with_most_points` to get the feedback with most points,
        and returns the ``grade``-attribute of that feedaback.

        :return: The grade or ``None``.
        """
        best = self.get_feedback_with_most_points()
        if best:
            return best.grade
        else:
            return None

    def _serialize_feedback(self, feedback):
        if feedback:
            return {'id': feedback.id,
                    'grade': feedback.grade,
                    'points': feedback.points,
                    'is_passing_grade': feedback.is_passing_grade,
                    'save_timestamp': feedback.save_timestamp}
        else:
            return None

    def _serialize_group(self, group):
        return {
            'id': group.id,
            'feedback': self._serialize_feedback(group.feedback),
            'status': group.get_status()
        }

    def serialize(self):
        return [self._serialize_group(group) for group in self]

class AggreatedRelatedStudentInfo(object):
    """
    Used by :class:`.GroupsGroupedByRelatedStudentAndAssignment` to stores all results for a
    single student on a period.
    """
    def __init__(self, user, assignments, relatedstudent=None):
        #: The Django user object for the student.
        self.user = user

        #: Dict of assignments where the key is the assignment-id, and the value is a :class:`.GroupList`.
        self.assignments = assignments

        #: The :class:`devilry.apps.core.models.RelatedStudent` for users that are related students.
        #: This is only available for the objects returned by
        #: :meth:`.GroupsGroupedByRelatedStudentAndAssignment.iter_relatedstudents_with_results`,
        #: and not for the objects returned by the ignored students iterators.
        self.relatedstudent = relatedstudent

    def iter_groups_by_assignment(self):
        """
        Returns an iterator over all :class:`.GroupList` objects for this student.
        Shortcut for ``self.assignments.itervalues()``.
        """
        return self.assignments.itervalues()

    def add_group(self, group):
        """
        Used by :class:`.GroupsGroupedByRelatedStudentAndAssignment` to add groups.
        """
        self.assignments[group.parentnode_id].append(group)

    def prettyprint(self):
        """
        Prettyprint for debugging.
        """
        print '{0}:'.format(self.user)
        for assignmentid, groups in self.assignments.iteritems():
            print '  - Assignment ID: {0}'.format(assignmentid)
            for group in groups:
                if group.feedback:
                    grade = '{0} (points:{1},passed:{2})'.format(group.feedback.grade,
                        group.feedback.points, group.feedback.is_passing_grade)
                else:
                    grade = None
                print '    - {0}: {1}'.format(group, grade)

    def __str__(self):
        """
        Returns a short representation of the object that should be useful when debugging.
        """
        return u'AggreatedRelatedStudentInfo(user={0}, assignmentcount={1}, grades={2!r})'.format(
            self.user.username,
            len(self.assignments),
            [str(gradelist.get_best_gradestring()) for gradelist in self.iter_groups_by_assignment()]
        )

    def _serialize_user(self, user):
        return {'id': user.id,
                'username': user.username,
                'email': user.email,
                'full_name': user.devilryuserprofile.full_name}

    def serialize(self):
        out = {'id': self.user.id, # NOTE: This is added to support stupid datamodel layers, like ExtJS, which does not support the ID of a record to be within an attribute
               'user': self._serialize_user(self.user),
               'groups_by_assignment': [],
               'relatedstudent': None}
        if self.relatedstudent:
            out['relatedstudent'] = {
                'id': self.relatedstudent.id,
                'tags': self.relatedstudent.tags,
                'candidate_id': self.relatedstudent.candidate_id}
        for assignmentid, grouplist in self.assignments.iteritems():
            out['groups_by_assignment'].append({'assignmentid': assignmentid,
                                                'grouplist': grouplist.serialize()})
        return out

class GroupsGroupedByRelatedStudentAndAssignment(object):
    """
    Provides an easy-to-use API for overviews over the results of all students
    in a period.
    """
    def __init__(self, period):
        """
        :param period: A :class:`devilry.apps.core.models.Period` object.
        """
        self.period = period
        self._collect_assignments()
        self._initialize_result()
        self._add_groups_to_result()

    def get_assignment_queryset(self):
        """
        Get the queryset used to fetch all assignments on the period.
        Override for custom ordering or if you need to optimize the query for
        your usecase (``select_related``, ``prefetch_related``, etc.)
        """
        return self.period.assignments.all().order_by('publishing_time')

    def get_relatedstudents_queryset(self):
        """
        Get the queryset used to fetch all relatedstudents on the period.
        Override if you need to optimize the query for your usecase
        (``select_related``, ``prefetch_related``, etc.)
        """
        return self.period.relatedstudent_set.all().select_related('user', 'user__devilryuserprofile')

    def get_groups_queryset(self):
        """
        Get the queryset used to fetch all groups on the period.
        Override if you need to optimize the query for your usecase
        (``select_related``, ``prefetch_related``, etc.)
        """
        groupqry = AssignmentGroup.objects.filter(parentnode__parentnode=self.period)
        groupqry = groupqry.select_related('parentnode', 'parentnode__parentnode', 'feedback')
        groupqry = groupqry.prefetch_related('candidates', 'candidates__student',
            'candidates__student__devilryuserprofile')
        return groupqry

    def _collect_assignments(self):
        self.assignments = []
        for assignment in self.get_assignment_queryset():
            self.assignments.append(assignment)

    def _create_assignmentsdict(self):
        dct = OrderedDict()
        for assignment in self.assignments:
            dct[assignment.id] = GroupList()
        return dct

    def _initialize_result(self):
        self.result = {}
        for relatedstudent in self.get_relatedstudents_queryset():
            self.result[relatedstudent.user_id] = AggreatedRelatedStudentInfo(
                user = relatedstudent.user,
                assignments = self._create_assignmentsdict(),
                relatedstudent=relatedstudent
            )

    def _create_or_add_ignoredgroup(self, ignoreddict, candidate):
        if not candidate.student_id in ignoreddict:
            ignoreddict[candidate.student_id] = AggreatedRelatedStudentInfo(
                user = candidate.student,
                assignments = self._create_assignmentsdict()
            )
        return ignoreddict[candidate.student_id]

    def _add_groups_to_result(self):
        groupqry = self.get_groups_queryset()
        self.ignored_students = {}
        self.ignored_students_with_results = {}
        for group in groupqry:
            for candidate in group.candidates.all():
                if candidate.student_id in self.result:
                    self.result[candidate.student_id].add_group(group)
                else:
                    self._create_or_add_ignoredgroup(self.ignored_students, candidate).add_group(group)
                    if group.feedback:
                        self._create_or_add_ignoredgroup(self.ignored_students_with_results, candidate).add_group(group)

    def iter_assignments(self):
        """
        Iterate over all the assignments, yielding Assignment-objects. The objects
        are iterated in the order returned by :meth:`get_assignment_queryset`.
        """
        return self.assignments.__iter__()

    def iter_relatedstudents_with_results(self):
        """
        Iterate over all relatedstudents, yielding a dict with the following attributes for each
        related student:

            user
                The Django user-object for the student.
            assignments
                An OrderedDict, ordered the same as :meth:`.iter_assignments`, where the key is
                the assignment-id, and the value is a list of AssignmentGroup-objects where the user
                is candidate. The list may have 0 or more groups, 0 if the user is not in any group
                on the assignment, and more than 1 if the user is in more than one group on the
                assignment.
        """
        return self.result.itervalues()

    def iter_students_that_is_candidate_but_not_in_related(self):
        """
        Iterate over the students that is candidate on one or more groups, but not registered as
        related students.
        """
        return self.ignored_students.itervalues()

    def iter_students_with_feedback_that_is_candidate_but_not_in_related(self):
        """
        Same as :meth:`.iter_students_that_is_candidate_but_not_in_related`, but it does not include
        the students that have no feedback.
        """
        return self.ignored_students_with_results.itervalues()

    def iter_students_with_no_feedback_that_is_candidate_but_not_in_related(self):
        """
        Iterate over everything returned by
        :meth:`.iter_students_that_is_candidate_but_not_in_related`
        except for the students returned by
        :meth:`.iter_students_with_feedback_that_is_candidate_but_not_in_related`
        """
        for userid, student in self.ignored_students.iteritems():
            if not userid in self.ignored_students_with_results:
                yield student

    def _serialize_assignment(self, basenode):
        return {'id': basenode.id,
                'short_name': basenode.short_name,
                'long_name': basenode.long_name}

    def serialize(self):
        """
        Serialize all the collected data as plain python objects.
        """
        out = {
            'relatedstudents':
                [r.serialize() for r in self.iter_relatedstudents_with_results()],
            'students_that_is_candidate_but_not_in_related':
                [r.serialize() for r in self.iter_students_that_is_candidate_but_not_in_related()],
            'students_with_feedback_that_is_candidate_but_not_in_related':
                [r.serialize() for r in self.iter_students_with_feedback_that_is_candidate_but_not_in_related()],
            'assignments':
                [self._serialize_assignment(a) for a in self.iter_assignments()]
        }
        return out<commit_msg>apps.utils.groups_groupedby_relatedstudent_and_assignment.get_feedback_with_most_points: Only check corrected.<commit_after>from devilry.utils import OrderedDict
from devilry.apps.core.models import AssignmentGroup

class GroupList(list):
    """
    Represents a list of :class:`devilry.apps.core.models.AssignmentGroup` objects,
    with utility functions for commonly needed actions. The list is ment to hold
    groups where the same student in candidate on a single assignment, and the
    utilities is ment to make it easier to work with the added complexity of
    supporting the same user in multiple groups on a single assignment.
    """
    def get_feedback_with_most_points(self):
        """
        Get the :class:`devilry.apps.core.models.StaticFeedback` with the most points in the list.
        """
        best = None
        for group in self:
            feedback = None
            if group.get_status() == 'corrected':
                feedback = group.feedback
            if best:
                if feedback and feedback.points > best.points:
                    best = group
            else:
                best = feedback
        return best

    def get_best_gradestring(self):
        """
        Uses :meth:`.get_feedback_with_most_points` to get the feedback with most points,
        and returns the ``grade``-attribute of that feedaback.

        :return: The grade or ``None``.
        """
        best = self.get_feedback_with_most_points()
        if best:
            return best.grade
        else:
            return None

    def _serialize_feedback(self, feedback):
        if feedback:
            return {'id': feedback.id,
                    'grade': feedback.grade,
                    'points': feedback.points,
                    'is_passing_grade': feedback.is_passing_grade,
                    'save_timestamp': feedback.save_timestamp}
        else:
            return None

    def _serialize_group(self, group):
        return {
            'id': group.id,
            'feedback': self._serialize_feedback(group.feedback),
            'status': group.get_status()
        }

    def serialize(self):
        return [self._serialize_group(group) for group in self]

class AggreatedRelatedStudentInfo(object):
    """
    Used by :class:`.GroupsGroupedByRelatedStudentAndAssignment` to stores all results for a
    single student on a period.
    """
    def __init__(self, user, assignments, relatedstudent=None):
        #: The Django user object for the student.
        self.user = user

        #: Dict of assignments where the key is the assignment-id, and the value is a :class:`.GroupList`.
        self.assignments = assignments

        #: The :class:`devilry.apps.core.models.RelatedStudent` for users that are related students.
        #: This is only available for the objects returned by
        #: :meth:`.GroupsGroupedByRelatedStudentAndAssignment.iter_relatedstudents_with_results`,
        #: and not for the objects returned by the ignored students iterators.
        self.relatedstudent = relatedstudent

    def iter_groups_by_assignment(self):
        """
        Returns an iterator over all :class:`.GroupList` objects for this student.
        Shortcut for ``self.assignments.itervalues()``.
        """
        return self.assignments.itervalues()

    def add_group(self, group):
        """
        Used by :class:`.GroupsGroupedByRelatedStudentAndAssignment` to add groups.
        """
        self.assignments[group.parentnode_id].append(group)

    def prettyprint(self):
        """
        Prettyprint for debugging.
        """
        print '{0}:'.format(self.user)
        for assignmentid, groups in self.assignments.iteritems():
            print '  - Assignment ID: {0}'.format(assignmentid)
            for group in groups:
                if group.feedback:
                    grade = '{0} (points:{1},passed:{2})'.format(group.feedback.grade,
                        group.feedback.points, group.feedback.is_passing_grade)
                else:
                    grade = None
                print '    - {0}: {1}'.format(group, grade)

    def __str__(self):
        """
        Returns a short representation of the object that should be useful when debugging.
        """
        return u'AggreatedRelatedStudentInfo(user={0}, assignmentcount={1}, grades={2!r})'.format(
            self.user.username,
            len(self.assignments),
            [str(gradelist.get_best_gradestring()) for gradelist in self.iter_groups_by_assignment()]
        )

    def _serialize_user(self, user):
        return {'id': user.id,
                'username': user.username,
                'email': user.email,
                'full_name': user.devilryuserprofile.full_name}

    def serialize(self):
        out = {'id': self.user.id, # NOTE: This is added to support stupid datamodel layers, like ExtJS, which does not support the ID of a record to be within an attribute
               'user': self._serialize_user(self.user),
               'groups_by_assignment': [],
               'relatedstudent': None}
        if self.relatedstudent:
            out['relatedstudent'] = {
                'id': self.relatedstudent.id,
                'tags': self.relatedstudent.tags,
                'candidate_id': self.relatedstudent.candidate_id}
        for assignmentid, grouplist in self.assignments.iteritems():
            out['groups_by_assignment'].append({'assignmentid': assignmentid,
                                                'grouplist': grouplist.serialize()})
        return out

class GroupsGroupedByRelatedStudentAndAssignment(object):
    """
    Provides an easy-to-use API for overviews over the results of all students
    in a period.
    """
    def __init__(self, period):
        """
        :param period: A :class:`devilry.apps.core.models.Period` object.
        """
        self.period = period
        self._collect_assignments()
        self._initialize_result()
        self._add_groups_to_result()

    def get_assignment_queryset(self):
        """
        Get the queryset used to fetch all assignments on the period.
        Override for custom ordering or if you need to optimize the query for
        your usecase (``select_related``, ``prefetch_related``, etc.)
        """
        return self.period.assignments.all().order_by('publishing_time')

    def get_relatedstudents_queryset(self):
        """
        Get the queryset used to fetch all relatedstudents on the period.
        Override if you need to optimize the query for your usecase
        (``select_related``, ``prefetch_related``, etc.)
        """
        return self.period.relatedstudent_set.all().select_related('user', 'user__devilryuserprofile')

    def get_groups_queryset(self):
        """
        Get the queryset used to fetch all groups on the period.
        Override if you need to optimize the query for your usecase
        (``select_related``, ``prefetch_related``, etc.)
        """
        groupqry = AssignmentGroup.objects.filter(parentnode__parentnode=self.period)
        groupqry = groupqry.select_related('parentnode', 'parentnode__parentnode', 'feedback')
        groupqry = groupqry.prefetch_related('candidates', 'candidates__student',
            'candidates__student__devilryuserprofile')
        return groupqry

    def _collect_assignments(self):
        self.assignments = []
        for assignment in self.get_assignment_queryset():
            self.assignments.append(assignment)

    def _create_assignmentsdict(self):
        dct = OrderedDict()
        for assignment in self.assignments:
            dct[assignment.id] = GroupList()
        return dct

    def _initialize_result(self):
        self.result = {}
        for relatedstudent in self.get_relatedstudents_queryset():
            self.result[relatedstudent.user_id] = AggreatedRelatedStudentInfo(
                user = relatedstudent.user,
                assignments = self._create_assignmentsdict(),
                relatedstudent=relatedstudent
            )

    def _create_or_add_ignoredgroup(self, ignoreddict, candidate):
        if not candidate.student_id in ignoreddict:
            ignoreddict[candidate.student_id] = AggreatedRelatedStudentInfo(
                user = candidate.student,
                assignments = self._create_assignmentsdict()
            )
        return ignoreddict[candidate.student_id]

    def _add_groups_to_result(self):
        groupqry = self.get_groups_queryset()
        self.ignored_students = {}
        self.ignored_students_with_results = {}
        for group in groupqry:
            for candidate in group.candidates.all():
                if candidate.student_id in self.result:
                    self.result[candidate.student_id].add_group(group)
                else:
                    self._create_or_add_ignoredgroup(self.ignored_students, candidate).add_group(group)
                    if group.feedback:
                        self._create_or_add_ignoredgroup(self.ignored_students_with_results, candidate).add_group(group)

    def iter_assignments(self):
        """
        Iterate over all the assignments, yielding Assignment-objects. The objects
        are iterated in the order returned by :meth:`get_assignment_queryset`.
        """
        return self.assignments.__iter__()

    def iter_relatedstudents_with_results(self):
        """
        Iterate over all relatedstudents, yielding a dict with the following attributes for each
        related student:

            user
                The Django user-object for the student.
            assignments
                An OrderedDict, ordered the same as :meth:`.iter_assignments`, where the key is
                the assignment-id, and the value is a list of AssignmentGroup-objects where the user
                is candidate. The list may have 0 or more groups, 0 if the user is not in any group
                on the assignment, and more than 1 if the user is in more than one group on the
                assignment.
        """
        return self.result.itervalues()

    def iter_students_that_is_candidate_but_not_in_related(self):
        """
        Iterate over the students that is candidate on one or more groups, but not registered as
        related students.
        """
        return self.ignored_students.itervalues()

    def iter_students_with_feedback_that_is_candidate_but_not_in_related(self):
        """
        Same as :meth:`.iter_students_that_is_candidate_but_not_in_related`, but it does not include
        the students that have no feedback.
        """
        return self.ignored_students_with_results.itervalues()

    def iter_students_with_no_feedback_that_is_candidate_but_not_in_related(self):
        """
        Iterate over everything returned by
        :meth:`.iter_students_that_is_candidate_but_not_in_related`
        except for the students returned by
        :meth:`.iter_students_with_feedback_that_is_candidate_but_not_in_related`
        """
        for userid, student in self.ignored_students.iteritems():
            if not userid in self.ignored_students_with_results:
                yield student

    def _serialize_assignment(self, basenode):
        return {'id': basenode.id,
                'short_name': basenode.short_name,
                'long_name': basenode.long_name}

    def serialize(self):
        """
        Serialize all the collected data as plain python objects.
        """
        out = {
            'relatedstudents':
                [r.serialize() for r in self.iter_relatedstudents_with_results()],
            'students_that_is_candidate_but_not_in_related':
                [r.serialize() for r in self.iter_students_that_is_candidate_but_not_in_related()],
            'students_with_feedback_that_is_candidate_but_not_in_related':
                [r.serialize() for r in self.iter_students_with_feedback_that_is_candidate_but_not_in_related()],
            'assignments':
                [self._serialize_assignment(a) for a in self.iter_assignments()]
        }
        return out<|endoftext|>