<commit_before>#__BEGIN_LICENSE__
# Copyright (c) 2015, United States Government, as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All rights reserved.
#
# The xGDS platform is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0.
#
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
#__END_LICENSE__

from django.contrib import admin

from xgds_core.models import *

admin.site.register(Constant)
admin.site.register(TimeZoneHistory)
admin.site.register(RelayEvent)
admin.site.register(RelayFile)
admin.site.register(ConditionHistory)
admin.site.register(State)
admin.site.register(ActiveFlight)
admin.site.register(Flight)
admin.site.register(Vehicle)

<commit_msg>added imported telemetry files to core<commit_after>#__BEGIN_LICENSE__
# Copyright (c) 2015, United States Government, as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All rights reserved.
#
# The xGDS platform is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0.
#
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
#__END_LICENSE__

from django.contrib import admin

from xgds_core.models import *

admin.site.register(Constant)
admin.site.register(TimeZoneHistory)
admin.site.register(RelayEvent)
admin.site.register(RelayFile)
admin.site.register(ConditionHistory)
admin.site.register(State)
admin.site.register(ActiveFlight)
admin.site.register(Flight)
admin.site.register(Vehicle)
admin.site.register(ImportedTelemetryFile)

<|endoftext|>


<commit_before>import re

from livestreamer.compat import urljoin
from livestreamer.plugin import Plugin
from livestreamer.plugin.api import http, validate
from livestreamer.stream import RTMPStream

RTMP_URL = "rtmp://{0}/app/{1}"
BALANCING_REQUEST_URI = "https://streamup-lancer.herokuapp.com/api/redirect/{0}"

_url_re = re.compile("http(s)?://(\w+\.)?streamup.com/(?P<channel>[^/?]+)")
_flashvars_re = re.compile("flashvars\.(?P<var>\w+)\s?=\s?'(?P<value>[^']+)';")
_swf_url_re = re.compile("swfobject.embedSWF\(\s*\"(?P<player_url>[^\"]+)\",")

_schema = validate.Schema(
    validate.union({
        "vars": validate.all(
            validate.transform(_flashvars_re.findall),
            validate.transform(dict),
            {
                "channel": validate.text,
            }
        ),
        "swf": validate.all(
            validate.transform(_swf_url_re.search),
            validate.get("player_url"),
            validate.endswith(".swf")
        )
    })
)

class StreamupCom(Plugin):
    @classmethod
    def can_handle_url(cls, url):
        return _url_re.match(url)

    def _get_streams(self):
        res = http.get(self.url, schema=_schema)
        if not res:
            return

        stream_ip = http.get(BALANCING_REQUEST_URI.format(res["vars"]["channel"])).text

        streams = {}
        streams["live"] = RTMPStream(self.session, {
            "rtmp": RTMP_URL.format(stream_ip, res["vars"]["channel"]),
            "pageUrl": self.url,
            "swfUrl": urljoin(self.url, res["swf"]),
            "live": True
        })
        return streams

__plugin__ = StreamupCom
<commit_msg>plugins.streamupcom: Check live status<commit_after>import re

from livestreamer.compat import urljoin
from livestreamer.plugin import Plugin
from livestreamer.plugin.api import http, validate
from livestreamer.stream import RTMPStream

RTMP_URL = "rtmp://{0}/app/{1}"
STATUS_REQUEST_URI = "https://lancer.streamup.com/api/channels/{0}"
BALANCING_REQUEST_URI = "https://streamup-lancer.herokuapp.com/api/redirect/{0}"

_url_re = re.compile("http(s)?://(\w+\.)?streamup.com/(?P<channel>[^/?]+)")
_flashvars_re = re.compile("flashvars\.(?P<var>\w+)\s?=\s?'(?P<value>[^']+)';")
_swf_url_re = re.compile("swfobject.embedSWF\(\s*\"(?P<player_url>[^\"]+)\",")

_schema = validate.Schema(
    validate.union({
        "vars": validate.all(
            validate.transform(_flashvars_re.findall),
            validate.transform(dict),
            {
                "channel": validate.text,
            }
        ),
        "swf": validate.all(
            validate.transform(_swf_url_re.search),
            validate.get("player_url"),
            validate.endswith(".swf")
        )
    })
)

class StreamupCom(Plugin):
    @classmethod
    def can_handle_url(cls, url):
        return _url_re.match(url)

    def _get_streams(self):
        res = http.get(self.url, schema=_schema)
        if not res:
            return
        channel_name = res["vars"]["channel"]
        swf_url = res["swf"]

        # Check if the stream is online
        res = http.get(STATUS_REQUEST_URI.format(channel_name), raise_for_status=False)
        if res.status_code == 404:
            return

        stream_ip = http.get(BALANCING_REQUEST_URI.format(channel_name)).text

        streams = {}
        streams["live"] = RTMPStream(self.session, {
            "rtmp": RTMP_URL.format(stream_ip, channel_name),
            "pageUrl": self.url,
            "swfUrl": urljoin(self.url, swf_url),
            "live": True
        })
        return streams

__plugin__ = StreamupCom
<|endoftext|>


<commit_before># coding=utf-8
# Copyright 2019 The Tensor2Tensor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tests for Transformer."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import numpy as np

from tensor2tensor.data_generators import problem_hparams
from tensor2tensor.models import transformer

import tensorflow as tf

BATCH_SIZE = 3
INPUT_LENGTH = 5
TARGET_LENGTH = 7
VOCAB_SIZE = 10

def get_model(hparams=None, mode=tf.estimator.ModeKeys.TRAIN,
              has_input=True, model_cls=transformer.Transformer):
  if hparams is None:
    hparams = transformer.transformer_tiny()
  hparams.hidden_size = 8
  hparams.filter_size = 32
  hparams.num_heads = 1
  hparams.layer_prepostprocess_dropout = 0.0

  p_hparams = problem_hparams.test_problem_hparams(VOCAB_SIZE,
                                                   VOCAB_SIZE,
                                                   hparams)
  if not has_input:
    del p_hparams.modality["inputs"]
  hparams.problem_hparams = p_hparams

  inputs = np.random.randint(
      VOCAB_SIZE, size=(BATCH_SIZE, INPUT_LENGTH, 1, 1))
  targets = np.random.randint(
      VOCAB_SIZE, size=(BATCH_SIZE, TARGET_LENGTH, 1, 1))
  features = {
      "targets": tf.constant(targets, dtype=tf.int32, name="targets"),
      "target_space_id": tf.constant(1, dtype=tf.int32)
  }
  if has_input:
    features["inputs"] = tf.constant(inputs, dtype=tf.int32, name="inputs")

  return model_cls(hparams, mode, p_hparams), features

class TransformerTest(tf.test.TestCase):

  def testTransformer(self):
    model, features = get_model(transformer.transformer_small())
    logits, _ = model(features)
    with self.test_session() as session:
      session.run(tf.global_variables_initializer())
      res = session.run(logits)
    self.assertEqual(res.shape, (BATCH_SIZE, TARGET_LENGTH, 1, 1, VOCAB_SIZE))

  def testTransformerRelative(self):
    model, features = get_model(transformer.transformer_relative_tiny())
    logits, _ = model(features)
    with self.test_session() as session:
      session.run(tf.global_variables_initializer())
      res = session.run(logits)
    self.assertEqual(res.shape, (BATCH_SIZE, TARGET_LENGTH, 1, 1, VOCAB_SIZE))

  def testSlowVsFast(self):
    model, features = get_model(transformer.transformer_small())

    decode_length = 3

    out_logits, _ = model(features)
    out_logits = tf.squeeze(out_logits, axis=[2, 3])
    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
        logits=tf.reshape(out_logits, [-1, VOCAB_SIZE]),
        labels=tf.reshape(features["targets"], [-1]))
    loss = tf.reduce_mean(loss)
    apply_grad = tf.train.AdamOptimizer(0.001).minimize(loss)

    with self.test_session():
      tf.global_variables_initializer().run()
      for _ in range(100):
        apply_grad.run()

    model.set_mode(tf.estimator.ModeKeys.PREDICT)

    with tf.variable_scope(tf.get_variable_scope(), reuse=True):
      greedy_result = model._slow_greedy_infer(
          features, decode_length)["outputs"]
      greedy_result = tf.squeeze(greedy_result, axis=[2, 3])

      fast_result = model._greedy_infer(features, decode_length)["outputs"]

    with self.test_session():
      greedy_res = greedy_result.eval()
      fast_res = fast_result.eval()

    self.assertEqual(fast_res.shape, (BATCH_SIZE, INPUT_LENGTH + decode_length))
    self.assertAllClose(greedy_res, fast_res)

  def testSlowVsFastNoInput(self):
    model, features = get_model(
        transformer.transformer_small(), has_input=False)

    decode_length = 3

    out_logits, _ = model(features)
    out_logits = tf.squeeze(out_logits, axis=[2, 3])
    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
        logits=tf.reshape(out_logits, [-1, VOCAB_SIZE]),
        labels=tf.reshape(features["targets"], [-1]))
    loss = tf.reduce_mean(loss)
    apply_grad = tf.train.AdamOptimizer(0.001).minimize(loss)

    with self.test_session():
      tf.global_variables_initializer().run()
      for _ in range(100):
        apply_grad.run()

    model.set_mode(tf.estimator.ModeKeys.PREDICT)

    with tf.variable_scope(tf.get_variable_scope(), reuse=True):
      slow_result = model._slow_greedy_infer(
          features, decode_length)["outputs"]
      slow_result = tf.squeeze(slow_result, axis=[2, 3])

      fast_result = model._greedy_infer(features, decode_length)["outputs"]

    with self.test_session():
      slow_res = slow_result.eval()
      fast_res = fast_result.eval()

    self.assertEqual(slow_res.shape, (BATCH_SIZE, decode_length))
    self.assertAllClose(slow_res, fast_res)

  def testBeamDecodeWithRelativeAttention(self):
    decode_length = 2
    model, features = get_model(transformer.transformer_relative_tiny())
    model.set_mode(tf.estimator.ModeKeys.PREDICT)

    beam_result = model._beam_decode(
        features, decode_length, beam_size=4, top_beams=1,
        alpha=1.0)["outputs"]

    with self.test_session():
      tf.global_variables_initializer().run()
      beam_result.eval()

    # TODO(petershaw): This test is flaky because the decode may hit EOS before
    # getting to the expected length.
    # self.assertEqual(beam_res.shape,
    #                  (BATCH_SIZE, INPUT_LENGTH + decode_length))

  def testBeamVsFast(self):
    model, features = get_model(transformer.transformer_small())

    decode_length = 2

    out_logits, _ = model(features)
    out_logits = tf.squeeze(out_logits, axis=[2, 3])
    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
        logits=tf.reshape(out_logits, [-1, VOCAB_SIZE]),
        labels=tf.reshape(features["targets"], [-1]))
    loss = tf.reduce_mean(loss)
    apply_grad = tf.train.AdamOptimizer(0.001).minimize(loss)

    with self.test_session():
      tf.global_variables_initializer().run()
      for _ in range(100):
        apply_grad.run()

    model.set_mode(tf.estimator.ModeKeys.PREDICT)

    with tf.variable_scope(tf.get_variable_scope(), reuse=True):
      beam_result = model._beam_decode_slow(
          features,
          decode_length,
          beam_size=4,
          top_beams=1,
          alpha=1.0)["outputs"]

      fast_result = model._beam_decode(
          features,
          decode_length,
          beam_size=4,
          top_beams=1,
          alpha=1.0)["outputs"]

    with self.test_session():
      beam_res = beam_result.eval()
      fast_res = fast_result.eval()

    self.assertAllClose(beam_res, fast_res)

  def testTransformerWithoutProblem(self):
    hparams = transformer.transformer_test()

    embedded_inputs = np.random.random_sample(
        (BATCH_SIZE, INPUT_LENGTH, 1, hparams.hidden_size))
    embedded_targets = np.random.random_sample(
        (BATCH_SIZE, TARGET_LENGTH, 1, hparams.hidden_size))

    transformed_features = {
        "inputs": tf.constant(embedded_inputs, dtype=tf.float32),
        "targets": tf.constant(embedded_targets, dtype=tf.float32)
    }

    model = transformer.Transformer(hparams)
    body_out, _ = model(transformed_features)

    self.assertAllEqual(
        body_out.get_shape().as_list(),
        [BATCH_SIZE, TARGET_LENGTH, 1, hparams.hidden_size])

  def testTransformerWithEncoderDecoderAttentionLoss(self):
    model, features = get_model(
        transformer.transformer_supervised_attention())
    expected_attention_weights = np.random.random_sample(
        size=(BATCH_SIZE, TARGET_LENGTH, INPUT_LENGTH))
    features["expected_attentions"] = tf.constant(
        expected_attention_weights, dtype=tf.float32)
    _, extra_loss = model(features)
    with self.test_session() as session:
      session.run(tf.global_variables_initializer())
      res = session.run(extra_loss["attention_loss"])
    self.assertEqual(res.shape, ())

  def _create_greedy_infer_model(self):
    """Creates model for greedy inference testing.

    Returns:
      model: A t2t model.
      features: An map of string to tensor.
    """
    model, features = get_model(transformer.transformer_small())

    out_logits, _ = model(features)
    out_logits = tf.squeeze(out_logits, axis=[2, 3])
    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
        logits=tf.reshape(out_logits, [-1, VOCAB_SIZE]),
        labels=tf.reshape(features["targets"], [-1]))
    loss = tf.reduce_mean(loss)
    apply_grad = tf.train.AdamOptimizer(0.001).minimize(loss)

    with self.test_session():
      tf.global_variables_initializer().run()
      for _ in range(100):
        apply_grad.run()

    model.set_mode(tf.estimator.ModeKeys.PREDICT)

    return model, features

  def testGreedySlowTPUVsNonTPU(self):
    decode_length = 3

    model, features = self._create_greedy_infer_model()

    with tf.variable_scope(tf.get_variable_scope(), reuse=True):
      slow_result_non_tpu = model._slow_greedy_infer(
          features, decode_length)["outputs"]
      slow_result_non_tpu = tf.squeeze(slow_result_non_tpu, axis=[2, 3])

      slow_result_tpu = model._slow_greedy_infer_tpu(
          features, decode_length)["outputs"]
      slow_result_tpu = tf.squeeze(slow_result_tpu, axis=[2, 3])

    with self.test_session():
      slow_non_tpu_res = slow_result_non_tpu.eval()
      slow_tpu_res = slow_result_tpu.eval()

    self.assertEqual(slow_tpu_res.shape,
                     (BATCH_SIZE, INPUT_LENGTH + decode_length))
    self.assertAllClose(slow_tpu_res, slow_non_tpu_res)

  def testGreedyFastTPUVsNonTPU(self):
    decode_length = 3

    model, features = self._create_greedy_infer_model()

    with tf.variable_scope(tf.get_variable_scope(), reuse=True):
      fast_result_non_tpu = model._greedy_infer(
          features, decode_length, use_tpu=False)["outputs"]

      fast_result_tpu = model._greedy_infer(
          features, decode_length, use_tpu=True)["outputs"]

    with self.test_session():
      fast_non_tpu_res = fast_result_non_tpu.eval()
      fast_tpu_res = fast_result_tpu.eval()

    self.assertEqual(fast_tpu_res.shape,
                     (BATCH_SIZE, INPUT_LENGTH + decode_length))
    self.assertAllClose(fast_tpu_res, fast_non_tpu_res)

  def testGreedyTPUSlowVsFast(self):
    decode_length = 3

    model, features = self._create_greedy_infer_model()

    with tf.variable_scope(tf.get_variable_scope(), reuse=True):
      slow_result = model._slow_greedy_infer_tpu(
          features, decode_length)["outputs"]
      slow_result = tf.squeeze(slow_result, axis=[2, 3])

      fast_result = model._greedy_infer(
          features, decode_length, use_tpu=True)["outputs"]

    with self.test_session():
      slow_res = slow_result.eval()
      fast_res = fast_result.eval()

    self.assertEqual(fast_res.shape,
                     (BATCH_SIZE, INPUT_LENGTH + decode_length))
    self.assertAllClose(fast_res, slow_res)

class TransformerScorerTest(tf.test.TestCase):

  def testReturnsScores(self):
    model, features = get_model(
        mode=tf.estimator.ModeKeys.PREDICT,
        model_cls=transformer.TransformerScorer)
    infer_out = model.infer(features)
    self.assertTrue("outputs" in infer_out)
    self.assertTrue("scores" in infer_out)

    with self.test_session() as session:
      session.run(tf.global_variables_initializer())
      infer_out = session.run(infer_out)
      self.assertEqual((BATCH_SIZE,), infer_out["scores"].shape)
      self.assertEqual((BATCH_SIZE, TARGET_LENGTH), infer_out["outputs"].shape)

  def testVarNames(self):
    with tf.Graph().as_default():
      model, features = get_model(
          mode=tf.estimator.ModeKeys.PREDICT,
          model_cls=transformer.TransformerScorer)
      _ = model.infer(features)
      scorer_vars = [v.name for v in tf.global_variables()]

    with tf.Graph().as_default():
      model, features = get_model(
          mode=tf.estimator.ModeKeys.EVAL,
          model_cls=transformer.TransformerScorer)
      _ = model(features)
      scorer_eval_vars = [v.name for v in tf.global_variables()]

    with tf.Graph().as_default():
      model, features = get_model(
          mode=tf.estimator.ModeKeys.EVAL,
          model_cls=transformer.Transformer)
      _ = model(features)
      transformer_vars = [v.name for v in tf.global_variables()]

    self.assertEqual(sorted(scorer_vars), sorted(transformer_vars))
    self.assertEqual(sorted(scorer_eval_vars), sorted(transformer_vars))

if __name__ == "__main__":
  tf.test.main()
<commit_msg>Add unit test for ASR input, and Area Attention.<commit_after># coding=utf-8
# Copyright 2019 The Tensor2Tensor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tests for Transformer."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import numpy as np

from tensor2tensor.data_generators import librispeech
from tensor2tensor.data_generators import problem_hparams
from tensor2tensor.models import transformer

import tensorflow as tf

BATCH_SIZE = 3
INPUT_LENGTH = 5
TARGET_LENGTH = 7
VOCAB_SIZE = 10

def get_model(hparams=None, mode=tf.estimator.ModeKeys.TRAIN,
              has_input=True, model_cls=transformer.Transformer):
  if hparams is None:
    hparams = transformer.transformer_tiny()
  hparams.hidden_size = 8
  hparams.filter_size = 32
  hparams.num_heads = 1
  hparams.layer_prepostprocess_dropout = 0.0

  if hparams.get("problem_hparams", None) is None:
    p_hparams = problem_hparams.test_problem_hparams(VOCAB_SIZE,
                                                     VOCAB_SIZE,
                                                     hparams)
  if not has_input:
    del p_hparams.modality["inputs"]
  hparams.problem_hparams = p_hparams

  inputs = np.random.randint(
      VOCAB_SIZE, size=(BATCH_SIZE, INPUT_LENGTH, 1, 1))
  targets = np.random.randint(
      VOCAB_SIZE, size=(BATCH_SIZE, TARGET_LENGTH, 1, 1))
  features = {
      "targets": tf.constant(targets, dtype=tf.int32, name="targets"),
      "target_space_id": tf.constant(1, dtype=tf.int32)
  }
  if has_input:
    features["inputs"] = tf.constant(inputs, dtype=tf.int32, name="inputs")

  return model_cls(hparams, mode, p_hparams), features

class TransformerTest(tf.test.TestCase):

  def testTransformer(self):
    model, features = get_model(transformer.transformer_small())
    logits, _ = model(features)
    with self.test_session() as session:
      session.run(tf.global_variables_initializer())
      res = session.run(logits)
    self.assertEqual(res.shape, (BATCH_SIZE, TARGET_LENGTH, 1, 1, VOCAB_SIZE))

  def testTransformerLibrispeech(self, params=None):
    model_hparams = transformer.transformer_small()
    if params is not None:  # Add or Set any provided HParams
      assert isinstance(params, dict)
      for param_name in params:
        if hasattr(model_hparams, param_name):
          model_hparams.set_hparam(param_name, params[param_name])
        else:
          model_hparams.add_hparam(param_name, params[param_name])
    problem = librispeech.Librispeech()
    model_hparams.problem_hparams = problem.get_hparams(model_hparams)
    model_hparams._problem_hparams = model_hparams.problem_hparams
    model, features = get_model(model_hparams)
    model._problem_hparams.modality = {"inputs": "speech_recognition",
                                       "targets": "symbol"}
    features["inputs"] = np.random.rand(
        BATCH_SIZE, INPUT_LENGTH, 80, 3).astype("float32")  # modify for speech

    logits, _ = model(features)
    with self.test_session() as session:
      session.run(tf.global_variables_initializer())
      res = session.run(logits)
    self.assertEqual(res.shape, (BATCH_SIZE, TARGET_LENGTH, 1, 1, VOCAB_SIZE))

  def testTransformerLibrispeechWithAreaAttention(self):
    self.testTransformerLibrispeech({"max_area_width": 2,
                                     "num_area_layers": 1,
                                     "area_key_mode": "mean",
                                     "area_value_mode": "sum"})

  def testTransformerRelative(self):
    model, features = get_model(transformer.transformer_relative_tiny())
    logits, _ = model(features)
    with self.test_session() as session:
      session.run(tf.global_variables_initializer())
      res = session.run(logits)
    self.assertEqual(res.shape, (BATCH_SIZE, TARGET_LENGTH, 1, 1, VOCAB_SIZE))

  def testSlowVsFast(self):
    model, features = get_model(transformer.transformer_small())

    decode_length = 3

    out_logits, _ = model(features)
    out_logits = tf.squeeze(out_logits, axis=[2, 3])
    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
        logits=tf.reshape(out_logits, [-1, VOCAB_SIZE]),
        labels=tf.reshape(features["targets"], [-1]))
    loss = tf.reduce_mean(loss)
    apply_grad = tf.train.AdamOptimizer(0.001).minimize(loss)

    with self.test_session():
      tf.global_variables_initializer().run()
      for _ in range(100):
        apply_grad.run()

    model.set_mode(tf.estimator.ModeKeys.PREDICT)

    with tf.variable_scope(tf.get_variable_scope(), reuse=True):
      greedy_result = model._slow_greedy_infer(
          features, decode_length)["outputs"]
      greedy_result = tf.squeeze(greedy_result, axis=[2, 3])

      fast_result = model._greedy_infer(features, decode_length)["outputs"]

    with self.test_session():
      greedy_res = greedy_result.eval()
      fast_res = fast_result.eval()

    self.assertEqual(fast_res.shape, (BATCH_SIZE, INPUT_LENGTH + decode_length))
    self.assertAllClose(greedy_res, fast_res)

  def testSlowVsFastNoInput(self):
    model, features = get_model(
        transformer.transformer_small(), has_input=False)

    decode_length = 3

    out_logits, _ = model(features)
    out_logits = tf.squeeze(out_logits, axis=[2, 3])
    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
        logits=tf.reshape(out_logits, [-1, VOCAB_SIZE]),
        labels=tf.reshape(features["targets"], [-1]))
    loss = tf.reduce_mean(loss)
    apply_grad = tf.train.AdamOptimizer(0.001).minimize(loss)

    with self.test_session():
      tf.global_variables_initializer().run()
      for _ in range(100):
        apply_grad.run()

    model.set_mode(tf.estimator.ModeKeys.PREDICT)

    with tf.variable_scope(tf.get_variable_scope(), reuse=True):
      slow_result = model._slow_greedy_infer(
          features, decode_length)["outputs"]
      slow_result = tf.squeeze(slow_result, axis=[2, 3])

      fast_result = model._greedy_infer(features, decode_length)["outputs"]

    with self.test_session():
      slow_res = slow_result.eval()
      fast_res = fast_result.eval()

    self.assertEqual(slow_res.shape, (BATCH_SIZE, decode_length))
    self.assertAllClose(slow_res, fast_res)

  def testBeamDecodeWithRelativeAttention(self):
    decode_length = 2
    model, features = get_model(transformer.transformer_relative_tiny())
    model.set_mode(tf.estimator.ModeKeys.PREDICT)

    beam_result = model._beam_decode(
        features, decode_length, beam_size=4, top_beams=1,
        alpha=1.0)["outputs"]

    with self.test_session():
      tf.global_variables_initializer().run()
      beam_result.eval()

    # TODO(petershaw): This test is flaky because the decode may hit EOS before
    # getting to the expected length.
    # self.assertEqual(beam_res.shape,
    #                  (BATCH_SIZE, INPUT_LENGTH + decode_length))

  def testBeamVsFast(self):
    model, features = get_model(transformer.transformer_small())

    decode_length = 2

    out_logits, _ = model(features)
    out_logits = tf.squeeze(out_logits, axis=[2, 3])
    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
        logits=tf.reshape(out_logits, [-1, VOCAB_SIZE]),
        labels=tf.reshape(features["targets"], [-1]))
    loss = tf.reduce_mean(loss)
    apply_grad = tf.train.AdamOptimizer(0.001).minimize(loss)

    with self.test_session():
      tf.global_variables_initializer().run()
      for _ in range(100):
        apply_grad.run()

    model.set_mode(tf.estimator.ModeKeys.PREDICT)

    with tf.variable_scope(tf.get_variable_scope(), reuse=True):
      beam_result = model._beam_decode_slow(
          features,
          decode_length,
          beam_size=4,
          top_beams=1,
          alpha=1.0)["outputs"]

      fast_result = model._beam_decode(
          features,
          decode_length,
          beam_size=4,
          top_beams=1,
          alpha=1.0)["outputs"]

    with self.test_session():
      beam_res = beam_result.eval()
      fast_res = fast_result.eval()

    self.assertAllClose(beam_res, fast_res)

  def testTransformerWithoutProblem(self):
    hparams = transformer.transformer_test()

    embedded_inputs = np.random.random_sample(
        (BATCH_SIZE, INPUT_LENGTH, 1, hparams.hidden_size))
    embedded_targets = np.random.random_sample(
        (BATCH_SIZE, TARGET_LENGTH, 1, hparams.hidden_size))

    transformed_features = {
        "inputs": tf.constant(embedded_inputs, dtype=tf.float32),
        "targets": tf.constant(embedded_targets, dtype=tf.float32)
    }

    model = transformer.Transformer(hparams)
    body_out, _ = model(transformed_features)

    self.assertAllEqual(
        body_out.get_shape().as_list(),
        [BATCH_SIZE, TARGET_LENGTH, 1, hparams.hidden_size])

  def testTransformerWithEncoderDecoderAttentionLoss(self):
    model, features = get_model(
        transformer.transformer_supervised_attention())
    expected_attention_weights = np.random.random_sample(
        size=(BATCH_SIZE, TARGET_LENGTH, INPUT_LENGTH))
    features["expected_attentions"] = tf.constant(
        expected_attention_weights, dtype=tf.float32)
    _, extra_loss = model(features)
    with self.test_session() as session:
      session.run(tf.global_variables_initializer())
      res = session.run(extra_loss["attention_loss"])
    self.assertEqual(res.shape, ())

  def _create_greedy_infer_model(self):
    """Creates model for greedy inference testing.

    Returns:
      model: A t2t model.
      features: An map of string to tensor.
    """
    model, features = get_model(transformer.transformer_small())

    out_logits, _ = model(features)
    out_logits = tf.squeeze(out_logits, axis=[2, 3])
    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
        logits=tf.reshape(out_logits, [-1, VOCAB_SIZE]),
        labels=tf.reshape(features["targets"], [-1]))
    loss = tf.reduce_mean(loss)
    apply_grad = tf.train.AdamOptimizer(0.001).minimize(loss)

    with self.test_session():
      tf.global_variables_initializer().run()
      for _ in range(100):
        apply_grad.run()

    model.set_mode(tf.estimator.ModeKeys.PREDICT)

    return model, features

  def testGreedySlowTPUVsNonTPU(self):
    decode_length = 3

    model, features = self._create_greedy_infer_model()

    with tf.variable_scope(tf.get_variable_scope(), reuse=True):
      slow_result_non_tpu = model._slow_greedy_infer(
          features, decode_length)["outputs"]
      slow_result_non_tpu = tf.squeeze(slow_result_non_tpu, axis=[2, 3])

      slow_result_tpu = model._slow_greedy_infer_tpu(
          features, decode_length)["outputs"]
      slow_result_tpu = tf.squeeze(slow_result_tpu, axis=[2, 3])

    with self.test_session():
      slow_non_tpu_res = slow_result_non_tpu.eval()
      slow_tpu_res = slow_result_tpu.eval()

    self.assertEqual(slow_tpu_res.shape,
                     (BATCH_SIZE, INPUT_LENGTH + decode_length))
    self.assertAllClose(slow_tpu_res, slow_non_tpu_res)

  def testGreedyFastTPUVsNonTPU(self):
    decode_length = 3

    model, features = self._create_greedy_infer_model()

    with tf.variable_scope(tf.get_variable_scope(), reuse=True):
      fast_result_non_tpu = model._greedy_infer(
          features, decode_length, use_tpu=False)["outputs"]

      fast_result_tpu = model._greedy_infer(
          features, decode_length, use_tpu=True)["outputs"]

    with self.test_session():
      fast_non_tpu_res = fast_result_non_tpu.eval()
      fast_tpu_res = fast_result_tpu.eval()

    self.assertEqual(fast_tpu_res.shape,
                     (BATCH_SIZE, INPUT_LENGTH + decode_length))
    self.assertAllClose(fast_tpu_res, fast_non_tpu_res)

  def testGreedyTPUSlowVsFast(self):
    decode_length = 3

    model, features = self._create_greedy_infer_model()

    with tf.variable_scope(tf.get_variable_scope(), reuse=True):
      slow_result = model._slow_greedy_infer_tpu(
          features, decode_length)["outputs"]
      slow_result = tf.squeeze(slow_result, axis=[2, 3])

      fast_result = model._greedy_infer(
          features, decode_length, use_tpu=True)["outputs"]

    with self.test_session():
      slow_res = slow_result.eval()
      fast_res = fast_result.eval()

    self.assertEqual(fast_res.shape,
                     (BATCH_SIZE, INPUT_LENGTH + decode_length))
    self.assertAllClose(fast_res, slow_res)

class TransformerScorerTest(tf.test.TestCase):

  def testReturnsScores(self):
    model, features = get_model(
        mode=tf.estimator.ModeKeys.PREDICT,
        model_cls=transformer.TransformerScorer)
    infer_out = model.infer(features)
    self.assertTrue("outputs" in infer_out)
    self.assertTrue("scores" in infer_out)

    with self.test_session() as session:
      session.run(tf.global_variables_initializer())
      infer_out = session.run(infer_out)
      self.assertEqual((BATCH_SIZE,), infer_out["scores"].shape)
      self.assertEqual((BATCH_SIZE, TARGET_LENGTH), infer_out["outputs"].shape)

  def testVarNames(self):
    with tf.Graph().as_default():
      model, features = get_model(
          mode=tf.estimator.ModeKeys.PREDICT,
          model_cls=transformer.TransformerScorer)
      _ = model.infer(features)
      scorer_vars = [v.name for v in tf.global_variables()]

    with tf.Graph().as_default():
      model, features = get_model(
          mode=tf.estimator.ModeKeys.EVAL,
          model_cls=transformer.TransformerScorer)
      _ = model(features)
      scorer_eval_vars = [v.name for v in tf.global_variables()]

    with tf.Graph().as_default():
      model, features = get_model(
          mode=tf.estimator.ModeKeys.EVAL,
          model_cls=transformer.Transformer)
      _ = model(features)
      transformer_vars = [v.name for v in tf.global_variables()]

    self.assertEqual(sorted(scorer_vars), sorted(transformer_vars))
    self.assertEqual(sorted(scorer_eval_vars), sorted(transformer_vars))

if __name__ == "__main__":
  tf.test.main()
<|endoftext|>


<commit_before># Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Basic TensorBoard functional tests using WebDriver."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import subprocess
import unittest
import tempfile
from selenium.webdriver.common import by
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.support import wait
from testing.web import webtest

from tensorboard.plugins.scalar import scalars_demo
from tensorboard.plugins.audio import audio_demo

class BasicTest(unittest.TestCase):
  """Tests that the basic chrome is displayed when there is no data."""

  @classmethod
  def setUpClass(cls):
    src_dir = os.environ["TEST_SRCDIR"]
    binary = os.path.join(src_dir,
                          "org_tensorflow_tensorboard/tensorboard/tensorboard")
    cls.logdir = tempfile.mkdtemp(prefix='core_test_%s_logdir_' % cls.__name__)
    cls.setUpData()
    cls.process = subprocess.Popen(
        [binary, "--port", "8000", "--logdir", cls.logdir])

  @classmethod
  def setUpData(cls):
    # Overridden by DashboardsTest.
    pass

  @classmethod
  def tearDownClass(cls):
    cls.process.kill()
    cls.process.wait()

  def setUp(self):
    self.driver = webtest.new_webdriver_session()
    self.driver.get("http://localhost:8000")
    self.wait = wait.WebDriverWait(self.driver, 2)

  def tearDown(self):
    try:
      self.driver.quit()
    finally:
      self.driver = None
      self.wait = None

  def testToolbarTitleDisplays(self):
    self.wait.until(
        expected_conditions.text_to_be_present_in_element((
            by.By.CLASS_NAME, "toolbar-title"), "TensorBoard"))

  def testLogdirDisplays(self):
    self.wait.until(
      expected_conditions.text_to_be_present_in_element((
        by.By.ID, "logdir"), self.logdir))

class DashboardsTest(BasicTest):
  """Tests basic behavior when there is some data in TensorBoard.

  This extends `BasicTest`, so it inherits its methods to test that the
  basic chrome is displayed. We also check that we can navigate around
  the various dashboards.
  """

  @classmethod
  def setUpData(cls):
    scalars_demo.run_all(cls.logdir, verbose=False)
    audio_demo.run_all(cls.logdir, verbose=False)

  def testDashboardSelection(self):
    """Test that we can navigate among the different dashboards."""
    selectors = {
      "scalars_tab": "paper-tab[data-dashboard=scalars]",
      "audio_tab": "paper-tab[data-dashboard=audio]",
      "graphs_tab": "paper-tab[data-dashboard=graphs]",
      "inactive_dropdown": "paper-dropdown-menu[label*=Inactive]",
      "images_menu_item": "paper-item[data-dashboard=images]",
      "reload_button": "paper-icon-button#reload-button",
    }
    elements = {}
    for (name, selector) in selectors.items():
      locator = (by.By.CSS_SELECTOR, selector)
      self.wait.until(expected_conditions.presence_of_element_located(locator))
      elements[name] = self.driver.find_element_by_css_selector(selector)

    # The implementation of paper-* components doesn't seem to play nice
    # with Selenium's `element.is_selected()` and `element.is_enabled()`
    # methods. Instead, we check the appropriate WAI-ARIA attributes.
    # (Also, though the docs for `get_attribute` say that the string
    # `"false"` is returned as `False`, this appears to be the case
    # _sometimes_ but not _always_, so we should take special care to
    # handle that.)
    def is_selected(element):
      attribute = element.get_attribute("aria-selected")
      return attribute and attribute != "false"

    def is_enabled(element):
      attribute = element.get_attribute("aria-disabled")
      is_disabled = attribute and attribute != "false"
      return not is_disabled

    def assert_selected_dashboard(polymer_component_name):
      expected = {polymer_component_name}
      actual = {
        container.find_element_by_css_selector("*").tag_name  # first child
        for container
        in self.driver.find_elements_by_css_selector(".dashboard-container")
        if container.is_displayed()
      }
      self.assertEqual(expected, actual)

    # The scalar and audio dashboards should be active, and the scalar
    # dashboard should be selected by default. The images menu item
    # should not be visible, as it's within the drop-down menu.
    self.assertTrue(elements["scalars_tab"].is_displayed())
    self.assertTrue(elements["audio_tab"].is_displayed())
    self.assertTrue(elements["graphs_tab"].is_displayed())
    self.assertTrue(is_selected(elements["scalars_tab"]))
    self.assertFalse(is_selected(elements["audio_tab"]))
    self.assertFalse(elements["images_menu_item"].is_displayed())
    self.assertFalse(is_selected(elements["images_menu_item"]))
    assert_selected_dashboard("tf-scalar-dashboard")

    # While we're on the scalar dashboard, we should be allowed to
    # reload the data.
    self.assertTrue(is_enabled(elements["reload_button"]))

    # We should be able to activate the audio dashboard.
    elements["audio_tab"].click()
    self.assertFalse(is_selected(elements["scalars_tab"]))
    self.assertTrue(is_selected(elements["audio_tab"]))
    self.assertFalse(is_selected(elements["graphs_tab"]))
    self.assertFalse(is_selected(elements["images_menu_item"]))
    assert_selected_dashboard("tf-audio-dashboard")
    self.assertTrue(is_enabled(elements["reload_button"]))

    # We should then be able to open the dropdown and navigate to the
    # image dashboard. (We have to wait until it's visible because of the
    # dropdown menu's animations.)
    elements["inactive_dropdown"].click()
    self.wait.until(
      expected_conditions.visibility_of(elements["images_menu_item"]))
    self.assertTrue(elements["images_menu_item"].is_displayed())
    elements["images_menu_item"].click()
    self.assertFalse(is_selected(elements["scalars_tab"]))
    self.assertFalse(is_selected(elements["audio_tab"]))
    self.assertFalse(is_selected(elements["graphs_tab"]))
    self.assertTrue(is_selected(elements["images_menu_item"]))
    assert_selected_dashboard("tf-image-dashboard")
    self.assertTrue(is_enabled(elements["reload_button"]))

    # Next, we should be able to navigate back to an active dashboard.
    # If we choose the graphs dashboard, the reload feature should be
    # disabled.
    elements["graphs_tab"].click()
    self.assertFalse(elements["images_menu_item"].is_displayed())
    self.assertFalse(is_selected(elements["scalars_tab"]))
    self.assertFalse(is_selected(elements["audio_tab"]))
    self.assertTrue(is_selected(elements["graphs_tab"]))
    self.assertFalse(is_selected(elements["images_menu_item"]))
    assert_selected_dashboard("tf-graph-dashboard")
    self.assertFalse(is_enabled(elements["reload_button"]))

    # Finally, we should be able to navigate back to the scalar dashboard.
    elements["scalars_tab"].click()
    self.assertTrue(is_selected(elements["scalars_tab"]))
    self.assertFalse(is_selected(elements["audio_tab"]))
    self.assertFalse(is_selected(elements["graphs_tab"]))
    self.assertFalse(is_selected(elements["images_menu_item"]))
    assert_selected_dashboard("tf-scalar-dashboard")
    self.assertTrue(is_enabled(elements["reload_button"]))

if __name__ == "__main__":
  unittest.main()
<commit_msg>Attempt to make Travis tests less flaky (#215)<commit_after># Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Basic TensorBoard functional tests using WebDriver."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import collections
import os
import subprocess
import unittest
import tempfile
from selenium.webdriver.common import by
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.support import wait
from testing.web import webtest

from tensorboard.plugins.scalar import scalars_demo
from tensorboard.plugins.audio import audio_demo

_BASE_PORT = 8000
_PORT_OFFSETS = collections.defaultdict(lambda: len(_PORT_OFFSETS))

class BasicTest(unittest.TestCase):
  """Tests that the basic chrome is displayed when there is no data."""

  @classmethod
  def setUpClass(cls):
    src_dir = os.environ["TEST_SRCDIR"]
    binary = os.path.join(src_dir,
                          "org_tensorflow_tensorboard/tensorboard/tensorboard")
    cls.logdir = tempfile.mkdtemp(prefix='core_test_%s_logdir_' % cls.__name__)
    cls.setUpData()
    cls.port = _BASE_PORT + _PORT_OFFSETS[cls]
    cls.process = subprocess.Popen(
        [binary, "--port", str(cls.port), "--logdir", cls.logdir])

  @classmethod
  def setUpData(cls):
    # Overridden by DashboardsTest.
    pass

  @classmethod
  def tearDownClass(cls):
    cls.process.kill()
    cls.process.wait()

  def setUp(self):
    self.driver = webtest.new_webdriver_session()
    self.driver.get("http://localhost:%s" % self.port)
    self.wait = wait.WebDriverWait(self.driver, 2)

  def tearDown(self):
    try:
      self.driver.quit()
    finally:
      self.driver = None
      self.wait = None

  def testToolbarTitleDisplays(self):
    self.wait.until(
        expected_conditions.text_to_be_present_in_element((
            by.By.CLASS_NAME, "toolbar-title"), "TensorBoard"))

  def testLogdirDisplays(self):
    self.wait.until(
      expected_conditions.text_to_be_present_in_element((
        by.By.ID, "logdir"), self.logdir))

class DashboardsTest(BasicTest):
  """Tests basic behavior when there is some data in TensorBoard.

  This extends `BasicTest`, so it inherits its methods to test that the
  basic chrome is displayed. We also check that we can navigate around
  the various dashboards.
  """

  @classmethod
  def setUpData(cls):
    scalars_demo.run_all(cls.logdir, verbose=False)
    audio_demo.run_all(cls.logdir, verbose=False)

  def testDashboardSelection(self):
    """Test that we can navigate among the different dashboards."""
    selectors = {
      "scalars_tab": "paper-tab[data-dashboard=scalars]",
      "audio_tab": "paper-tab[data-dashboard=audio]",
      "graphs_tab": "paper-tab[data-dashboard=graphs]",
      "inactive_dropdown": "paper-dropdown-menu[label*=Inactive]",
      "images_menu_item": "paper-item[data-dashboard=images]",
      "reload_button": "paper-icon-button#reload-button",
    }
    elements = {}
    for (name, selector) in selectors.items():
      locator = (by.By.CSS_SELECTOR, selector)
      self.wait.until(expected_conditions.presence_of_element_located(locator))
      elements[name] = self.driver.find_element_by_css_selector(selector)

    # The implementation of paper-* components doesn't seem to play nice
    # with Selenium's `element.is_selected()` and `element.is_enabled()`
    # methods. Instead, we check the appropriate WAI-ARIA attributes.
    # (Also, though the docs for `get_attribute` say that the string
    # `"false"` is returned as `False`, this appears to be the case
    # _sometimes_ but not _always_, so we should take special care to
    # handle that.)
    def is_selected(element):
      attribute = element.get_attribute("aria-selected")
      return attribute and attribute != "false"

    def is_enabled(element):
      attribute = element.get_attribute("aria-disabled")
      is_disabled = attribute and attribute != "false"
      return not is_disabled

    def assert_selected_dashboard(polymer_component_name):
      expected = {polymer_component_name}
      actual = {
        container.find_element_by_css_selector("*").tag_name  # first child
        for container
        in self.driver.find_elements_by_css_selector(".dashboard-container")
        if container.is_displayed()
      }
      self.assertEqual(expected, actual)

    # The scalar and audio dashboards should be active, and the scalar
    # dashboard should be selected by default. The images menu item
    # should not be visible, as it's within the drop-down menu.
    self.assertTrue(elements["scalars_tab"].is_displayed())
    self.assertTrue(elements["audio_tab"].is_displayed())
    self.assertTrue(elements["graphs_tab"].is_displayed())
    self.assertTrue(is_selected(elements["scalars_tab"]))
    self.assertFalse(is_selected(elements["audio_tab"]))
    self.assertFalse(elements["images_menu_item"].is_displayed())
    self.assertFalse(is_selected(elements["images_menu_item"]))
    assert_selected_dashboard("tf-scalar-dashboard")

    # While we're on the scalar dashboard, we should be allowed to
    # reload the data.
    self.assertTrue(is_enabled(elements["reload_button"]))

    # We should be able to activate the audio dashboard.
    elements["audio_tab"].click()
    self.assertFalse(is_selected(elements["scalars_tab"]))
    self.assertTrue(is_selected(elements["audio_tab"]))
    self.assertFalse(is_selected(elements["graphs_tab"]))
    self.assertFalse(is_selected(elements["images_menu_item"]))
    assert_selected_dashboard("tf-audio-dashboard")
    self.assertTrue(is_enabled(elements["reload_button"]))

    # We should then be able to open the dropdown and navigate to the
    # image dashboard. (We have to wait until it's visible because of the
    # dropdown menu's animations.)
    elements["inactive_dropdown"].click()
    self.wait.until(
      expected_conditions.visibility_of(elements["images_menu_item"]))
    self.assertTrue(elements["images_menu_item"].is_displayed())
    elements["images_menu_item"].click()
    self.assertFalse(is_selected(elements["scalars_tab"]))
    self.assertFalse(is_selected(elements["audio_tab"]))
    self.assertFalse(is_selected(elements["graphs_tab"]))
    self.assertTrue(is_selected(elements["images_menu_item"]))
    assert_selected_dashboard("tf-image-dashboard")
    self.assertTrue(is_enabled(elements["reload_button"]))

    # Next, we should be able to navigate back to an active dashboard.
    # If we choose the graphs dashboard, the reload feature should be
    # disabled.
    elements["graphs_tab"].click()
    self.assertFalse(elements["images_menu_item"].is_displayed())
    self.assertFalse(is_selected(elements["scalars_tab"]))
    self.assertFalse(is_selected(elements["audio_tab"]))
    self.assertTrue(is_selected(elements["graphs_tab"]))
    self.assertFalse(is_selected(elements["images_menu_item"]))
    assert_selected_dashboard("tf-graph-dashboard")
    self.assertFalse(is_enabled(elements["reload_button"]))

    # Finally, we should be able to navigate back to the scalar dashboard.
    elements["scalars_tab"].click()
    self.assertTrue(is_selected(elements["scalars_tab"]))
    self.assertFalse(is_selected(elements["audio_tab"]))
    self.assertFalse(is_selected(elements["graphs_tab"]))
    self.assertFalse(is_selected(elements["images_menu_item"]))
    assert_selected_dashboard("tf-scalar-dashboard")
    self.assertTrue(is_enabled(elements["reload_button"]))

if __name__ == "__main__":
  unittest.main()
<|endoftext|>


<commit_before>#-*- coding: utf-8 -*-

from django.contrib.auth.models import User, AnonymousUser
from django.core.cache import cache
from rulez.exceptions import RulesException
import time

"""
Cache keys:

For the list of roles, per user, per instance:
<prefix>-<user.id>-<user counter>-<obj.type>-<obj.id>-<obj counter>

For the counter , per instance:
<prefix>-<obj.type>-<obj.id>

"""

HOUR = 60*60
#===============================================================================
# Counter handling
#===============================================================================
def counter_key(obj):
    if obj.__class__ in (User, AnonymousUser,):
        pk = get_user_pk(obj)
    else:
        pk = obj.pk
    obj_type = str(obj.__class__.__name__).lower()
    return "%s-%s" % (obj_type, pk)

def increment_counter(obj):
    """
    Invalidate the cache for the passed object.
    """
    if obj: # If the object is None, do nothing (it's pointless)
        cache.set(counter_key(obj), int(time.time()), 1*HOUR)

def get_counter(obj):
    """
    Returns the cached counter for the given object instance
    """
    counter = cache.get(counter_key(obj))
    if not counter:
        counter = 0
    return counter

def roles_key(user, obj):
    if obj.__class__ in (User, AnonymousUser,):
        obj_id = get_user_pk(obj)
    else:
        obj_id = obj.pk
    obj_type = str(obj.__class__.__name__).lower()
    obj_counter = get_counter(obj)
    user_id = get_user_pk(user)
    user_counter = get_counter(user)
    return "%s-%s-%s-%s-%s" % (user_id, user_counter, obj_type, obj_id,
                               obj_counter)

def get_user_pk(user):
    if not user or (user and user.is_anonymous()):
        return 'anonymous'
    else:
        return user.pk

#===============================================================================
# Main function
#===============================================================================

def get_roles(user, obj):
    """
    Get a list of roles assigned to a user for a specific instance from the
    cache, or builds such a list if it is not found.
    """
    # get roles for the user, if present:
    roles = cache.get(roles_key(user, obj))
    if roles:
        # Cache hit
        return roles
    else:
        # we need to recompute roles for this model
        user_roles = []
        if not hasattr(obj, 'relevant_roles'):
            raise RulesException('Cannot build roles cache for %s instance. Did you forget to define a "relevant_roles()" method on %s?' % (obj.__class__, obj.__class__))

        relevant = obj.relevant_roles()
        for role in relevant:
            if role.is_member(user, obj):
                user_roles.append(role)
        cache.set(roles_key(user, obj), user_roles, 1*HOUR)
        return user_roles
<commit_msg>Modified cach_helper.py to treat cached empty perm lists as a cache hit<commit_after>#-*- coding: utf-8 -*-

from django.contrib.auth.models import User, AnonymousUser
from django.core.cache import cache
from rulez.exceptions import RulesException
import time

"""
Cache keys:

For the list of roles, per user, per instance:
<prefix>-<user.id>-<user counter>-<obj.type>-<obj.id>-<obj counter>

For the counter , per instance:
<prefix>-<obj.type>-<obj.id>

"""

HOUR = 60*60
#===============================================================================
# Counter handling
#===============================================================================
def counter_key(obj):
    if obj.__class__ in (User, AnonymousUser,):
        pk = get_user_pk(obj)
    else:
        pk = obj.pk
    obj_type = str(obj.__class__.__name__).lower()
    return "%s-%s" % (obj_type, pk)

def increment_counter(obj):
    """
    Invalidate the cache for the passed object.
    """
    if obj: # If the object is None, do nothing (it's pointless)
        cache.set(counter_key(obj), int(time.time()), 1*HOUR)

def get_counter(obj):
    """
    Returns the cached counter for the given object instance
    """
    counter = cache.get(counter_key(obj))
    if not counter:
        counter = 0
    return counter

def roles_key(user, obj):
    if obj.__class__ in (User, AnonymousUser,):
        obj_id = get_user_pk(obj)
    else:
        obj_id = obj.pk
    obj_type = str(obj.__class__.__name__).lower()
    obj_counter = get_counter(obj)
    user_id = get_user_pk(user)
    user_counter = get_counter(user)
    return "%s-%s-%s-%s-%s" % (user_id, user_counter, obj_type, obj_id,
                               obj_counter)

def get_user_pk(user):
    if not user or (user and user.is_anonymous()):
        return 'anonymous'
    else:
        return user.pk

#===============================================================================
# Main function
#===============================================================================

def get_roles(user, obj):
    """
    Get a list of roles assigned to a user for a specific instance from the
    cache, or builds such a list if it is not found.
    """
    # get roles for the user, if present:
    roles = cache.get(roles_key(user, obj))
    if isinstance(roles, list):
        # Cache hit (a miss returns NoneType rather than an empty list)
        return roles
    else:
        # we need to recompute roles for this model
        user_roles = []
        if not hasattr(obj, 'relevant_roles'):
            raise RulesException('Cannot build roles cache for %s instance. Did you forget to define a "relevant_roles()" method on %s?' % (obj.__class__, obj.__class__))

        relevant = obj.relevant_roles()
        for role in relevant:
            if role.is_member(user, obj):
                user_roles.append(role)
        cache.set(roles_key(user, obj), user_roles, 1*HOUR)
        return user_roles
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright 2010 Eric Entzel <eric@ubermac.net>
#

import ecal
from google.appengine.ext.webapp.util import run_wsgi_app
from google.appengine.ext.webapp.mail_handlers import InboundMailHandler
from google.appengine.api import mail
from google.appengine.runtime import apiproxy_errors
from gdata.service import RequestError
from email.header import decode_header
import logging
import urllib
from datetime import datetime
import google_api
import outgoing_mail
import re
import string
import pickle

class FeedbackHandler(object):
    """Abstract class to send email feedback to the user that their
    event creation succeeded/failed.

    Swallows all exceptions so that the HTTP request that GAE sends us
    for incoming mail doesn't throw a 500 error -- if it did, GAE
    would keep retrying and creating duplicate events.

    Subclasses must implement a values() method and a template_name
    attribute.  Subclasses can optionally implement a warning
    attribute, which will be logged.
    """
    def send(self):
        recipient = self.message.sender
        if hasattr(self, 'warning'):
            logging.warn(self.warning)
            logging.info("Sender: " + recipient)
            if hasattr(self.message, 'subject'):
                logging.info("Subject: " + self.message.subject)
        try:
            outgoing_mail.reply_to(self.message, self.template_name, self.values())
        except Exception:
            logging.exception("unable to send email to %s using template %s" %
                              (recipient, self.template_name))

class NoSubjectHandler(FeedbackHandler):
    template_name = 'no_subject'
    warning = "Couldn't create event because message had no subject"
    def __init__(self, message):
        self.message = message
    def values(self):
        return { 'original_body': self.message.body }

class NoSuchAddressHandler(FeedbackHandler):
    template_name = 'no_such_address'
    def __init__(self, message, adr):
        self.message = message
        self.adr = adr
        self.warning = "Couldn't create event for user with address " + self.adr

    def values(self):
        return { 'address': self.adr,
                 'subject': self.message.subject }

class SuccessHandler(FeedbackHandler):
    template_name = 'event_created'
    def __init__(self, message, event):
        self.message = message
        self.event = event

    def values(self):
        start_time = self.event.FindExtensions(tag='when')[0].attributes['startTime']
        return { 'link': self.event.GetHtmlLink().href,
                 'when': CreateEventHandler._format_date(start_time),
                 'title': self.event.title.text }

class TokenRevokedHandler(FeedbackHandler):
    template_name = 'token_revoked'
    def __init__(self, message, adr):
        self.message = message
        self.adr = adr

    def values(self):
        return { 'address': self.adr,
                 'subject': self.message.subject }

class CreateEventHandler(InboundMailHandler):
    # self.request.path will contain something like:
    # /_ah/mail/f832ofhAau%40myeventbot.appspotmail.com
    def _get_email_address(self):
        decoded_path = urllib.unquote(self.request.path)
        return decoded_path.replace('/_ah/mail/', '')

    def _get_user(self):
        local_part = self._get_email_address().split('@')[0]
        query = ecal.EcalUser.gql("WHERE email_address = :email", email=local_part)
        return query.get()

    @staticmethod
    def _format_date(str):
        """Format a date from GCal into a user-friendly string."""
        str = re.sub(r".000[-+Z].*$", "", str)
        try:
            date = datetime.strptime(str, "%Y-%m-%dT%H:%M:%S")
            retval = date.strftime("%a %b %e %I:%M %p")
        except ValueError:
            date = datetime.strptime(str, "%Y-%m-%d")
            retval = date.strftime("%a %b %e")
        retval = retval.replace('  ', ' ')
        retval = retval.replace(' 0', ' ')
        return retval

    @staticmethod
    def header_to_utf8(header, default='utf-8'):
        """It appears that GAE does not decode rfc2047-encoded headers
        for us, so we have to do it manually.  Up to three steps may
        be needed:
            1) quoted-printable or base64 decoding, performed by decode_header()
            2) character set (e.g., utf-8 or iso-8859-1) decoding, performed by decode()
            3) conversion into UTF-8, performed by encode()
        """
        try:
            parts = decode_header(header)
            decoded = [value.decode(charset or default) for value, charset in parts]
            return u"".join(decoded).encode('utf-8')
        except UnicodeDecodeError, err:
            logging.exception("Couldn't decode header: " + pickle.dumps(header))

    @staticmethod
    def strip_bcc(body):
        lines = string.split(body, '\n')
        return '\n'.join([l for l in lines if l[:4] != 'Bcc:'])

    def post(self):
        # Overridden to strip the 'Bcc' header before creating an
        # InboundEmailMessage, since we never need it and
        # InboundEmailMessage barfs on an empty 'Bcc'
        new_body = CreateEventHandler.strip_bcc(self.request.body)
        self.receive(mail.InboundEmailMessage(new_body))

    def receive(self, message):
        if not hasattr(message, 'subject'):
            NoSubjectHandler(message).send()
            return
        current_user = self._get_user()
        if current_user == None:
            NoSuchAddressHandler(message, self._get_email_address()).send()
            return
        token = current_user.auth_token
        message.subject = CreateEventHandler.header_to_utf8(message.subject)
        logging.info("Decoded Subject: " + message.subject)
        # TODO: everything after this point should move into a task queue
        try:
            event = google_api.quickadd_event_using_token(message.subject, token)
        except RequestError, err:
            if err.args[0]['status'] == 401:
                # TODO: we're hitting this when we shouldn't, with a
                # HTTP 401 response "Unknown authorization header".
                # Can we log the headers of the POST we sent, to see
                # if there's anything obviously wrong with them?
                logging.exception("Token doesn't appear to be valid anymore")
                TokenRevokedHandler(message, self._get_email_address()).send()
            else:
                # Don't know what the error is, let's re-raise it so
                # it gets logged and the message retried
                raise
            return
        try:
            action = ecal.EcalAction(type="event_created", user=current_user)
            action.put()
        except apiproxy_errors.CapabilityDisabledError:
            # The event's already been created, if we can't record the
            # EcalAction, so be it
            pass
        if current_user.send_emails:
            SuccessHandler(message, event).send()

def main():
    application = ecal.EcalWSGIApplication([CreateEventHandler.mapping()])
    run_wsgi_app(application)

if __name__ == "__main__":
  main()
<commit_msg>Log events that don't have a startTime<commit_after>#!/usr/bin/env python
#
# Copyright 2010 Eric Entzel <eric@ubermac.net>
#

import ecal
from google.appengine.ext.webapp.util import run_wsgi_app
from google.appengine.ext.webapp.mail_handlers import InboundMailHandler
from google.appengine.api import mail
from google.appengine.runtime import apiproxy_errors
from gdata.service import RequestError
from email.header import decode_header
import logging
import urllib
from datetime import datetime
import google_api
import outgoing_mail
import re
import string
import pickle

class FeedbackHandler(object):
    """Abstract class to send email feedback to the user that their
    event creation succeeded/failed.

    Swallows all exceptions so that the HTTP request that GAE sends us
    for incoming mail doesn't throw a 500 error -- if it did, GAE
    would keep retrying and creating duplicate events.

    Subclasses must implement a values() method and a template_name
    attribute.  Subclasses can optionally implement a warning
    attribute, which will be logged.
    """
    def send(self):
        recipient = self.message.sender
        if hasattr(self, 'warning'):
            logging.warn(self.warning)
            logging.info("Sender: " + recipient)
            if hasattr(self.message, 'subject'):
                logging.info("Subject: " + self.message.subject)
        try:
            outgoing_mail.reply_to(self.message, self.template_name, self.values())
        except Exception:
            logging.exception("unable to send email to %s using template %s" %
                              (recipient, self.template_name))

class NoSubjectHandler(FeedbackHandler):
    template_name = 'no_subject'
    warning = "Couldn't create event because message had no subject"
    def __init__(self, message):
        self.message = message
    def values(self):
        return { 'original_body': self.message.body }

class NoSuchAddressHandler(FeedbackHandler):
    template_name = 'no_such_address'
    def __init__(self, message, adr):
        self.message = message
        self.adr = adr
        self.warning = "Couldn't create event for user with address " + self.adr

    def values(self):
        return { 'address': self.adr,
                 'subject': self.message.subject }

class SuccessHandler(FeedbackHandler):
    template_name = 'event_created'
    def __init__(self, message, event):
        self.message = message
        self.event = event

    def values(self):
        when = self.event.FindExtensions(tag='when')
        if len(when) > 0:
            start_time = when[0].attributes['startTime']
            return { 'link': self.event.GetHtmlLink().href,
                     'when': CreateEventHandler._format_date(start_time),
                     'title': self.event.title.text }
        else:
            logging.warn("returned event had no startTime")
            logging.warn(self.event.ToString())
            return when[0]      # intentional exception

class TokenRevokedHandler(FeedbackHandler):
    template_name = 'token_revoked'
    def __init__(self, message, adr):
        self.message = message
        self.adr = adr

    def values(self):
        return { 'address': self.adr,
                 'subject': self.message.subject }

class CreateEventHandler(InboundMailHandler):
    # self.request.path will contain something like:
    # /_ah/mail/f832ofhAau%40myeventbot.appspotmail.com
    def _get_email_address(self):
        decoded_path = urllib.unquote(self.request.path)
        return decoded_path.replace('/_ah/mail/', '')

    def _get_user(self):
        local_part = self._get_email_address().split('@')[0]
        query = ecal.EcalUser.gql("WHERE email_address = :email", email=local_part)
        return query.get()

    @staticmethod
    def _format_date(str):
        """Format a date from GCal into a user-friendly string."""
        str = re.sub(r".000[-+Z].*$", "", str)
        try:
            date = datetime.strptime(str, "%Y-%m-%dT%H:%M:%S")
            retval = date.strftime("%a %b %e %I:%M %p")
        except ValueError:
            date = datetime.strptime(str, "%Y-%m-%d")
            retval = date.strftime("%a %b %e")
        retval = retval.replace('  ', ' ')
        retval = retval.replace(' 0', ' ')
        return retval

    @staticmethod
    def header_to_utf8(header, default='utf-8'):
        """It appears that GAE does not decode rfc2047-encoded headers
        for us, so we have to do it manually.  Up to three steps may
        be needed:
            1) quoted-printable or base64 decoding, performed by decode_header()
            2) character set (e.g., utf-8 or iso-8859-1) decoding, performed by decode()
            3) conversion into UTF-8, performed by encode()
        """
        try:
            parts = decode_header(header)
            decoded = [value.decode(charset or default) for value, charset in parts]
            return u"".join(decoded).encode('utf-8')
        except UnicodeDecodeError, err:
            logging.exception("Couldn't decode header: " + pickle.dumps(header))

    @staticmethod
    def strip_bcc(body):
        lines = string.split(body, '\n')
        return '\n'.join([l for l in lines if l[:4] != 'Bcc:'])

    def post(self):
        # Overridden to strip the 'Bcc' header before creating an
        # InboundEmailMessage, since we never need it and
        # InboundEmailMessage barfs on an empty 'Bcc'
        new_body = CreateEventHandler.strip_bcc(self.request.body)
        self.receive(mail.InboundEmailMessage(new_body))

    def receive(self, message):
        if not hasattr(message, 'subject'):
            NoSubjectHandler(message).send()
            return
        current_user = self._get_user()
        if current_user == None:
            NoSuchAddressHandler(message, self._get_email_address()).send()
            return
        token = current_user.auth_token
        message.subject = CreateEventHandler.header_to_utf8(message.subject)
        logging.info("Decoded Subject: " + message.subject)
        # TODO: everything after this point should move into a task queue
        try:
            event = google_api.quickadd_event_using_token(message.subject, token)
        except RequestError, err:
            if err.args[0]['status'] == 401:
                # TODO: we're hitting this when we shouldn't, with a
                # HTTP 401 response "Unknown authorization header".
                # Can we log the headers of the POST we sent, to see
                # if there's anything obviously wrong with them?
                logging.exception("Token doesn't appear to be valid anymore")
                TokenRevokedHandler(message, self._get_email_address()).send()
            else:
                # Don't know what the error is, let's re-raise it so
                # it gets logged and the message retried
                raise
            return
        try:
            action = ecal.EcalAction(type="event_created", user=current_user)
            action.put()
        except apiproxy_errors.CapabilityDisabledError:
            # The event's already been created, if we can't record the
            # EcalAction, so be it
            pass
        if current_user.send_emails:
            SuccessHandler(message, event).send()

def main():
    application = ecal.EcalWSGIApplication([CreateEventHandler.mapping()])
    run_wsgi_app(application)

if __name__ == "__main__":
  main()
<|endoftext|>


<commit_before>from requests.exceptions import HTTPError
import logging

def http_exception_handler(f):
    """
    Decorator to keep try catch block dry for all API calls.

    :param f: function being wrapped
    :return:
    """
    def wrapper():
        try:
            response = f()
            response.raise_for_status()
            return response.json()
        except HTTPError as error:
            logging.error(error.message)
        except Exception as error:
            logging.error(error.message)

    return wrapper
<commit_msg>Added ability for http_exception decorator to accept methods with parameters<commit_after>from requests.exceptions import HTTPError
import logging

def http_exception_handler(f):
    """
    Decorator to keep try catch block dry for all API calls.

    :param f: function being wrapped
    :return:
    """
    def wrapper(*args, **kwargs):
        try:
            response = f(*args, **kwargs)
            response.raise_for_status()
            return response.json()
        except HTTPError as error:
            logging.error(error.message)
        except Exception as error:
            logging.error(error.message)

    return wrapper
<|endoftext|>


<commit_before># Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Imports unittest as a replacement for testing.pybase.googletest."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import inspect
import itertools
import os
import sys
import tempfile

# go/tf-wildcard-import
# pylint: disable=wildcard-import
from unittest import *
# pylint: enable=wildcard-import

from tensorflow.python.platform import benchmark  # pylint: disable=unused-import

Benchmark = benchmark.TensorFlowBenchmark  # pylint: disable=invalid-name

unittest_main = main

# pylint: disable=invalid-name
# pylint: disable=undefined-variable
def g_main(*args, **kwargs):
  """Delegate to unittest.main after redefining testLoader."""
  if 'TEST_SHARD_STATUS_FILE' in os.environ:
    try:
      f = None
      try:
        f = open(os.environ['TEST_SHARD_STATUS_FILE'], 'w')
        f.write('')
      except IOError:
        sys.stderr.write('Error opening TEST_SHARD_STATUS_FILE (%s). Exiting.'
                         % os.environ['TEST_SHARD_STATUS_FILE'])
        sys.exit(1)
    finally:
      if f is not None: f.close()

  if ('TEST_TOTAL_SHARDS' not in os.environ or
      'TEST_SHARD_INDEX' not in os.environ):
    return unittest_main(*args, **kwargs)

  total_shards = int(os.environ['TEST_TOTAL_SHARDS'])
  shard_index = int(os.environ['TEST_SHARD_INDEX'])
  base_loader = TestLoader()

  delegate_get_names = base_loader.getTestCaseNames
  bucket_iterator = itertools.cycle(range(total_shards))

  def getShardedTestCaseNames(testCaseClass):
    filtered_names = []
    for testcase in sorted(delegate_get_names(testCaseClass)):
      bucket = next(bucket_iterator)
      if bucket == shard_index:
        filtered_names.append(testcase)
    return filtered_names

  # Override getTestCaseNames
  base_loader.getTestCaseNames = getShardedTestCaseNames

  kwargs['testLoader'] = base_loader
  unittest_main(*args, **kwargs)

# Redefine main to allow running benchmarks
def main():  # pylint: disable=function-redefined
  benchmark.benchmarks_main(true_main=g_main)

def GetTempDir():
  first_frame = inspect.stack()[-1][0]
  temp_dir = os.path.join(
      tempfile.gettempdir(), os.path.basename(inspect.getfile(first_frame)))
  temp_dir = temp_dir.rstrip('.py')
  if not os.path.isdir(temp_dir):
    os.mkdir(temp_dir, 0o755)
  return temp_dir

def test_src_dir_path(relative_path):
  """Creates an absolute test srcdir path given a relative path.

  Args:
    relative_path: a path relative to tensorflow root.
      e.g. "contrib/session_bundle/example".

  Returns:
    An absolute path to the linked in runfiles.
  """
  return os.path.join(os.environ['TEST_SRCDIR'],
                      "org_tensorflow/tensorflow", relative_path)

def StatefulSessionAvailable():
  return False

class StubOutForTesting(object):
  """Support class for stubbing methods out for unit testing.

  Sample Usage:

  You want os.path.exists() to always return true during testing.

     stubs = StubOutForTesting()
     stubs.Set(os.path, 'exists', lambda x: 1)
       ...
     stubs.CleanUp()

  The above changes os.path.exists into a lambda that returns 1.  Once
  the ... part of the code finishes, the CleanUp() looks up the old
  value of os.path.exists and restores it.
  """

  def __init__(self):
    self.cache = []
    self.stubs = []

  def __del__(self):
    """Do not rely on the destructor to undo your stubs.

    You cannot guarantee exactly when the destructor will get called without
    relying on implementation details of a Python VM that may change.
    """
    self.CleanUp()

  # __enter__ and __exit__ allow use as a context manager.
  def __enter__(self):
    return self

  def __exit__(self, unused_exc_type, unused_exc_value, unused_tb):
    self.CleanUp()

  def CleanUp(self):
    """Undoes all SmartSet() & Set() calls, restoring original definitions."""
    self.SmartUnsetAll()
    self.UnsetAll()

  def SmartSet(self, obj, attr_name, new_attr):
    """Replace obj.attr_name with new_attr.

    This method is smart and works at the module, class, and instance level
    while preserving proper inheritance. It will not stub out C types however
    unless that has been explicitly allowed by the type.

    This method supports the case where attr_name is a staticmethod or a
    classmethod of obj.

    Notes:
      - If obj is an instance, then it is its class that will actually be
        stubbed. Note that the method Set() does not do that: if obj is
        an instance, it (and not its class) will be stubbed.
      - The stubbing is using the builtin getattr and setattr. So, the __get__
        and __set__ will be called when stubbing (TODO: A better idea would
        probably be to manipulate obj.__dict__ instead of getattr() and
        setattr()).

    Args:
      obj: The object whose attributes we want to modify.
      attr_name: The name of the attribute to modify.
      new_attr: The new value for the attribute.

    Raises:
      AttributeError: If the attribute cannot be found.
    """
    if (inspect.ismodule(obj) or
        (not inspect.isclass(obj) and attr_name in obj.__dict__)):
      orig_obj = obj
      orig_attr = getattr(obj, attr_name)
    else:
      if not inspect.isclass(obj):
        mro = list(inspect.getmro(obj.__class__))
      else:
        mro = list(inspect.getmro(obj))

      mro.reverse()

      orig_attr = None
      found_attr = False

      for cls in mro:
        try:
          orig_obj = cls
          orig_attr = getattr(obj, attr_name)
          found_attr = True
        except AttributeError:
          continue

      if not found_attr:
        raise AttributeError('Attribute not found.')

    # Calling getattr() on a staticmethod transforms it to a 'normal' function.
    # We need to ensure that we put it back as a staticmethod.
    old_attribute = obj.__dict__.get(attr_name)
    if old_attribute is not None and isinstance(old_attribute, staticmethod):
      orig_attr = staticmethod(orig_attr)

    self.stubs.append((orig_obj, attr_name, orig_attr))
    setattr(orig_obj, attr_name, new_attr)

  def SmartUnsetAll(self):
    """Reverses SmartSet() calls, restoring things to original definitions.

    This method is automatically called when the StubOutForTesting()
    object is deleted; there is no need to call it explicitly.

    It is okay to call SmartUnsetAll() repeatedly, as later calls have
    no effect if no SmartSet() calls have been made.
    """
    for args in reversed(self.stubs):
      setattr(*args)

    self.stubs = []

  def Set(self, parent, child_name, new_child):
    """In parent, replace child_name's old definition with new_child.

    The parent could be a module when the child is a function at
    module scope.  Or the parent could be a class when a class' method
    is being replaced.  The named child is set to new_child, while the
    prior definition is saved away for later, when UnsetAll() is
    called.

    This method supports the case where child_name is a staticmethod or a
    classmethod of parent.

    Args:
      parent: The context in which the attribute child_name is to be changed.
      child_name: The name of the attribute to change.
      new_child: The new value of the attribute.
    """
    old_child = getattr(parent, child_name)

    old_attribute = parent.__dict__.get(child_name)
    if old_attribute is not None and isinstance(old_attribute, staticmethod):
      old_child = staticmethod(old_child)

    self.cache.append((parent, old_child, child_name))
    setattr(parent, child_name, new_child)

  def UnsetAll(self):
    """Reverses Set() calls, restoring things to their original definitions.

    This method is automatically called when the StubOutForTesting()
    object is deleted; there is no need to call it explicitly.

    It is okay to call UnsetAll() repeatedly, as later calls have no
    effect if no Set() calls have been made.
    """
    # Undo calls to Set() in reverse order, in case Set() was called on the
    # same arguments repeatedly (want the original call to be last one undone)
    for (parent, old_child, child_name) in reversed(self.cache):
      setattr(parent, child_name, old_child)
    self.cache = []
<commit_msg>Fix parsing of Python command-line arguments in tests. Previously command-line arguments to Python tests were ignored. However, since we accept command line arguments to Python binaries, we may as well accept them as arguments to tests too. Change: 143549767<commit_after># Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Imports unittest as a replacement for testing.pybase.googletest."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import inspect
import itertools
import os
import sys
import tempfile

# go/tf-wildcard-import
# pylint: disable=wildcard-import
from unittest import *
# pylint: enable=wildcard-import

from tensorflow.python.platform import app
from tensorflow.python.platform import benchmark  # pylint: disable=unused-import

Benchmark = benchmark.TensorFlowBenchmark  # pylint: disable=invalid-name

unittest_main = main

# pylint: disable=invalid-name
# pylint: disable=undefined-variable
def g_main(argv):
  """Delegate to unittest.main after redefining testLoader."""
  if 'TEST_SHARD_STATUS_FILE' in os.environ:
    try:
      f = None
      try:
        f = open(os.environ['TEST_SHARD_STATUS_FILE'], 'w')
        f.write('')
      except IOError:
        sys.stderr.write('Error opening TEST_SHARD_STATUS_FILE (%s). Exiting.'
                         % os.environ['TEST_SHARD_STATUS_FILE'])
        sys.exit(1)
    finally:
      if f is not None: f.close()

  if ('TEST_TOTAL_SHARDS' not in os.environ or
      'TEST_SHARD_INDEX' not in os.environ):
    return unittest_main(argv=argv)

  total_shards = int(os.environ['TEST_TOTAL_SHARDS'])
  shard_index = int(os.environ['TEST_SHARD_INDEX'])
  base_loader = TestLoader()

  delegate_get_names = base_loader.getTestCaseNames
  bucket_iterator = itertools.cycle(range(total_shards))

  def getShardedTestCaseNames(testCaseClass):
    filtered_names = []
    for testcase in sorted(delegate_get_names(testCaseClass)):
      bucket = next(bucket_iterator)
      if bucket == shard_index:
        filtered_names.append(testcase)
    return filtered_names

  # Override getTestCaseNames
  base_loader.getTestCaseNames = getShardedTestCaseNames

  unittest_main(argv=argv, testLoader=base_loader)

# Redefine main to allow running benchmarks
def main():  # pylint: disable=function-redefined
  def main_wrapper():
    return app.run(main=g_main, argv=sys.argv)
  benchmark.benchmarks_main(true_main=main_wrapper)

def GetTempDir():
  first_frame = inspect.stack()[-1][0]
  temp_dir = os.path.join(
      tempfile.gettempdir(), os.path.basename(inspect.getfile(first_frame)))
  temp_dir = temp_dir.rstrip('.py')
  if not os.path.isdir(temp_dir):
    os.mkdir(temp_dir, 0o755)
  return temp_dir

def test_src_dir_path(relative_path):
  """Creates an absolute test srcdir path given a relative path.

  Args:
    relative_path: a path relative to tensorflow root.
      e.g. "contrib/session_bundle/example".

  Returns:
    An absolute path to the linked in runfiles.
  """
  return os.path.join(os.environ['TEST_SRCDIR'],
                      "org_tensorflow/tensorflow", relative_path)

def StatefulSessionAvailable():
  return False

class StubOutForTesting(object):
  """Support class for stubbing methods out for unit testing.

  Sample Usage:

  You want os.path.exists() to always return true during testing.

     stubs = StubOutForTesting()
     stubs.Set(os.path, 'exists', lambda x: 1)
       ...
     stubs.CleanUp()

  The above changes os.path.exists into a lambda that returns 1.  Once
  the ... part of the code finishes, the CleanUp() looks up the old
  value of os.path.exists and restores it.
  """

  def __init__(self):
    self.cache = []
    self.stubs = []

  def __del__(self):
    """Do not rely on the destructor to undo your stubs.

    You cannot guarantee exactly when the destructor will get called without
    relying on implementation details of a Python VM that may change.
    """
    self.CleanUp()

  # __enter__ and __exit__ allow use as a context manager.
  def __enter__(self):
    return self

  def __exit__(self, unused_exc_type, unused_exc_value, unused_tb):
    self.CleanUp()

  def CleanUp(self):
    """Undoes all SmartSet() & Set() calls, restoring original definitions."""
    self.SmartUnsetAll()
    self.UnsetAll()

  def SmartSet(self, obj, attr_name, new_attr):
    """Replace obj.attr_name with new_attr.

    This method is smart and works at the module, class, and instance level
    while preserving proper inheritance. It will not stub out C types however
    unless that has been explicitly allowed by the type.

    This method supports the case where attr_name is a staticmethod or a
    classmethod of obj.

    Notes:
      - If obj is an instance, then it is its class that will actually be
        stubbed. Note that the method Set() does not do that: if obj is
        an instance, it (and not its class) will be stubbed.
      - The stubbing is using the builtin getattr and setattr. So, the __get__
        and __set__ will be called when stubbing (TODO: A better idea would
        probably be to manipulate obj.__dict__ instead of getattr() and
        setattr()).

    Args:
      obj: The object whose attributes we want to modify.
      attr_name: The name of the attribute to modify.
      new_attr: The new value for the attribute.

    Raises:
      AttributeError: If the attribute cannot be found.
    """
    if (inspect.ismodule(obj) or
        (not inspect.isclass(obj) and attr_name in obj.__dict__)):
      orig_obj = obj
      orig_attr = getattr(obj, attr_name)
    else:
      if not inspect.isclass(obj):
        mro = list(inspect.getmro(obj.__class__))
      else:
        mro = list(inspect.getmro(obj))

      mro.reverse()

      orig_attr = None
      found_attr = False

      for cls in mro:
        try:
          orig_obj = cls
          orig_attr = getattr(obj, attr_name)
          found_attr = True
        except AttributeError:
          continue

      if not found_attr:
        raise AttributeError('Attribute not found.')

    # Calling getattr() on a staticmethod transforms it to a 'normal' function.
    # We need to ensure that we put it back as a staticmethod.
    old_attribute = obj.__dict__.get(attr_name)
    if old_attribute is not None and isinstance(old_attribute, staticmethod):
      orig_attr = staticmethod(orig_attr)

    self.stubs.append((orig_obj, attr_name, orig_attr))
    setattr(orig_obj, attr_name, new_attr)

  def SmartUnsetAll(self):
    """Reverses SmartSet() calls, restoring things to original definitions.

    This method is automatically called when the StubOutForTesting()
    object is deleted; there is no need to call it explicitly.

    It is okay to call SmartUnsetAll() repeatedly, as later calls have
    no effect if no SmartSet() calls have been made.
    """
    for args in reversed(self.stubs):
      setattr(*args)

    self.stubs = []

  def Set(self, parent, child_name, new_child):
    """In parent, replace child_name's old definition with new_child.

    The parent could be a module when the child is a function at
    module scope.  Or the parent could be a class when a class' method
    is being replaced.  The named child is set to new_child, while the
    prior definition is saved away for later, when UnsetAll() is
    called.

    This method supports the case where child_name is a staticmethod or a
    classmethod of parent.

    Args:
      parent: The context in which the attribute child_name is to be changed.
      child_name: The name of the attribute to change.
      new_child: The new value of the attribute.
    """
    old_child = getattr(parent, child_name)

    old_attribute = parent.__dict__.get(child_name)
    if old_attribute is not None and isinstance(old_attribute, staticmethod):
      old_child = staticmethod(old_child)

    self.cache.append((parent, old_child, child_name))
    setattr(parent, child_name, new_child)

  def UnsetAll(self):
    """Reverses Set() calls, restoring things to their original definitions.

    This method is automatically called when the StubOutForTesting()
    object is deleted; there is no need to call it explicitly.

    It is okay to call UnsetAll() repeatedly, as later calls have no
    effect if no Set() calls have been made.
    """
    # Undo calls to Set() in reverse order, in case Set() was called on the
    # same arguments repeatedly (want the original call to be last one undone)
    for (parent, old_child, child_name) in reversed(self.cache):
      setattr(parent, child_name, old_child)
    self.cache = []
<|endoftext|>


<commit_before>######
#  Menu for Community Scripts
#
#  Author:        Christoph Stoettner
#  Mail:          christoph.stoettner@stoeps.de
#  Documentation: http://scripting101.stoeps.de
#
#  Version:       2.0
#  Date:          2014-06-04
#
#  License:       Apache 2.0
#
#  History:       Changed by Jan Alderlieste

import ibmcnx.functions
import ibmcnx.menu.MenuClass

# Load all jython commands, when they are not loaded
#try:
#    NewsActivityStreamService.listApplicationRegistrations()
#except NameError:
#    print "Connections Commands not loaded! Load now: "
#    execfile("ibmcnx/loadCnxApps.py")

cfg = ibmcnx.menu.MenuClass.cnxMenu()
cfg.AddItem( "Configure DataSources (cfgDataSource.py)", ibmcnx.functions.cfgDataSource )
cfg.AddItem( 'Backup J2EE Roles of all Applications (cfgJ2EERoleBackup.py)', ibmcnx.functions.cfgJ2EERoleBackup )
cfg.AddItem( 'Restore J2EE Roles of all Applications (cfgJ2EERoleRestore.py)', ibmcnx.functions.cfgJ2EERoleRestore )
cfg.AddItem( 'Set J2EE Roles initially (restricted) (cfgJ2EERolesRestricted.py)', ibmcnx.functions.cfgJ2EERolesRestricted )
cfg.AddItem( 'Set J2EE Roles initially (unrestricted) (cfgJ2EERolesUnrestricted.py)', ibmcnx.functions.cfgJ2EERolesUnrestricted )
cfg.AddItem( 'Set J2EE Roles for Moderator Roles (cfgJ2EERoleGlobalModerator.py)', ibmcnx.functions.cfgJ2EERoleGlobalModerator )
cfg.AddItem( 'Set J2EE Role for Metrics Reader (cfgJ2EERoleMetricsReader.py)', ibmcnx.functions.cfgJ2EERoleMetricsReader )
cfg.AddItem( 'Set J2EE Role for Metrics Report Run (cfgJ2EERoleMetricsReportRun)', ibmcnx.functions.cfgJ2EERoleMetricsReportRun )
cfg.AddItem( 'Set J2EE Role for SocialMail (cfgJ2EERoleSocialMail)', ibmcnx.functions.cfgJ2EERoleSocialMail )
cfg.AddItem( 'Configure JVM Heap Sizes (cfgJVMHeap.py)', ibmcnx.functions.cfgJVMHeap )
cfg.AddItem( 'Set Custom Parameter for Cache Issues in JVM (cfgJVMCustProp.py)', ibmcnx.functions.cfgJVMCustProp )
cfg.AddItem( 'Configure SystemOut/Err Log Size (cfgLogFiles.py)', ibmcnx.functions.cfgLogFiles )
cfg.AddItem( 'Configure Monitoring Policy (cfgMonitoringPolicy.py)', ibmcnx.functions.cfgMonitoringPolicy )
cfg.AddItem( 'Work with Files Policies (cnxFilesPolicies.py)', ibmcnx.functions.cnxFilesPolicies )
cfg.AddItem( 'Work with Libraries (cnxLibraryPolicies.py)', ibmcnx.functions.cnxLibraryPolicies )
cfg.AddItem( 'Back to Main Menu (cnxmenu.py)', ibmcnx.functions.cnxBackToMainMenu )
cfg.AddItem( "Exit", ibmcnx.functions.bye )

state_cfg = 'True'

while state_cfg == 'True':
    count = cfg.Show() + 1
    cfg.Show()
    print count

    ###########################
    ## Robust error handling ##
    ## only accept int       ##
    ###########################
    ## Wait for valid input in while...not ###
    is_valid_cfg = 0
    while not is_valid_cfg :
        try :
                inputstring = 'Enter your choice [1-' + count +']: '
                n = int ( raw_input( inputstring ) )

                if n < count and n > 0:
				    is_valid_cfg = 1 ## set it to 1 to validate input and to terminate the while..not loop
                else:
                    print ( "'%s' is not a valid menu option.") % n
        except ValueError, e :
                print ("'%s' is not a valid integer." % e.args[0].split(": ")[1])
   # n = input( "your choice> " )
    cfg.Do( n - 1 )
<commit_msg>Some Menu work<commit_after>######
#  Menu for Community Scripts
#
#  Author:        Christoph Stoettner
#  Mail:          christoph.stoettner@stoeps.de
#  Documentation: http://scripting101.stoeps.de
#
#  Version:       2.0
#  Date:          2014-06-04
#
#  License:       Apache 2.0
#
#  History:       Changed by Jan Alderlieste

import ibmcnx.functions
import ibmcnx.menu.MenuClass

# Load all jython commands, when they are not loaded
#try:
#    NewsActivityStreamService.listApplicationRegistrations()
#except NameError:
#    print "Connections Commands not loaded! Load now: "
#    execfile("ibmcnx/loadCnxApps.py")

cfg = ibmcnx.menu.MenuClass.cnxMenu()
cfg.AddItem( "Configure DataSources (cfgDataSource.py)", ibmcnx.functions.cfgDataSource )
cfg.AddItem( 'Backup J2EE Roles of all Applications (cfgJ2EERoleBackup.py)', ibmcnx.functions.cfgJ2EERoleBackup )
cfg.AddItem( 'Restore J2EE Roles of all Applications (cfgJ2EERoleRestore.py)', ibmcnx.functions.cfgJ2EERoleRestore )
cfg.AddItem( 'Set J2EE Roles initially (restricted) (cfgJ2EERolesRestricted.py)', ibmcnx.functions.cfgJ2EERolesRestricted )
cfg.AddItem( 'Set J2EE Roles initially (unrestricted) (cfgJ2EERolesUnrestricted.py)', ibmcnx.functions.cfgJ2EERolesUnrestricted )
cfg.AddItem( 'Set J2EE Roles for Moderator Roles (cfgJ2EERoleGlobalModerator.py)', ibmcnx.functions.cfgJ2EERoleGlobalModerator )
cfg.AddItem( 'Set J2EE Role for Metrics Reader (cfgJ2EERoleMetricsReader.py)', ibmcnx.functions.cfgJ2EERoleMetricsReader )
cfg.AddItem( 'Set J2EE Role for Metrics Report Run (cfgJ2EERoleMetricsReportRun)', ibmcnx.functions.cfgJ2EERoleMetricsReportRun )
cfg.AddItem( 'Set J2EE Role for SocialMail (cfgJ2EERoleSocialMail)', ibmcnx.functions.cfgJ2EERoleSocialMail )
cfg.AddItem( 'Configure JVM Heap Sizes (cfgJVMHeap.py)', ibmcnx.functions.cfgJVMHeap )
cfg.AddItem( 'Set Custom Parameter for Cache Issues in JVM (cfgJVMCustProp.py)', ibmcnx.functions.cfgJVMCustProp )
cfg.AddItem( 'Configure SystemOut/Err Log Size (cfgLogFiles.py)', ibmcnx.functions.cfgLogFiles )
cfg.AddItem( 'Configure Monitoring Policy (cfgMonitoringPolicy.py)', ibmcnx.functions.cfgMonitoringPolicy )
cfg.AddItem( 'Work with Files Policies (cnxFilesPolicies.py)', ibmcnx.functions.cnxFilesPolicies )
cfg.AddItem( 'Work with Libraries (cnxLibraryPolicies.py)', ibmcnx.functions.cnxLibraryPolicies )
cfg.AddItem( 'Back to Main Menu (cnxmenu.py)', ibmcnx.functions.cnxBackToMainMenu )
cfg.AddItem( "Exit", ibmcnx.functions.bye )

state_cfg = 'True'

while state_cfg == 'True':
    count = cfg.Show() + 1
    cfg.Show()
    print cfg.menuitems

    ###########################
    ## Robust error handling ##
    ## only accept int       ##
    ###########################
    ## Wait for valid input in while...not ###
    is_valid_cfg = 0
    while not is_valid_cfg :
        try :
                inputstring = 'Enter your choice [1-' + count +']: '
                n = int ( raw_input( inputstring ) )

                if n < count and n > 0:
				    is_valid_cfg = 1 ## set it to 1 to validate input and to terminate the while..not loop
                else:
                    print ( "'%s' is not a valid menu option.") % n
        except ValueError, e :
                print ("'%s' is not a valid integer." % e.args[0].split(": ")[1])
   # n = input( "your choice> " )
    cfg.Do( n - 1 )
<|endoftext|>