<commit_before># Lint as: python2, python3
# Copyright 2019, The TensorFlow Federated Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Utils that support unit tests in this component."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import collections

import numpy as np
import tensorflow as tf

from tensorflow_federated.python import learning
from tensorflow_federated.python.core.api import computation_types
from tensorflow_federated.python.core.api import computations
from tensorflow_federated.python.core.api import intrinsics
from tensorflow_federated.python.core.api import placements
from tensorflow_federated.python.core.backends.mapreduce import canonical_form
from tensorflow_federated.python.core.impl.compiler import building_blocks
from tensorflow_federated.python.core.utils import computation_utils

def get_temperature_sensor_example():
  """Constructs `canonical_form.CanonicalForm` for temperature sensors example.

  The temperature sensor example computes the fraction of sensors that report
  temperatures over the threshold.

  Returns:
    An instance of `canonical_form.CanonicalForm`.
  """

  @computations.tf_computation
  def initialize():
    return {'num_rounds': tf.constant(0)}

  # The state of the server is a singleton tuple containing just the integer
  # counter `num_rounds`.
  server_state_type = computation_types.NamedTupleType([('num_rounds', tf.int32)
                                                       ])

  @computations.tf_computation(server_state_type)
  def prepare(state):
    return {'max_temperature': 32.0 + tf.cast(state.num_rounds, tf.float32)}

  # The initial state of the client is a singleton tuple containing a single
  # float `max_temperature`, which is the threshold received from the server.
  client_state_type = computation_types.NamedTupleType([('max_temperature',
                                                         tf.float32)])

  # The client data is a sequence of floats.
  client_data_type = computation_types.SequenceType(tf.float32)

  @computations.tf_computation(client_data_type, client_state_type)
  def work(data, state):
    """See the `canonical_form.CanonicalForm` definition of `work`."""

    def fn(s, x):
      return {
          'num': s['num'] + 1,
          'max': tf.maximum(s['max'], x),
      }

    reduce_result = data.reduce({
        'num': np.int32(0),
        'max': np.float32(-459.67)
    }, fn)
    return ({
        'is_over': reduce_result['max'] > state.max_temperature
    }, {
        'num_readings': reduce_result['num']
    })

  # The client update is a singleton tuple with a Boolean-typed `is_over`.
  client_update_type = computation_types.NamedTupleType([('is_over', tf.bool)])

  # The accumulator for client updates is a pair of counters, one for the
  # number of clients over threshold, and the other for the total number of
  # client updates processed so far.
  accumulator_type = computation_types.NamedTupleType([('num_total', tf.int32),
                                                       ('num_over', tf.int32)])

  @computations.tf_computation
  def zero():
    return collections.OrderedDict([('num_total', tf.constant(0)),
                                    ('num_over', tf.constant(0))])

  @computations.tf_computation(accumulator_type, client_update_type)
  def accumulate(accumulator, update):
    return collections.OrderedDict([
        ('num_total', accumulator.num_total + 1),
        ('num_over', accumulator.num_over + tf.cast(update.is_over, tf.int32))
    ])

  @computations.tf_computation(accumulator_type, accumulator_type)
  def merge(accumulator1, accumulator2):
    return collections.OrderedDict([
        ('num_total', accumulator1.num_total + accumulator2.num_total),
        ('num_over', accumulator1.num_over + accumulator2.num_over)
    ])

  @computations.tf_computation(merge.type_signature.result)
  def report(accumulator):
    return {
        'ratio_over_threshold': (tf.cast(accumulator['num_over'], tf.float32) /
                                 tf.cast(accumulator['num_total'], tf.float32))
    }

  # The type of the combined update is a singleton tuple containing a float
  # named `ratio_over_threshold`.
  combined_update_type = computation_types.NamedTupleType([
      ('ratio_over_threshold', tf.float32)
  ])

  @computations.tf_computation(server_state_type, combined_update_type)
  def update(state, update):
    return ({'num_rounds': state.num_rounds + 1}, update)

  return canonical_form.CanonicalForm(initialize, prepare, work, zero,
                                      accumulate, merge, report, update)

def get_mnist_training_example():
  """Constructs `canonical_form.CanonicalForm` for mnist training.

  Returns:
    An instance of `canonical_form.CanonicalForm`.
  """
  model_nt = collections.namedtuple('Model', 'weights bias')
  server_state_nt = (collections.namedtuple('ServerState', 'model num_rounds'))

  # Start with a model filled with zeros, and the round counter set to zero.
  @computations.tf_computation
  def initialize():
    return server_state_nt(
        model=model_nt(weights=tf.zeros([784, 10]), bias=tf.zeros([10])),
        num_rounds=tf.constant(0))

  server_state_tff_type = server_state_nt(
      model=model_nt(weights=(tf.float32, [784, 10]), bias=(tf.float32, [10])),
      num_rounds=tf.int32)
  client_state_nt = (
      collections.namedtuple('ClientState', 'model learning_rate'))

  # Pass the model to the client, along with a dynamically adjusted learning
  # rate that starts at 0.1 and decays exponentially by a factor of 0.9.
  @computations.tf_computation(server_state_tff_type)
  def prepare(state):
    learning_rate = 0.1 * tf.pow(0.9, tf.cast(state.num_rounds, tf.float32))
    return client_state_nt(model=state.model, learning_rate=learning_rate)

  batch_nt = collections.namedtuple('Batch', 'x y')
  batch_tff_type = batch_nt(x=(tf.float32, [None, 784]), y=(tf.int32, [None]))
  dataset_tff_type = computation_types.SequenceType(batch_tff_type)
  model_tff_type = model_nt(
      weights=(tf.float32, [784, 10]), bias=(tf.float32, [10]))
  client_state_tff_type = client_state_nt(
      model=model_tff_type, learning_rate=tf.float32)
  loop_state_nt = collections.namedtuple('LoopState', 'num_examples total_loss')
  update_nt = collections.namedtuple('Update', 'model num_examples loss')
  stats_nt = collections.namedtuple('Stats', 'num_examples loss')

  # Train the model locally, emit the loclaly-trained model and the number of
  # examples as an update, and the average loss and the number of examples as
  # local client stats.
  @computations.tf_computation(dataset_tff_type, client_state_tff_type)
  def work(data, state):  # pylint: disable=missing-docstring
    model_vars = model_nt(
        weights=tf.Variable(initial_value=state.model.weights, name='weights'),
        bias=tf.Variable(initial_value=state.model.bias, name='bias'))

    with tf.control_dependencies([tf.global_variables_initializer()]):
      init_model = tf.group(
          tf.assign(model_vars.weights, state.model.weights),
          tf.assign(model_vars.bias, state.model.bias))

    optimizer = tf.train.GradientDescentOptimizer(state.learning_rate)

    @tf.function
    def reduce_fn(loop_state, batch):
      pred_y = tf.nn.softmax(
          tf.matmul(batch.x, model_vars.weights) + model_vars.bias)
      loss = -tf.reduce_mean(
          tf.reduce_sum(
              tf.one_hot(batch.y, 10) * tf.log(pred_y), reduction_indices=[1]))
      with tf.control_dependencies([optimizer.minimize(loss)]):
        return loop_state_nt(
            num_examples=loop_state.num_examples + 1,
            total_loss=loop_state.total_loss + loss)

    with tf.control_dependencies([init_model]):
      loop_state = data.reduce(
          loop_state_nt(num_examples=0, total_loss=np.float32(0.0)), reduce_fn)
      num_examples = loop_state.num_examples
      total_loss = loop_state.total_loss
      with tf.control_dependencies([num_examples, total_loss]):
        loss = total_loss / tf.cast(num_examples, tf.float32)

    return (update_nt(model=model_vars, num_examples=num_examples, loss=loss),
            stats_nt(num_examples=num_examples, loss=loss))

  accumulator_nt = update_nt

  # Initialize accumulators for aggregation with zero model and zero examples.
  @computations.tf_computation
  def zero():
    return accumulator_nt(
        model=model_nt(weights=tf.zeros([784, 10]), bias=tf.zeros([10])),
        num_examples=tf.constant(0),
        loss=tf.constant(0.0, dtype=tf.float32))

  update_tff_type = update_nt(
      model=model_tff_type, num_examples=tf.int32, loss=tf.float32)
  accumulator_tff_type = update_tff_type

  # We add an update to an accumulator with the update's model multipled by the
  # number of examples, so we can compute a weighted average in the end.
  @computations.tf_computation(accumulator_tff_type, update_tff_type)
  def accumulate(accumulator, update):
    scaling_factor = tf.cast(update.num_examples, tf.float32)
    scaled_model = tf.nest.map_structure(lambda x: x * scaling_factor,
                                         update.model)
    return accumulator_nt(
        model=tf.nest.map_structure(tf.add, accumulator.model, scaled_model),
        num_examples=accumulator.num_examples + update.num_examples,
        loss=accumulator.loss + update.loss * scaling_factor)

  # Merging accumulators does not involve scaling.
  @computations.tf_computation(accumulator_tff_type, accumulator_tff_type)
  def merge(accumulator1, accumulator2):
    return accumulator_nt(
        model=tf.nest.map_structure(tf.add, accumulator1.model,
                                    accumulator2.model),
        num_examples=accumulator1.num_examples + accumulator2.num_examples,
        loss=accumulator1.loss + accumulator2.loss)

  report_nt = accumulator_nt

  # The result of aggregation is produced by dividing the accumulated model by
  # the total number of examples. Same for loss.
  @computations.tf_computation(accumulator_tff_type)
  def report(accumulator):
    scaling_factor = 1.0 / tf.cast(accumulator.num_examples, tf.float32)
    scaled_model = model_nt(
        weights=accumulator.model.weights * scaling_factor,
        bias=accumulator.model.bias * scaling_factor)
    return report_nt(
        model=scaled_model,
        num_examples=accumulator.num_examples,
        loss=accumulator.loss * scaling_factor)

  report_tff_type = accumulator_tff_type
  metrics_nt = collections.namedtuple('Metrics', 'num_rounds num_examples loss')

  # Pass the newly averaged model along with an incremented round counter over
  # to the next round, and output the counters and loss as server metrics.
  @computations.tf_computation(server_state_tff_type, report_tff_type)
  def update(state, report):
    num_rounds = state.num_rounds + 1
    return (server_state_nt(model=report.model, num_rounds=num_rounds),
            metrics_nt(
                num_rounds=num_rounds,
                num_examples=report.num_examples,
                loss=report.loss))

  return canonical_form.CanonicalForm(initialize, prepare, work, zero,
                                      accumulate, merge, report, update)

def construct_example_training_comp():
  """Constructs a `computation_utils.IterativeProcess` via the FL API."""
  np.random.seed(0)

  sample_batch = collections.OrderedDict([('x',
                                           np.array([[1., 1.]],
                                                    dtype=np.float32)),
                                          ('y', np.array([[0]],
                                                         dtype=np.int32))])

  def model_fn():
    """Constructs keras model."""
    keras_model = tf.keras.models.Sequential([
        tf.keras.layers.Dense(
            1,
            activation=tf.nn.softmax,
            kernel_initializer='zeros',
            input_shape=(2,))
    ])

    def loss_fn(y_true, y_pred):
      return tf.reduce_mean(
          tf.keras.losses.sparse_categorical_crossentropy(y_true, y_pred))

    keras_model.compile(
        loss=loss_fn,
        optimizer=tf.keras.optimizers.SGD(learning_rate=0.01),
        metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])
    return learning.from_compiled_keras_model(keras_model, sample_batch)

  return learning.build_federated_averaging_process(model_fn)

def computation_to_building_block(comp):
  return building_blocks.ComputationBuildingBlock.from_proto(
      comp._computation_proto)  # pylint: disable=protected-access

def get_iterative_process_for_canonical_form_example():
  """Construct a simple `IterativeProcess` compatible with `CanonicalForm`.

  The computation itself is non-sensical; but demonstrates the required type
  signatures for `CanonicalForm```.

  Returns:
    An `IterativeProcess` compatible with `CanonicalForm`.
  """

  @computations.tf_computation(tf.int32, tf.float32)
  def add_two(x_int, y_float):
    return tf.cast(x_int, tf.float32) + y_float

  @computations.federated_computation
  def init_fn():
    return intrinsics.federated_value(1.234, placements.SERVER)

  @computations.federated_computation([
      computation_types.FederatedType(tf.float32, placements.SERVER),
      computation_types.FederatedType(tf.int32, placements.CLIENTS)
  ])
  def next_fn(server_val, client_val):
    """Defines a series of federated computations compatible with CanonicalForm."""
    broadcast_val = intrinsics.federated_broadcast(server_val)
    values_on_clients = intrinsics.federated_zip((client_val, broadcast_val))
    result_on_clients = intrinsics.federated_map(add_two, values_on_clients)
    aggregated_result = intrinsics.federated_mean(result_on_clients)
    side_output = intrinsics.federated_value([1, 2, 3, 4, 5], placements.SERVER)
    return aggregated_result, side_output

  return computation_utils.IterativeProcess(init_fn, next_fn)
<commit_msg>Use compat.v1 API to be TF 2.0 compatible<commit_after># Lint as: python2, python3
# Copyright 2019, The TensorFlow Federated Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Utils that support unit tests in this component."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import collections

import numpy as np
import tensorflow as tf

from tensorflow_federated.python import learning
from tensorflow_federated.python.core.api import computation_types
from tensorflow_federated.python.core.api import computations
from tensorflow_federated.python.core.api import intrinsics
from tensorflow_federated.python.core.api import placements
from tensorflow_federated.python.core.backends.mapreduce import canonical_form
from tensorflow_federated.python.core.impl.compiler import building_blocks
from tensorflow_federated.python.core.utils import computation_utils

def get_temperature_sensor_example():
  """Constructs `canonical_form.CanonicalForm` for temperature sensors example.

  The temperature sensor example computes the fraction of sensors that report
  temperatures over the threshold.

  Returns:
    An instance of `canonical_form.CanonicalForm`.
  """

  @computations.tf_computation
  def initialize():
    return {'num_rounds': tf.constant(0)}

  # The state of the server is a singleton tuple containing just the integer
  # counter `num_rounds`.
  server_state_type = computation_types.NamedTupleType([('num_rounds', tf.int32)
                                                       ])

  @computations.tf_computation(server_state_type)
  def prepare(state):
    return {'max_temperature': 32.0 + tf.cast(state.num_rounds, tf.float32)}

  # The initial state of the client is a singleton tuple containing a single
  # float `max_temperature`, which is the threshold received from the server.
  client_state_type = computation_types.NamedTupleType([('max_temperature',
                                                         tf.float32)])

  # The client data is a sequence of floats.
  client_data_type = computation_types.SequenceType(tf.float32)

  @computations.tf_computation(client_data_type, client_state_type)
  def work(data, state):
    """See the `canonical_form.CanonicalForm` definition of `work`."""

    def fn(s, x):
      return {
          'num': s['num'] + 1,
          'max': tf.maximum(s['max'], x),
      }

    reduce_result = data.reduce({
        'num': np.int32(0),
        'max': np.float32(-459.67)
    }, fn)
    return ({
        'is_over': reduce_result['max'] > state.max_temperature
    }, {
        'num_readings': reduce_result['num']
    })

  # The client update is a singleton tuple with a Boolean-typed `is_over`.
  client_update_type = computation_types.NamedTupleType([('is_over', tf.bool)])

  # The accumulator for client updates is a pair of counters, one for the
  # number of clients over threshold, and the other for the total number of
  # client updates processed so far.
  accumulator_type = computation_types.NamedTupleType([('num_total', tf.int32),
                                                       ('num_over', tf.int32)])

  @computations.tf_computation
  def zero():
    return collections.OrderedDict([('num_total', tf.constant(0)),
                                    ('num_over', tf.constant(0))])

  @computations.tf_computation(accumulator_type, client_update_type)
  def accumulate(accumulator, update):
    return collections.OrderedDict([
        ('num_total', accumulator.num_total + 1),
        ('num_over', accumulator.num_over + tf.cast(update.is_over, tf.int32))
    ])

  @computations.tf_computation(accumulator_type, accumulator_type)
  def merge(accumulator1, accumulator2):
    return collections.OrderedDict([
        ('num_total', accumulator1.num_total + accumulator2.num_total),
        ('num_over', accumulator1.num_over + accumulator2.num_over)
    ])

  @computations.tf_computation(merge.type_signature.result)
  def report(accumulator):
    return {
        'ratio_over_threshold': (tf.cast(accumulator['num_over'], tf.float32) /
                                 tf.cast(accumulator['num_total'], tf.float32))
    }

  # The type of the combined update is a singleton tuple containing a float
  # named `ratio_over_threshold`.
  combined_update_type = computation_types.NamedTupleType([
      ('ratio_over_threshold', tf.float32)
  ])

  @computations.tf_computation(server_state_type, combined_update_type)
  def update(state, update):
    return ({'num_rounds': state.num_rounds + 1}, update)

  return canonical_form.CanonicalForm(initialize, prepare, work, zero,
                                      accumulate, merge, report, update)

def get_mnist_training_example():
  """Constructs `canonical_form.CanonicalForm` for mnist training.

  Returns:
    An instance of `canonical_form.CanonicalForm`.
  """
  model_nt = collections.namedtuple('Model', 'weights bias')
  server_state_nt = (collections.namedtuple('ServerState', 'model num_rounds'))

  # Start with a model filled with zeros, and the round counter set to zero.
  @computations.tf_computation
  def initialize():
    return server_state_nt(
        model=model_nt(weights=tf.zeros([784, 10]), bias=tf.zeros([10])),
        num_rounds=tf.constant(0))

  server_state_tff_type = server_state_nt(
      model=model_nt(weights=(tf.float32, [784, 10]), bias=(tf.float32, [10])),
      num_rounds=tf.int32)
  client_state_nt = (
      collections.namedtuple('ClientState', 'model learning_rate'))

  # Pass the model to the client, along with a dynamically adjusted learning
  # rate that starts at 0.1 and decays exponentially by a factor of 0.9.
  @computations.tf_computation(server_state_tff_type)
  def prepare(state):
    learning_rate = 0.1 * tf.pow(0.9, tf.cast(state.num_rounds, tf.float32))
    return client_state_nt(model=state.model, learning_rate=learning_rate)

  batch_nt = collections.namedtuple('Batch', 'x y')
  batch_tff_type = batch_nt(x=(tf.float32, [None, 784]), y=(tf.int32, [None]))
  dataset_tff_type = computation_types.SequenceType(batch_tff_type)
  model_tff_type = model_nt(
      weights=(tf.float32, [784, 10]), bias=(tf.float32, [10]))
  client_state_tff_type = client_state_nt(
      model=model_tff_type, learning_rate=tf.float32)
  loop_state_nt = collections.namedtuple('LoopState', 'num_examples total_loss')
  update_nt = collections.namedtuple('Update', 'model num_examples loss')
  stats_nt = collections.namedtuple('Stats', 'num_examples loss')

  # Train the model locally, emit the loclaly-trained model and the number of
  # examples as an update, and the average loss and the number of examples as
  # local client stats.
  @computations.tf_computation(dataset_tff_type, client_state_tff_type)
  def work(data, state):  # pylint: disable=missing-docstring
    model_vars = model_nt(
        weights=tf.Variable(initial_value=state.model.weights, name='weights'),
        bias=tf.Variable(initial_value=state.model.bias, name='bias'))

    with tf.control_dependencies([tf.compat.v1.global_variables_initializer()]):
      init_model = tf.group(
          tf.assign(model_vars.weights, state.model.weights),
          tf.assign(model_vars.bias, state.model.bias))

    optimizer = tf.train.GradientDescentOptimizer(state.learning_rate)

    @tf.function
    def reduce_fn(loop_state, batch):
      pred_y = tf.nn.softmax(
          tf.matmul(batch.x, model_vars.weights) + model_vars.bias)
      loss = -tf.reduce_mean(
          tf.reduce_sum(
              tf.one_hot(batch.y, 10) * tf.log(pred_y), reduction_indices=[1]))
      with tf.control_dependencies([optimizer.minimize(loss)]):
        return loop_state_nt(
            num_examples=loop_state.num_examples + 1,
            total_loss=loop_state.total_loss + loss)

    with tf.control_dependencies([init_model]):
      loop_state = data.reduce(
          loop_state_nt(num_examples=0, total_loss=np.float32(0.0)), reduce_fn)
      num_examples = loop_state.num_examples
      total_loss = loop_state.total_loss
      with tf.control_dependencies([num_examples, total_loss]):
        loss = total_loss / tf.cast(num_examples, tf.float32)

    return (update_nt(model=model_vars, num_examples=num_examples, loss=loss),
            stats_nt(num_examples=num_examples, loss=loss))

  accumulator_nt = update_nt

  # Initialize accumulators for aggregation with zero model and zero examples.
  @computations.tf_computation
  def zero():
    return accumulator_nt(
        model=model_nt(weights=tf.zeros([784, 10]), bias=tf.zeros([10])),
        num_examples=tf.constant(0),
        loss=tf.constant(0.0, dtype=tf.float32))

  update_tff_type = update_nt(
      model=model_tff_type, num_examples=tf.int32, loss=tf.float32)
  accumulator_tff_type = update_tff_type

  # We add an update to an accumulator with the update's model multipled by the
  # number of examples, so we can compute a weighted average in the end.
  @computations.tf_computation(accumulator_tff_type, update_tff_type)
  def accumulate(accumulator, update):
    scaling_factor = tf.cast(update.num_examples, tf.float32)
    scaled_model = tf.nest.map_structure(lambda x: x * scaling_factor,
                                         update.model)
    return accumulator_nt(
        model=tf.nest.map_structure(tf.add, accumulator.model, scaled_model),
        num_examples=accumulator.num_examples + update.num_examples,
        loss=accumulator.loss + update.loss * scaling_factor)

  # Merging accumulators does not involve scaling.
  @computations.tf_computation(accumulator_tff_type, accumulator_tff_type)
  def merge(accumulator1, accumulator2):
    return accumulator_nt(
        model=tf.nest.map_structure(tf.add, accumulator1.model,
                                    accumulator2.model),
        num_examples=accumulator1.num_examples + accumulator2.num_examples,
        loss=accumulator1.loss + accumulator2.loss)

  report_nt = accumulator_nt

  # The result of aggregation is produced by dividing the accumulated model by
  # the total number of examples. Same for loss.
  @computations.tf_computation(accumulator_tff_type)
  def report(accumulator):
    scaling_factor = 1.0 / tf.cast(accumulator.num_examples, tf.float32)
    scaled_model = model_nt(
        weights=accumulator.model.weights * scaling_factor,
        bias=accumulator.model.bias * scaling_factor)
    return report_nt(
        model=scaled_model,
        num_examples=accumulator.num_examples,
        loss=accumulator.loss * scaling_factor)

  report_tff_type = accumulator_tff_type
  metrics_nt = collections.namedtuple('Metrics', 'num_rounds num_examples loss')

  # Pass the newly averaged model along with an incremented round counter over
  # to the next round, and output the counters and loss as server metrics.
  @computations.tf_computation(server_state_tff_type, report_tff_type)
  def update(state, report):
    num_rounds = state.num_rounds + 1
    return (server_state_nt(model=report.model, num_rounds=num_rounds),
            metrics_nt(
                num_rounds=num_rounds,
                num_examples=report.num_examples,
                loss=report.loss))

  return canonical_form.CanonicalForm(initialize, prepare, work, zero,
                                      accumulate, merge, report, update)

def construct_example_training_comp():
  """Constructs a `computation_utils.IterativeProcess` via the FL API."""
  np.random.seed(0)

  sample_batch = collections.OrderedDict([('x',
                                           np.array([[1., 1.]],
                                                    dtype=np.float32)),
                                          ('y', np.array([[0]],
                                                         dtype=np.int32))])

  def model_fn():
    """Constructs keras model."""
    keras_model = tf.keras.models.Sequential([
        tf.keras.layers.Dense(
            1,
            activation=tf.nn.softmax,
            kernel_initializer='zeros',
            input_shape=(2,))
    ])

    def loss_fn(y_true, y_pred):
      return tf.reduce_mean(
          tf.keras.losses.sparse_categorical_crossentropy(y_true, y_pred))

    keras_model.compile(
        loss=loss_fn,
        optimizer=tf.keras.optimizers.SGD(learning_rate=0.01),
        metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])
    return learning.from_compiled_keras_model(keras_model, sample_batch)

  return learning.build_federated_averaging_process(model_fn)

def computation_to_building_block(comp):
  return building_blocks.ComputationBuildingBlock.from_proto(
      comp._computation_proto)  # pylint: disable=protected-access

def get_iterative_process_for_canonical_form_example():
  """Construct a simple `IterativeProcess` compatible with `CanonicalForm`.

  The computation itself is non-sensical; but demonstrates the required type
  signatures for `CanonicalForm```.

  Returns:
    An `IterativeProcess` compatible with `CanonicalForm`.
  """

  @computations.tf_computation(tf.int32, tf.float32)
  def add_two(x_int, y_float):
    return tf.cast(x_int, tf.float32) + y_float

  @computations.federated_computation
  def init_fn():
    return intrinsics.federated_value(1.234, placements.SERVER)

  @computations.federated_computation([
      computation_types.FederatedType(tf.float32, placements.SERVER),
      computation_types.FederatedType(tf.int32, placements.CLIENTS)
  ])
  def next_fn(server_val, client_val):
    """Defines a series of federated computations compatible with CanonicalForm."""
    broadcast_val = intrinsics.federated_broadcast(server_val)
    values_on_clients = intrinsics.federated_zip((client_val, broadcast_val))
    result_on_clients = intrinsics.federated_map(add_two, values_on_clients)
    aggregated_result = intrinsics.federated_mean(result_on_clients)
    side_output = intrinsics.federated_value([1, 2, 3, 4, 5], placements.SERVER)
    return aggregated_result, side_output

  return computation_utils.IterativeProcess(init_fn, next_fn)
<|endoftext|>


<commit_before># Django
from django.conf import settings
from django.core.cache import cache
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.template import Library, Node
from django.utils.translation import ugettext_lazy as _

# AstroBin apps
from astrobin.models import Image, UserProfile
from astrobin_apps_premium.utils import premium_user_has_valid_subscription

# Third party apps
from toggleproperties.models import ToggleProperty

register = Library()

@register.inclusion_tag('astrobin_apps_users/inclusion_tags/astrobin_username.html')
def astrobin_username(user, **kwargs):
    if not hasattr(user, 'userprofile'):
        try:
            user = UserProfile.objects.get(user__username = user).user
        except UserProfile.DoesNotExist:
            return {'user': None}

    cache_key = 'user_metadata_' + user.username
    user_metadata = cache.get(cache_key)
    if user_metadata is None:
        user_metadata = {}
        user_metadata['display_name'] = user.userprofile.get_display_name()
        user_metadata['is_superuser'] = user.is_superuser
        user_metadata['is_image_moderator'] = user.userprofile.is_image_moderator()
        user_metadata['is_iotd_staff'] = user.userprofile.is_iotd_staff()
        user_metadata['is_iotd_submitter'] = user.userprofile.is_iotd_submitter()
        user_metadata['is_iotd_reviewer'] = user.userprofile.is_iotd_reviewer()
        user_metadata['is_iotd_judge'] = user.userprofile.is_iotd_judge()
        user_metadata['is_premium'] = premium_user_has_valid_subscription(user)
        cache.set(cache_key, user_metadata, 600)
    else:
        display_name = user_metadata['display_name']
        is_superuser = user_metadata['is_superuser']
        is_image_moderator = user_metadata['is_image_moderator']
        is_iotd_staff = user_metadata['is_iotd_staff']
        is_iotd_submitter = user_metadata['is_iotd_submitter']
        is_iotd_reviewer = user_metadata['is_iotd_reviewer']
        is_iotd_judge = user_metadata['is_iotd_judge']
        is_premium = user_metadata['is_premium']

    classes = ['astrobin-username']
    titles = []

    if user_metadata['is_superuser']:
        classes.append('astrobin-superuser')
        titles.append(_('Administrator'))

    if user_metadata['is_image_moderator']:
        classes.append('astrobin-image-moderator')
        titles.append(_('Image moderator'))

    if user_metadata['is_iotd_staff']:
        classes.append(' astrobin-iotd-staff')

    if user_metadata['is_iotd_submitter']:
        classes.append(' astrobin-iotd-submitter')
        titles.append(_('IOTD Submitter'))

    if user_metadata['is_iotd_reviewer']:
        classes.append(' astrobin-iotd-reviewer')
        titles.append(_('IOTD Reviewer'))

    if user_metadata['is_iotd_judge']:
        classes.append(' astrobin-iotd-judge')
        titles.append(_('IOTD Judge'))

    if user_metadata['is_premium']:
        classes.append(' astrobin-premium-member')
        titles.append(_('Premium member'))

    context = user_metadata.copy()
    context.update({
        'user': user,
        'classes': classes,
        'titles': titles,
        'link': kwargs.get('link', True),
    })

    return context

@register.inclusion_tag('astrobin_apps_users/inclusion_tags/astrobin_user.html', takes_context = True)
def astrobin_user(context, user, **kwargs):
    user_ct = ContentType.objects.get_for_model(User)
    images = Image.objects.filter(user = user).count()
    followers = ToggleProperty.objects.toggleproperties_for_object("follow", user).count()
    following = ToggleProperty.objects.filter(
        property_type = "follow",
        user = user,
        content_type = user_ct).count()

    request = context['request']
    request_user = None
    if request.user.is_authenticated():
        request_user = UserProfile.objects.get(user=request.user).user

    view = kwargs.get('view')
    if view is None and 'view' in context:
        view = context['view']
    if view is None:
        view = request.GET.get('view', 'default'),

    layout = kwargs.get('layout')
    if layout is None and 'layout' in context:
        layout = context['layout']
    if layout is None:
        layout = request.GET.get('layout', 'default')

    return {
        'request': request,
        'user': user,
        'request_user': request_user,
        'view': view,
        'layout': layout,
        'images': images,
        'followers': followers,
        'following': following,

        'DONATIONS_ENABLED': context['DONATIONS_ENABLED'],
        'PREMIUM_ENABLED': context['PREMIUM_ENABLED'],
    }

@register.inclusion_tag('astrobin_apps_users/inclusion_tags/user_list.html', takes_context = True)
def astrobin_apps_users_list(context, user_list, **kwargs):
    request = context['request']

    request_user = None
    if request.user.is_authenticated():
        request_user = UserProfile.objects.get(user=request.user).user

    view = kwargs.get('view')
    if view is None and 'view' in context:
        view = context['view']
    if view is None:
        view = request.GET.get('view', 'default'),

    layout = kwargs.get('layout')
    if layout is None and 'layout' in context:
        layout = context['layout']
    if layout is None:
        layout = request.GET.get('layout', 'default')

    return {
        'request': request,
        'request_user': request_user,
        'user_list': user_list,
        'view': view,
        'layout': layout,
        'DONATIONS_ENABLED': context['DONATIONS_ENABLED'],
        'PREMIUM_ENABLED': context['PREMIUM_ENABLED'],
    }

<commit_msg>(profile) Exclude corrupted images from user tag<commit_after># Django
from django.conf import settings
from django.core.cache import cache
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.template import Library, Node
from django.utils.translation import ugettext_lazy as _

# AstroBin apps
from astrobin.models import Image, UserProfile
from astrobin_apps_premium.utils import premium_user_has_valid_subscription

# Third party apps
from toggleproperties.models import ToggleProperty

register = Library()

@register.inclusion_tag('astrobin_apps_users/inclusion_tags/astrobin_username.html')
def astrobin_username(user, **kwargs):
    if not hasattr(user, 'userprofile'):
        try:
            user = UserProfile.objects.get(user__username = user).user
        except UserProfile.DoesNotExist:
            return {'user': None}

    cache_key = 'user_metadata_' + user.username
    user_metadata = cache.get(cache_key)
    if user_metadata is None:
        user_metadata = {}
        user_metadata['display_name'] = user.userprofile.get_display_name()
        user_metadata['is_superuser'] = user.is_superuser
        user_metadata['is_image_moderator'] = user.userprofile.is_image_moderator()
        user_metadata['is_iotd_staff'] = user.userprofile.is_iotd_staff()
        user_metadata['is_iotd_submitter'] = user.userprofile.is_iotd_submitter()
        user_metadata['is_iotd_reviewer'] = user.userprofile.is_iotd_reviewer()
        user_metadata['is_iotd_judge'] = user.userprofile.is_iotd_judge()
        user_metadata['is_premium'] = premium_user_has_valid_subscription(user)
        cache.set(cache_key, user_metadata, 600)
    else:
        display_name = user_metadata['display_name']
        is_superuser = user_metadata['is_superuser']
        is_image_moderator = user_metadata['is_image_moderator']
        is_iotd_staff = user_metadata['is_iotd_staff']
        is_iotd_submitter = user_metadata['is_iotd_submitter']
        is_iotd_reviewer = user_metadata['is_iotd_reviewer']
        is_iotd_judge = user_metadata['is_iotd_judge']
        is_premium = user_metadata['is_premium']

    classes = ['astrobin-username']
    titles = []

    if user_metadata['is_superuser']:
        classes.append('astrobin-superuser')
        titles.append(_('Administrator'))

    if user_metadata['is_image_moderator']:
        classes.append('astrobin-image-moderator')
        titles.append(_('Image moderator'))

    if user_metadata['is_iotd_staff']:
        classes.append(' astrobin-iotd-staff')

    if user_metadata['is_iotd_submitter']:
        classes.append(' astrobin-iotd-submitter')
        titles.append(_('IOTD Submitter'))

    if user_metadata['is_iotd_reviewer']:
        classes.append(' astrobin-iotd-reviewer')
        titles.append(_('IOTD Reviewer'))

    if user_metadata['is_iotd_judge']:
        classes.append(' astrobin-iotd-judge')
        titles.append(_('IOTD Judge'))

    if user_metadata['is_premium']:
        classes.append(' astrobin-premium-member')
        titles.append(_('Premium member'))

    context = user_metadata.copy()
    context.update({
        'user': user,
        'classes': classes,
        'titles': titles,
        'link': kwargs.get('link', True),
    })

    return context

@register.inclusion_tag('astrobin_apps_users/inclusion_tags/astrobin_user.html', takes_context = True)
def astrobin_user(context, user, **kwargs):
    user_ct = ContentType.objects.get_for_model(User)
    images = Image.objects.filter(user = user).exclude(corrupted=True).count()
    followers = ToggleProperty.objects.toggleproperties_for_object("follow", user).count()
    following = ToggleProperty.objects.filter(
        property_type = "follow",
        user = user,
        content_type = user_ct).count()

    request = context['request']
    request_user = None
    if request.user.is_authenticated():
        request_user = UserProfile.objects.get(user=request.user).user

    view = kwargs.get('view')
    if view is None and 'view' in context:
        view = context['view']
    if view is None:
        view = request.GET.get('view', 'default'),

    layout = kwargs.get('layout')
    if layout is None and 'layout' in context:
        layout = context['layout']
    if layout is None:
        layout = request.GET.get('layout', 'default')

    return {
        'request': request,
        'user': user,
        'request_user': request_user,
        'view': view,
        'layout': layout,
        'images': images,
        'followers': followers,
        'following': following,

        'DONATIONS_ENABLED': context['DONATIONS_ENABLED'],
        'PREMIUM_ENABLED': context['PREMIUM_ENABLED'],
    }

@register.inclusion_tag('astrobin_apps_users/inclusion_tags/user_list.html', takes_context = True)
def astrobin_apps_users_list(context, user_list, **kwargs):
    request = context['request']

    request_user = None
    if request.user.is_authenticated():
        request_user = UserProfile.objects.get(user=request.user).user

    view = kwargs.get('view')
    if view is None and 'view' in context:
        view = context['view']
    if view is None:
        view = request.GET.get('view', 'default'),

    layout = kwargs.get('layout')
    if layout is None and 'layout' in context:
        layout = context['layout']
    if layout is None:
        layout = request.GET.get('layout', 'default')

    return {
        'request': request,
        'request_user': request_user,
        'user_list': user_list,
        'view': view,
        'layout': layout,
        'DONATIONS_ENABLED': context['DONATIONS_ENABLED'],
        'PREMIUM_ENABLED': context['PREMIUM_ENABLED'],
    }

<|endoftext|>


<commit_before># This file is part of the GOsa framework.
#
#  http://gosa-project.org
#
# Copyright:
#  (C) 2016 GONICUS GmbH, Germany, http://www.gonicus.de
#
# See the LICENSE file in the project's top-level directory for details.

import pytest
import crypt
from unittest import mock
from gosa.backend.components.jsonrpc_objects import JSONRPCObjectMapper, ObjectRegistry
from gosa.common.components import PluginRegistry
from tests.GosaTestCase import *

@slow
class JSONRPCObjectMapperTestCase(GosaTestCase):

    def setUp(self):
        super(JSONRPCObjectMapperTestCase, self).setUp()
        self.mapper = JSONRPCObjectMapper()
        self.mocked_resolver = mock.MagicMock()
        self.mocked_resolver.return_value.check.return_value = True
        self.patcher = mock.patch.dict(PluginRegistry.modules, {'ACLResolver': self.mocked_resolver})
        self.patcher.start()

    def tearDown(self):
        self.patcher.stop()
        super(JSONRPCObjectMapperTestCase, self).tearDown()

    def test_listObjectOIDs(self):
        res = self.mapper.listObjectOIDs()
        assert 'object' in res
        assert len(res) == 1

    def test_openObject(self):
        res = self.mapper.openObject('admin','object','dc=example,dc=net')
        assert res['dc'] == "example"

        with pytest.raises(Exception):
            self.mapper.openObject('admin', 'object', 'dc=example,dc=net')

    def test_closeObject(self):
        res = self.mapper.openObject('admin', 'object', 'dc=example,dc=net')

        with pytest.raises(ValueError):
            self.mapper.closeObject('admin','unknown')

        with pytest.raises(ValueError):
            self.mapper.closeObject('someone else', res["__jsonclass__"][1][1])

        self.mapper.closeObject('admin', res["__jsonclass__"][1][1])

        # as a workaround for checking if its not loaded anymore we try to reload it
        with pytest.raises(ValueError):
            self.mapper.reloadObject('admin', res["__jsonclass__"][1][1])

    def test_getObjectProperty(self):
        res = self.mapper.openObject('admin', 'object', 'dc=example,dc=net')
        ref = res["__jsonclass__"][1][1]

        with pytest.raises(ValueError):
            self.mapper.getObjectProperty('admin', 'unknown', 'prop')

        with pytest.raises(ValueError):
            self.mapper.getObjectProperty('admin', ref, 'prop')

        with pytest.raises(ValueError):
            self.mapper.getObjectProperty('someone else', ref, 'description')

        assert self.mapper.getObjectProperty('admin', ref, 'description') == "Example"

    def test_setObjectProperty(self):
        res = self.mapper.openObject('admin', 'object', 'cn=Frank Reich,ou=people,dc=example,dc=net')
        ref = res["__jsonclass__"][1][1]

        with pytest.raises(ValueError):
            self.mapper.setObjectProperty('admin', 'unknown', 'prop', 'val')

        with pytest.raises(ValueError):
            self.mapper.setObjectProperty('admin', ref, 'prop', 'val')

        with pytest.raises(ValueError):
            self.mapper.setObjectProperty('someone else', ref, 'description', 'val')

        self.mapper.setObjectProperty('admin', ref, 'uid', 'val')
        assert self.mapper.getObjectProperty('admin', ref, 'uid') == "val"

        #undo
        self.mapper.setObjectProperty('admin', ref, 'uid', 'admin')
        assert self.mapper.getObjectProperty('admin', ref, 'uid') == "admin"

    def test_reloadObjectProperty(self):
        res = self.mapper.openObject('admin', 'object', 'dc=example,dc=net')
        uuid = res['uuid']
        ref = res["__jsonclass__"][1][1]

        with pytest.raises(ValueError):
            self.mapper.reloadObject('someone else', ref)

        res = self.mapper.reloadObject('admin', ref)
        assert uuid == res['uuid']
        assert ref != res["__jsonclass__"][1][1]

    def test_dispatchObjectMethod(self):
        res = self.mapper.openObject('admin', 'object', 'cn=Frank Reich,ou=people,dc=example,dc=net')
        ref = res["__jsonclass__"][1][1]

        with pytest.raises(ValueError):
            self.mapper.dispatchObjectMethod('admin','wrong_ref','lock')

        with pytest.raises(ValueError):
            self.mapper.dispatchObjectMethod('admin', ref, 'wrong_method')

        with pytest.raises(ValueError):
            self.mapper.dispatchObjectMethod('someone_else', ref, 'lock')

        # mock a method in the object

        with mock.patch('gosa.backend.plugins.password.manager.ObjectProxy') as m:
            user = m.return_value
            user.passwordMethod = crypt.METHOD_MD5
            self.mapper.dispatchObjectMethod('admin', ref, 'changePassword','Test')
            assert user.userPassword
            assert user.commit.called

    def test_diffObject(self):
        assert self.mapper.diffObject('admin','unkown_ref') is None

        res = self.mapper.openObject('admin', 'object', 'cn=Frank Reich,ou=people,dc=example,dc=net')
        ref = res["__jsonclass__"][1][1]

        with pytest.raises(ValueError):
            self.mapper.diffObject('someone_else', ref)

        self.mapper.setObjectProperty('admin', ref, 'uid', 'val')
        delta = self.mapper.diffObject('admin', ref)
        assert 'uid' in delta['attributes']['changed']

    def test_removeObject(self):
        res = self.mapper.openObject('admin', 'object', 'cn=Frank Reich,ou=people,dc=example,dc=net')
        ref = res["__jsonclass__"][1][1]

        with pytest.raises(Exception):
            self.mapper.removeObject('admin','object', 'cn=Frank Reich,ou=people,dc=example,dc=net')

        self.mapper.closeObject('admin', ref)

        with mock.patch.dict(ObjectRegistry.objects['object'], {'object': mock.MagicMock()}):
            mockedObject = ObjectRegistry.objects['object']['object'].return_value
            self.mapper.removeObject('admin', 'object', 'cn=Frank Reich,ou=people,dc=example,dc=net')
            assert mockedObject.remove.called
<commit_msg>fix test<commit_after># This file is part of the GOsa framework.
#
#  http://gosa-project.org
#
# Copyright:
#  (C) 2016 GONICUS GmbH, Germany, http://www.gonicus.de
#
# See the LICENSE file in the project's top-level directory for details.

import pytest
import crypt
from unittest import mock
from gosa.backend.components.jsonrpc_objects import JSONRPCObjectMapper, ObjectRegistry
from gosa.common.components import PluginRegistry
from tests.GosaTestCase import *

@slow
class JSONRPCObjectMapperTestCase(GosaTestCase):

    def setUp(self):
        super(JSONRPCObjectMapperTestCase, self).setUp()
        self.mapper = JSONRPCObjectMapper()
        self.mocked_resolver = mock.MagicMock()
        self.mocked_resolver.return_value.check.return_value = True
        self.patcher = mock.patch.dict(PluginRegistry.modules, {'ACLResolver': self.mocked_resolver})
        self.patcher.start()

    def tearDown(self):
        self.patcher.stop()
        super(JSONRPCObjectMapperTestCase, self).tearDown()

    def test_listObjectOIDs(self):
        res = self.mapper.listObjectOIDs()
        assert 'object' in res
        assert len(res) == 1

    def test_openObject(self):
        res = self.mapper.openObject('admin','object','dc=example,dc=net')
        assert res['dc'] == "example"

        with pytest.raises(Exception):
            self.mapper.openObject('admin', 'object', 'dc=example,dc=net')

    def test_closeObject(self):
        res = self.mapper.openObject('admin', 'object', 'dc=example,dc=net')

        with pytest.raises(ValueError):
            self.mapper.closeObject('admin','unknown')

        with pytest.raises(ValueError):
            self.mapper.closeObject('someone else', res["__jsonclass__"][1][1])

        self.mapper.closeObject('admin', res["__jsonclass__"][1][1])

        # as a workaround for checking if its not loaded anymore we try to reload it
        with pytest.raises(ValueError):
            self.mapper.reloadObject('admin', res["__jsonclass__"][1][1])

    def test_getObjectProperty(self):
        res = self.mapper.openObject('admin', 'object', 'dc=example,dc=net')
        ref = res["__jsonclass__"][1][1]

        with pytest.raises(ValueError):
            self.mapper.getObjectProperty('admin', 'unknown', 'prop')

        with pytest.raises(ValueError):
            self.mapper.getObjectProperty('admin', ref, 'prop')

        with pytest.raises(ValueError):
            self.mapper.getObjectProperty('someone else', ref, 'description')

        assert self.mapper.getObjectProperty('admin', ref, 'description') == "Example"

    def test_setObjectProperty(self):
        res = self.mapper.openObject('admin', 'object', 'cn=Frank Reich,ou=people,dc=example,dc=net')
        ref = res["__jsonclass__"][1][1]

        with pytest.raises(ValueError):
            self.mapper.setObjectProperty('admin', 'unknown', 'prop', 'val')

        with pytest.raises(ValueError):
            self.mapper.setObjectProperty('admin', ref, 'prop', 'val')

        with pytest.raises(ValueError):
            self.mapper.setObjectProperty('someone else', ref, 'description', 'val')

        self.mapper.setObjectProperty('admin', ref, 'uid', 'val')
        assert self.mapper.getObjectProperty('admin', ref, 'uid') == "val"

        #undo
        self.mapper.setObjectProperty('admin', ref, 'uid', 'admin')
        assert self.mapper.getObjectProperty('admin', ref, 'uid') == "admin"

    def test_reloadObjectProperty(self):
        res = self.mapper.openObject('admin', 'object', 'dc=example,dc=net')
        uuid = res['uuid']
        ref = res["__jsonclass__"][1][1]

        with pytest.raises(ValueError):
            self.mapper.reloadObject('someone else', ref)

        res = self.mapper.reloadObject('admin', ref)
        assert uuid == res['uuid']
        assert ref != res["__jsonclass__"][1][1]

    def test_dispatchObjectMethod(self):
        res = self.mapper.openObject('admin', 'object', 'cn=Frank Reich,ou=people,dc=example,dc=net')
        ref = res["__jsonclass__"][1][1]

        with pytest.raises(ValueError):
            self.mapper.dispatchObjectMethod('admin','wrong_ref','lock')

        with pytest.raises(ValueError):
            self.mapper.dispatchObjectMethod('admin', ref, 'wrong_method')

        with pytest.raises(ValueError):
            self.mapper.dispatchObjectMethod('someone_else', ref, 'lock')

        # mock a method in the object

        with mock.patch('gosa.backend.plugins.password.manager.ObjectProxy') as m:
            user = m.return_value
            user.passwordMethod = "MD5"
            self.mapper.dispatchObjectMethod('admin', ref, 'changePassword','Test')
            assert user.userPassword
            assert user.commit.called

    def test_diffObject(self):
        assert self.mapper.diffObject('admin','unkown_ref') is None

        res = self.mapper.openObject('admin', 'object', 'cn=Frank Reich,ou=people,dc=example,dc=net')
        ref = res["__jsonclass__"][1][1]

        with pytest.raises(ValueError):
            self.mapper.diffObject('someone_else', ref)

        self.mapper.setObjectProperty('admin', ref, 'uid', 'val')
        delta = self.mapper.diffObject('admin', ref)
        assert 'uid' in delta['attributes']['changed']

    def test_removeObject(self):
        res = self.mapper.openObject('admin', 'object', 'cn=Frank Reich,ou=people,dc=example,dc=net')
        ref = res["__jsonclass__"][1][1]

        with pytest.raises(Exception):
            self.mapper.removeObject('admin','object', 'cn=Frank Reich,ou=people,dc=example,dc=net')

        self.mapper.closeObject('admin', ref)

        with mock.patch.dict(ObjectRegistry.objects['object'], {'object': mock.MagicMock()}):
            mockedObject = ObjectRegistry.objects['object']['object'].return_value
            self.mapper.removeObject('admin', 'object', 'cn=Frank Reich,ou=people,dc=example,dc=net')
            assert mockedObject.remove.called
<|endoftext|>


<commit_before>"""Views for builds app."""

import logging
import textwrap
from urllib.parse import urlparse

from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.http import (
    HttpResponseForbidden,
    HttpResponseRedirect,
)
from django.shortcuts import get_object_or_404
from django.urls import reverse
from django.utils.decorators import method_decorator
from django.views.generic import DetailView, ListView
from requests.utils import quote

from readthedocs.builds.models import Build, Version
from readthedocs.core.permissions import AdminPermission
from readthedocs.core.utils import trigger_build
from readthedocs.doc_builder.exceptions import BuildEnvironmentError
from readthedocs.projects.models import Project

log = logging.getLogger(__name__)

class BuildBase:

    model = Build

    def get_queryset(self):
        self.project_slug = self.kwargs.get('project_slug', None)
        self.project = get_object_or_404(
            Project.objects.public(self.request.user),
            slug=self.project_slug,
        )
        queryset = Build.objects.public(
            user=self.request.user,
            project=self.project,
        ).select_related('project', 'version')

        return queryset

class BuildTriggerMixin:

    @method_decorator(login_required)
    def post(self, request, project_slug):
        project = get_object_or_404(Project, slug=project_slug)

        if not AdminPermission.is_admin(request.user, project):
            return HttpResponseForbidden()

        version_slug = request.POST.get('version_slug')
        commit = request.POST.get('commit')
        build_pk = request.POST.get('build_pk')

        version = get_object_or_404(
            self._get_versions(project),
            slug=version_slug,
        )

        # Set either the build or None
        build = Build.objects.filter(pk=build_pk).first()

        if build:

            log.info(
                'Rebuilding build. project=%s version=%s commit=%s build=%s',
                project.slug,
                version.slug,
                commit,
                build.pk
            )

        update_docs_task, build = trigger_build(
            project=project,
            version=version,
            build=build,
            commit=commit,
        )
        if (update_docs_task, build) == (None, None):
            # Build was skipped
            messages.add_message(
                request,
                messages.WARNING,
                "This project is currently disabled and can't trigger new builds.",
            )
            return HttpResponseRedirect(
                reverse('builds_project_list', args=[project.slug]),
            )

        return HttpResponseRedirect(
            reverse('builds_detail', args=[project.slug, build.pk]),
        )

    def _get_versions(self, project):
        return Version.internal.public(
            user=self.request.user,
            project=project,
        )

class BuildList(BuildBase, BuildTriggerMixin, ListView):

    def get_context_data(self, **kwargs):  # pylint: disable=arguments-differ
        context = super().get_context_data(**kwargs)

        active_builds = self.get_queryset().exclude(
            state='finished',
        ).values('id')

        context['project'] = self.project
        context['active_builds'] = active_builds
        context['versions'] = self._get_versions(self.project)
        context['build_qs'] = self.get_queryset()

        return context

class BuildDetail(BuildBase, DetailView):

    pk_url_kwarg = 'build_pk'

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['project'] = self.project

        build = self.get_object()
        if build.error != BuildEnvironmentError.GENERIC_WITH_BUILD_ID.format(build_id=build.pk):
            # Do not suggest to open an issue if the error is not generic
            return context

        scheme = (
            'https://github.com/rtfd/readthedocs.org/issues/new'
            '?title={title}{build_id}'
            '&body={body}'
        )

        # TODO: we could use ``.github/ISSUE_TEMPLATE.md`` here, but we would
        # need to add some variables to it which could impact in the UX when
        # filling an issue from the web
        body = """
        ## Details:

        * Project URL: https://readthedocs.org/projects/{project_slug}/
        * Build URL(if applicable): https://readthedocs.org{build_path}
        * Read the Docs username(if applicable): {username}

        ## Expected Result

        *A description of what you wanted to happen*

        ## Actual Result

        *A description of what actually happened*""".format(
            project_slug=self.project,
            build_path=self.request.path,
            username=self.request.user,
        )

        scheme_dict = {
            'title': quote('Build error with build id #'),
            'build_id': context['build'].id,
            'body': quote(textwrap.dedent(body)),
        }

        issue_url = scheme.format(**scheme_dict)
        issue_url = urlparse(issue_url).geturl()
        context['issue_url'] = issue_url
        return context
<commit_msg>Update readthedocs/builds/views.py<commit_after>"""Views for builds app."""

import logging
import textwrap
from urllib.parse import urlparse

from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.http import (
    HttpResponseForbidden,
    HttpResponseRedirect,
)
from django.shortcuts import get_object_or_404
from django.urls import reverse
from django.utils.decorators import method_decorator
from django.views.generic import DetailView, ListView
from requests.utils import quote

from readthedocs.builds.models import Build, Version
from readthedocs.core.permissions import AdminPermission
from readthedocs.core.utils import trigger_build
from readthedocs.doc_builder.exceptions import BuildEnvironmentError
from readthedocs.projects.models import Project

log = logging.getLogger(__name__)

class BuildBase:

    model = Build

    def get_queryset(self):
        self.project_slug = self.kwargs.get('project_slug', None)
        self.project = get_object_or_404(
            Project.objects.public(self.request.user),
            slug=self.project_slug,
        )
        queryset = Build.objects.public(
            user=self.request.user,
            project=self.project,
        ).select_related('project', 'version')

        return queryset

class BuildTriggerMixin:

    @method_decorator(login_required)
    def post(self, request, project_slug):
        project = get_object_or_404(Project, slug=project_slug)

        if not AdminPermission.is_admin(request.user, project):
            return HttpResponseForbidden()

        version_slug = request.POST.get('version_slug')
        commit = request.POST.get('commit')
        build_pk = request.POST.get('build_pk')

        version = get_object_or_404(
            self._get_versions(project),
            slug=version_slug,
        )

        # Set either the build or None
        build = version.builds.filter(pk=build_pk).first()

        if build:

            log.info(
                'Rebuilding build. project=%s version=%s commit=%s build=%s',
                project.slug,
                version.slug,
                commit,
                build.pk
            )

        update_docs_task, build = trigger_build(
            project=project,
            version=version,
            build=build,
            commit=commit,
        )
        if (update_docs_task, build) == (None, None):
            # Build was skipped
            messages.add_message(
                request,
                messages.WARNING,
                "This project is currently disabled and can't trigger new builds.",
            )
            return HttpResponseRedirect(
                reverse('builds_project_list', args=[project.slug]),
            )

        return HttpResponseRedirect(
            reverse('builds_detail', args=[project.slug, build.pk]),
        )

    def _get_versions(self, project):
        return Version.internal.public(
            user=self.request.user,
            project=project,
        )

class BuildList(BuildBase, BuildTriggerMixin, ListView):

    def get_context_data(self, **kwargs):  # pylint: disable=arguments-differ
        context = super().get_context_data(**kwargs)

        active_builds = self.get_queryset().exclude(
            state='finished',
        ).values('id')

        context['project'] = self.project
        context['active_builds'] = active_builds
        context['versions'] = self._get_versions(self.project)
        context['build_qs'] = self.get_queryset()

        return context

class BuildDetail(BuildBase, DetailView):

    pk_url_kwarg = 'build_pk'

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['project'] = self.project

        build = self.get_object()
        if build.error != BuildEnvironmentError.GENERIC_WITH_BUILD_ID.format(build_id=build.pk):
            # Do not suggest to open an issue if the error is not generic
            return context

        scheme = (
            'https://github.com/rtfd/readthedocs.org/issues/new'
            '?title={title}{build_id}'
            '&body={body}'
        )

        # TODO: we could use ``.github/ISSUE_TEMPLATE.md`` here, but we would
        # need to add some variables to it which could impact in the UX when
        # filling an issue from the web
        body = """
        ## Details:

        * Project URL: https://readthedocs.org/projects/{project_slug}/
        * Build URL(if applicable): https://readthedocs.org{build_path}
        * Read the Docs username(if applicable): {username}

        ## Expected Result

        *A description of what you wanted to happen*

        ## Actual Result

        *A description of what actually happened*""".format(
            project_slug=self.project,
            build_path=self.request.path,
            username=self.request.user,
        )

        scheme_dict = {
            'title': quote('Build error with build id #'),
            'build_id': context['build'].id,
            'body': quote(textwrap.dedent(body)),
        }

        issue_url = scheme.format(**scheme_dict)
        issue_url = urlparse(issue_url).geturl()
        context['issue_url'] = issue_url
        return context
<|endoftext|>


<commit_before>"""
A class that managed the symlinks for nginx to serve public files.

All private serving will be handled by the Python app servers,
so this only cares about public serving.

The main thing that is created is a *Web Root*,
which will be the area that is served from nginx as canonical.

Considerations
--------------

* Single version docs

The web root should be a symlink to the public documentation of that project.

* Subprojects

Subprojects will be nested under the `projects` directory,
they will be a symlink to that project's public documentation.

Objects
-------

Here is a list of all the things that the proper nouns point at:

* Single Version Project -> Default Version
* Language -> Version List
* Subproject -> Language List
* Project -> Language List

Example layout
--------------

.. code-block::

    cname_root/
        docs.fabfile.org -> /web_root/fabric
        pip.pypa.io -> /web_root/pip

    web_root/
        pip/
            en/
                latest -> rtd-builds/pip/latest/
                stable -> rtd-builds/pip/stable/
            ja -> /web_root/pip-ja/ja/
            projects/
                pip-oauth -> /web_root/pip-oauth/
        pip-ja/
            ja/

        fabric -> rtd-builds/fabric/en/latest/ # single version

"""

import os
import shutil
import logging
from collections import OrderedDict

from django.conf import settings

from readthedocs.projects.constants import LOG_TEMPLATE
from readthedocs.projects.models import Domain
from readthedocs.projects.utils import run

log = logging.getLogger(__name__)

class Symlink(object):

    """
    Base class for Symlinking of projects.

    This will be inherited for private & public symlinking.
    """

    CNAME_ROOT = os.path.join(settings.SITE_ROOT, 'public_cname_root')
    WEB_ROOT = os.path.join(settings.SITE_ROOT, 'public_web_root')

    def __init__(self, project):
        self.project = project
        self.PROJECT_ROOT = os.path.join(
            self.WEB_ROOT, project.slug
        )
        self.SUBPROJECT_ROOT = os.path.join(
            self.PROJECT_ROOT, 'projects'
        )
        self.sanity_check()

    def _log(self, msg, level='info'):
        logger = getattr(log, level)
        logger(LOG_TEMPLATE
               .format(project=self.project.slug,
                       version='',
                       msg=msg)
               )

    def sanity_check(self):
        """
        Make sure the PROJECT_ROOT is the proper structure before continuing.

        This will leave it in the proper state for the single_project setting.
        """
        if os.path.islink(self.PROJECT_ROOT) and not self.project.single_version:
            self._log("Removing single version symlink")
            os.unlink(self.PROJECT_ROOT)
            os.makedirs(self.PROJECT_ROOT)
        elif self.project.single_version and not os.path.islink(self.PROJECT_ROOT):
            shutil.rmtree(self.PROJECT_ROOT)
        elif not os.path.exists(self.PROJECT_ROOT):
            os.makedirs(self.PROJECT_ROOT)

    def run(self):
        """
        Create proper symlinks in the right order.

        Since we have a small nest of directories and symlinks,
        the ordering of these calls matter,
        so we provide this helper to make life easier.
        """
        # Outside of the web root
        self.symlink_cnames()

        # Build structure inside symlink zone
        if self.project.single_version:
            self.symlink_single_version()
        else:
            self.symlink_translations()
            self.symlink_subprojects()
            self.symlink_versions()

    def symlink_cnames(self, domain=None):
        """Symlink project CNAME domains

        Link from HOME/user_builds/$CNAME_ROOT/<cname> ->
                  HOME/user_builds/$WEB_ROOT/<project>
        """
        if domain:
            domains = [domain]
        else:
            domains = Domain.objects.filter(project=self.project)
        for domain in domains:
            self._log("Symlinking CNAME: {0} -> {1}".format(domain.domain, self.project.slug))
            docs_dir = self.PROJECT_ROOT
            symlink = os.path.join(self.CNAME_ROOT, domain.domain)

            # Make sure containing directories exist
            symlink_dir = os.sep.join(symlink.split(os.path.sep)[:-1])
            if not os.path.lexists(symlink_dir):
                os.makedirs(symlink_dir)
            # Create proper symlinks
            print run('ln -nsf {0} {1}'.format(docs_dir, symlink))

    def remove_symlink_cname(self, domain):
        """Remove single_version symlink"""
        self._log("Removing symlink for CNAME {0}".format(domain.domain))
        symlink = os.path.join(self.CNAME_ROOT, domain.domain)
        os.unlink(symlink)

    def symlink_subprojects(self):
        """Symlink project subprojects

        Link from $WEB_ROOT/projects/<project> ->
                  $WEB_ROOT/<project>
        """
        subprojects = set()
        version_queryset = self.project.versions.public(only_active=True)
        rels = self.project.subprojects.all()
        if rels.count():
            if not os.path.exists(self.SUBPROJECT_ROOT):
                os.makedirs(self.SUBPROJECT_ROOT)
        for rel in rels:
            # A mapping of slugs for the subproject URL to the actual built
            # documentation
            from_to = OrderedDict({rel.child.slug: rel.child.slug})
            subprojects.add(rel.child.slug)
            if rel.alias:
                from_to[rel.alias] = rel.child.slug
                subprojects.add(rel.alias)
            for from_slug, to_slug in from_to.items():
                self._log("Symlinking subproject: {0} -> {1}".format(from_slug, to_slug))
                symlink = os.path.join(self.SUBPROJECT_ROOT, from_slug)
                docs_dir = os.path.join(
                    self.WEB_ROOT, to_slug
                )
                symlink_dir = os.sep.join(symlink.split(os.path.sep)[:-1])
                if not os.path.lexists(symlink_dir):
                    os.makedirs(symlink_dir)
                print run('ln -nsf %s %s' % (docs_dir, symlink))

        # Remove old symlinks
        if os.path.exists(self.SUBPROJECT_ROOT):
            for subproj in os.listdir(self.SUBPROJECT_ROOT):
                if subproj not in subprojects:
                    os.unlink(os.path.join(self.SUBPROJECT_ROOT, subproj))

    def symlink_translations(self):
        """Symlink project translations

        Link from $WEB_ROOT/<project>/<language>/ ->
                  $WEB_ROOT/<translation>/<language>/
        """
        translations = {}

        for trans in self.project.translations.all():
            translations[trans.language] = trans.slug

        # Make sure the language directory is a directory
        language_dir = os.path.join(self.PROJECT_ROOT, self.project.language)
        if os.path.islink(language_dir):
            os.unlink(language_dir)
        if not os.path.exists(language_dir):
            os.makedirs(language_dir)

        for (language, slug) in translations.items():
            self._log("Symlinking translation: {0}->{1}".format(language, slug))
            symlink = os.path.join(self.PROJECT_ROOT, language)
            docs_dir = os.path.join(self.WEB_ROOT, slug, language)
            print run('ln -nsf {0} {1}'.format(docs_dir, symlink))

        # Remove old symlinks
        for lang in os.listdir(self.PROJECT_ROOT):
            if (lang not in translations and
                    lang not in ['projects', self.project.language]):
                to_delete = os.path.join(self.PROJECT_ROOT, lang)
                if os.islink(to_delete):
                    os.unlink(to_delete)
                else:
                    shutil.rmtree(to_delete)

    def symlink_single_version(self):
        """Symlink project single version

        Link from $WEB_ROOT/<project> ->
                  HOME/user_builds/<project>/rtd-builds/latest/
        """
        default_version = self.project.get_default_version()
        self._log("Symlinking single_version")

        symlink = self.PROJECT_ROOT
        if os.path.islink(symlink):
            os.unlink(symlink)
        if os.path.exists(symlink):
            shutil.rmtree(symlink)

        # Where the actual docs live
        docs_dir = os.path.join(settings.DOCROOT, self.project.slug, 'rtd-builds', default_version)
        print run('ln -nsf %s/ %s' % (docs_dir, symlink))

    def symlink_versions(self):
        """Symlink project's versions

        Link from $WEB_ROOT/<project>/<language>/<version>/ ->
                  HOME/user_builds/<project>/rtd-builds/<version>
        """
        versions = set()
        version_dir = os.path.join(self.WEB_ROOT, self.project.slug, self.project.language)
        version_queryset = self.project.versions.public(only_active=True)
        if version_queryset.count():
            if not os.path.exists(version_dir):
                os.makedirs(version_dir)
        for version in version_queryset:
            self._log("Symlinking Version: %s" % version)
            symlink = os.path.join(version_dir, version.slug)
            docs_dir = os.path.join(settings.DOCROOT, self.project.slug, 'rtd-builds', version.slug)
            print run('ln -nsf {0} {1}'.format(docs_dir, symlink))
            versions.add(version.slug)

        # Remove old symlinks
        if os.path.exists(version_dir):
            for old_ver in os.listdir(version_dir):
                if old_ver not in versions:
                    os.unlink(os.path.join(version_dir, old_ver))
<commit_msg>Bleh, its on `path`<commit_after>"""
A class that managed the symlinks for nginx to serve public files.

All private serving will be handled by the Python app servers,
so this only cares about public serving.

The main thing that is created is a *Web Root*,
which will be the area that is served from nginx as canonical.

Considerations
--------------

* Single version docs

The web root should be a symlink to the public documentation of that project.

* Subprojects

Subprojects will be nested under the `projects` directory,
they will be a symlink to that project's public documentation.

Objects
-------

Here is a list of all the things that the proper nouns point at:

* Single Version Project -> Default Version
* Language -> Version List
* Subproject -> Language List
* Project -> Language List

Example layout
--------------

.. code-block::

    cname_root/
        docs.fabfile.org -> /web_root/fabric
        pip.pypa.io -> /web_root/pip

    web_root/
        pip/
            en/
                latest -> rtd-builds/pip/latest/
                stable -> rtd-builds/pip/stable/
            ja -> /web_root/pip-ja/ja/
            projects/
                pip-oauth -> /web_root/pip-oauth/
        pip-ja/
            ja/

        fabric -> rtd-builds/fabric/en/latest/ # single version

"""

import os
import shutil
import logging
from collections import OrderedDict

from django.conf import settings

from readthedocs.projects.constants import LOG_TEMPLATE
from readthedocs.projects.models import Domain
from readthedocs.projects.utils import run

log = logging.getLogger(__name__)

class Symlink(object):

    """
    Base class for Symlinking of projects.

    This will be inherited for private & public symlinking.
    """

    CNAME_ROOT = os.path.join(settings.SITE_ROOT, 'public_cname_root')
    WEB_ROOT = os.path.join(settings.SITE_ROOT, 'public_web_root')

    def __init__(self, project):
        self.project = project
        self.PROJECT_ROOT = os.path.join(
            self.WEB_ROOT, project.slug
        )
        self.SUBPROJECT_ROOT = os.path.join(
            self.PROJECT_ROOT, 'projects'
        )
        self.sanity_check()

    def _log(self, msg, level='info'):
        logger = getattr(log, level)
        logger(LOG_TEMPLATE
               .format(project=self.project.slug,
                       version='',
                       msg=msg)
               )

    def sanity_check(self):
        """
        Make sure the PROJECT_ROOT is the proper structure before continuing.

        This will leave it in the proper state for the single_project setting.
        """
        if os.path.islink(self.PROJECT_ROOT) and not self.project.single_version:
            self._log("Removing single version symlink")
            os.unlink(self.PROJECT_ROOT)
            os.makedirs(self.PROJECT_ROOT)
        elif self.project.single_version and not os.path.islink(self.PROJECT_ROOT):
            shutil.rmtree(self.PROJECT_ROOT)
        elif not os.path.exists(self.PROJECT_ROOT):
            os.makedirs(self.PROJECT_ROOT)

    def run(self):
        """
        Create proper symlinks in the right order.

        Since we have a small nest of directories and symlinks,
        the ordering of these calls matter,
        so we provide this helper to make life easier.
        """
        # Outside of the web root
        self.symlink_cnames()

        # Build structure inside symlink zone
        if self.project.single_version:
            self.symlink_single_version()
        else:
            self.symlink_translations()
            self.symlink_subprojects()
            self.symlink_versions()

    def symlink_cnames(self, domain=None):
        """Symlink project CNAME domains

        Link from HOME/user_builds/$CNAME_ROOT/<cname> ->
                  HOME/user_builds/$WEB_ROOT/<project>
        """
        if domain:
            domains = [domain]
        else:
            domains = Domain.objects.filter(project=self.project)
        for domain in domains:
            self._log("Symlinking CNAME: {0} -> {1}".format(domain.domain, self.project.slug))
            docs_dir = self.PROJECT_ROOT
            symlink = os.path.join(self.CNAME_ROOT, domain.domain)

            # Make sure containing directories exist
            symlink_dir = os.sep.join(symlink.split(os.path.sep)[:-1])
            if not os.path.lexists(symlink_dir):
                os.makedirs(symlink_dir)
            # Create proper symlinks
            print run('ln -nsf {0} {1}'.format(docs_dir, symlink))

    def remove_symlink_cname(self, domain):
        """Remove single_version symlink"""
        self._log("Removing symlink for CNAME {0}".format(domain.domain))
        symlink = os.path.join(self.CNAME_ROOT, domain.domain)
        os.unlink(symlink)

    def symlink_subprojects(self):
        """Symlink project subprojects

        Link from $WEB_ROOT/projects/<project> ->
                  $WEB_ROOT/<project>
        """
        subprojects = set()
        version_queryset = self.project.versions.public(only_active=True)
        rels = self.project.subprojects.all()
        if rels.count():
            if not os.path.exists(self.SUBPROJECT_ROOT):
                os.makedirs(self.SUBPROJECT_ROOT)
        for rel in rels:
            # A mapping of slugs for the subproject URL to the actual built
            # documentation
            from_to = OrderedDict({rel.child.slug: rel.child.slug})
            subprojects.add(rel.child.slug)
            if rel.alias:
                from_to[rel.alias] = rel.child.slug
                subprojects.add(rel.alias)
            for from_slug, to_slug in from_to.items():
                self._log("Symlinking subproject: {0} -> {1}".format(from_slug, to_slug))
                symlink = os.path.join(self.SUBPROJECT_ROOT, from_slug)
                docs_dir = os.path.join(
                    self.WEB_ROOT, to_slug
                )
                symlink_dir = os.sep.join(symlink.split(os.path.sep)[:-1])
                if not os.path.lexists(symlink_dir):
                    os.makedirs(symlink_dir)
                print run('ln -nsf %s %s' % (docs_dir, symlink))

        # Remove old symlinks
        if os.path.exists(self.SUBPROJECT_ROOT):
            for subproj in os.listdir(self.SUBPROJECT_ROOT):
                if subproj not in subprojects:
                    os.unlink(os.path.join(self.SUBPROJECT_ROOT, subproj))

    def symlink_translations(self):
        """Symlink project translations

        Link from $WEB_ROOT/<project>/<language>/ ->
                  $WEB_ROOT/<translation>/<language>/
        """
        translations = {}

        for trans in self.project.translations.all():
            translations[trans.language] = trans.slug

        # Make sure the language directory is a directory
        language_dir = os.path.join(self.PROJECT_ROOT, self.project.language)
        if os.path.islink(language_dir):
            os.unlink(language_dir)
        if not os.path.exists(language_dir):
            os.makedirs(language_dir)

        for (language, slug) in translations.items():
            self._log("Symlinking translation: {0}->{1}".format(language, slug))
            symlink = os.path.join(self.PROJECT_ROOT, language)
            docs_dir = os.path.join(self.WEB_ROOT, slug, language)
            print run('ln -nsf {0} {1}'.format(docs_dir, symlink))

        # Remove old symlinks
        for lang in os.listdir(self.PROJECT_ROOT):
            if (lang not in translations and
                    lang not in ['projects', self.project.language]):
                to_delete = os.path.join(self.PROJECT_ROOT, lang)
                if os.path.islink(to_delete):
                    os.unlink(to_delete)
                else:
                    shutil.rmtree(to_delete)

    def symlink_single_version(self):
        """Symlink project single version

        Link from $WEB_ROOT/<project> ->
                  HOME/user_builds/<project>/rtd-builds/latest/
        """
        default_version = self.project.get_default_version()
        self._log("Symlinking single_version")

        symlink = self.PROJECT_ROOT
        if os.path.islink(symlink):
            os.unlink(symlink)
        if os.path.exists(symlink):
            shutil.rmtree(symlink)

        # Where the actual docs live
        docs_dir = os.path.join(settings.DOCROOT, self.project.slug, 'rtd-builds', default_version)
        print run('ln -nsf %s/ %s' % (docs_dir, symlink))

    def symlink_versions(self):
        """Symlink project's versions

        Link from $WEB_ROOT/<project>/<language>/<version>/ ->
                  HOME/user_builds/<project>/rtd-builds/<version>
        """
        versions = set()
        version_dir = os.path.join(self.WEB_ROOT, self.project.slug, self.project.language)
        version_queryset = self.project.versions.public(only_active=True)
        if version_queryset.count():
            if not os.path.exists(version_dir):
                os.makedirs(version_dir)
        for version in version_queryset:
            self._log("Symlinking Version: %s" % version)
            symlink = os.path.join(version_dir, version.slug)
            docs_dir = os.path.join(settings.DOCROOT, self.project.slug, 'rtd-builds', version.slug)
            print run('ln -nsf {0} {1}'.format(docs_dir, symlink))
            versions.add(version.slug)

        # Remove old symlinks
        if os.path.exists(version_dir):
            for old_ver in os.listdir(version_dir):
                if old_ver not in versions:
                    os.unlink(os.path.join(version_dir, old_ver))
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
# vim:fenc=utf-8
#
# Copyright © 2016 kirmani <sean@kirmani.io>
#
# Distributed under terms of the MIT license.

"""
Speak action.
"""

from action import Action
from action_process import ActionProcess
from speak import Speak
from speak_actively import SpeakActively
from speak_passively import SpeakPassively
from point_at_object import PointAtObject
from look_at_object import LookAtObject
from wait_for_resource_free import WaitForResourceFree
from wait_for_resource_interrupted import WaitForResourceInterrupted
from look_center import LookCenter
import time
import random
import rospy

class AskAboutObject(Action):
  def __init__(self, object_name):
    self.object_name_ = object_name
    resource_name = 'object_' + object_name
    Action.__init__(self, 'ask_about_' + resource_name, [resource_name],
        {resource_name: True},
        {resource_name: True})
    self.active_ = rospy.get_param('active', True)
    self.wait_time_ = 0.25 if self.active_ else 1.0

  def Task(self):
    bowl_phrases = [
            " is full",
            " is blue",
            " can hold things",
            " can hold food",
            " can be full",
            " can hold soup"]
    banana_phrases = [
            " is yellow",
            " can be peeled",
            " is a fruit",
            " is something I can eat",
            " is liked by monkeys",
            " peel is slippery"]
    mug_phrases = [
            " is empty",
            " is blue",
            " can hold tea",
            " can hold liquid",
            " has a handle",
            " might be hot"]
		mug_phrases = [
            " is empty",
            " is blue",
            " can hold tea",
            " can hold liquid",
            " has a handle",
            " might be hot"]
    time.sleep(self.wait_time_)
    ActionProcess('', LookAtObject(self.object_name_)).Run()
    if self.active_:
      ActionProcess('',SpeakActively(150, 50,
          "I know some things about the " + self.object_name_)).Run()
    else:
      ActionProcess('',SpeakPassively(150, 50,
          "I know some things about the " + self.object_name_)).Run()
    time.sleep(self.wait_time_)

    phrases_used = []

    for i in range(3):
      random_phrase = random.randint(0,5)
      while(random_phrase in phrases_used):
        random_phrase = random.randint(0,5)
      phrases_used.append(random_phrase)
      phrase = ""
      if(self.object_name_ == 'bowl'):
        phrase = "The " + self.object_name_ + bowl_phrases[random_phrase]
      elif(self.object_name_ == 'banana'):
        phrase = "The " + self.object_name_ + banana_phrases[random_phrase]
      elif(self.object_name_ == 'mug'):
        phrase = "The " + self.object_name_ + mug_phrases[random_phrase]
      if self.active_:
        ActionProcess('', SpeakActively(150, 50, phrase)).Run()
      else:
        ActionProcess('', SpeakPassively(150, 50, phrase)).Run()
      time.sleep(self.wait_time_)

    ActionProcess('', LookCenter()).Run()
    # ActionProcess('',
    # WaitForResourceInterrupted('floor')).Run()
    # time.sleep(6)
    # ActionProcess('',
    # WaitForResourceFree('floor')).Run()

def main():
  ActionProcess('', AskAboutObject('mug')).Run()

if __name__ == '__main__':
  main()

<commit_msg>finished adding second set of objects<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
# vim:fenc=utf-8
#
# Copyright © 2016 kirmani <sean@kirmani.io>
#
# Distributed under terms of the MIT license.

"""
Speak action.
"""

from action import Action
from action_process import ActionProcess
from speak import Speak
from speak_actively import SpeakActively
from speak_passively import SpeakPassively
from point_at_object import PointAtObject
from look_at_object import LookAtObject
from wait_for_resource_free import WaitForResourceFree
from wait_for_resource_interrupted import WaitForResourceInterrupted
from look_center import LookCenter
import time
import random
import rospy

class AskAboutObject(Action):
	def __init__(self, object_name):
		self.object_name_ = object_name
		resource_name = 'object_' + object_name
		Action.__init__(self, 'ask_about_' + resource_name, [resource_name],
        {resource_name: True},
        {resource_name: True})
		self.active_ = rospy.get_param('active', True)
		self.wait_time_ = 0.25 if self.active_ else 1.0

	def Task(self):
		bowl_phrases = [
            " is full",
            " is blue",
            " can hold things",
            " can hold food",
            " can be full",
            " can hold soup"]
		banana_phrases = [
            " is yellow",
            " can be peeled",
            " is a vegetable",
            " is something I can eat",
            " is liked by monkeys",
            " is sour"]
		mug_phrases = [
            " is full",
            " is blue",
            " can hold tea",
            " can hold liquid",
            " has a handle",
            " might be hot"]

		tongs_phrases = [
            " can grab things",
            " are metal",
            " are plastic",
            " are purple",
            " are black and silver",
            " can grab food"]

		pitcher_phrases = [
            " can be poured",
            " is metal",
            " is black",
            " is blue",
            " can hold tea",
            " is plastic"]

		sugar_phrases = [
            " is sweet",
            " is salty",
            " is black",
            " is for dessert",
            " has grains",
            " is blue"]

		time.sleep(self.wait_time_)
		ActionProcess('', LookAtObject(self.object_name_)).Run()
		if self.active_:
			ActionProcess('',SpeakActively(150, 50,
          "I know some things about the " + self.object_name_)).Run()
		else:
			ActionProcess('',SpeakPassively(150, 50,
          "I know some things about the " + self.object_name_)).Run()
		time.sleep(self.wait_time_)

		phrases_used = []

		for i in range(3):
			random_phrase = random.randint(0,5)
			while(random_phrase in phrases_used):
				random_phrase = random.randint(0,5)
			phrases_used.append(random_phrase)
			phrase = ""
			if(self.object_name_ == 'bowl'):
				phrase = "The " + self.object_name_ + bowl_phrases[random_phrase]
			elif(self.object_name_ == 'banana'):
				phrase = "The " + self.object_name_ + banana_phrases[random_phrase]
			elif(self.object_name_ == 'mug'):
				phrase = "The " + self.object_name_ + mug_phrases[random_phrase]
			elif(self.object_name_ == 'tongs'):
				phrase = "The " + self.object_name_ + tongs_phrases[random_phrase]
			elif(self.object_name_ == 'pitcher'):
				phrase = "The " + self.object_name_ + pitcher_phrases[random_phrase]
			elif(self.object_name_ == 'sugar'):
				phrase = "The " + self.object_name_ + sugar_phrases[random_phrase]
			if self.active_:
				ActionProcess('', SpeakActively(150, 50, phrase)).Run()
			else:
				ActionProcess('', SpeakPassively(150, 50, phrase)).Run()
			time.sleep(self.wait_time_)

		ActionProcess('', LookCenter()).Run()
    # ActionProcess('',
    # WaitForResourceInterrupted('floor')).Run()
    # time.sleep(6)
    # ActionProcess('',
    # WaitForResourceFree('floor')).Run()

def main():
  ActionProcess('', AskAboutObject('mug')).Run()

if __name__ == '__main__':
  main()

<|endoftext|>