<commit_before># Copyright 2020 The TensorFlow Probability Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the _License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Tests for weighted resampling methods."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import tensorflow.compat.v2 as tf
import tensorflow_probability as tfp
from tensorflow_probability.python.experimental.mcmc.weighted_resampling import _resample_using_log_points
from tensorflow_probability.python.experimental.mcmc.weighted_resampling import _scatter_nd_batch
from tensorflow_probability.python.experimental.mcmc.weighted_resampling import resample_deterministic_minimum_error
from tensorflow_probability.python.experimental.mcmc.weighted_resampling import resample_independent
from tensorflow_probability.python.experimental.mcmc.weighted_resampling import resample_stratified
from tensorflow_probability.python.experimental.mcmc.weighted_resampling import resample_systematic
from tensorflow_probability.python.internal import distribution_util as dist_util
from tensorflow_probability.python.internal import test_util

tfb = tfp.bijectors
tfd = tfp.distributions

@test_util.test_all_tf_execution_regimes
class _SMCResamplersTest(test_util.TestCase):

  # TODO(b/153689734): rewrite so as not to use `move_dimension`.
  def test_categorical_resampler_chi2(self):
    # Test categorical resampler using chi-squared test.
    if self.use_xla and tf.executing_eagerly():
      self.skipTest('No need to test XLA under all execution regimes.')

    num_probs = 50
    num_distributions = 3
    unnormalized_probs = tfd.Uniform(
        low=self.dtype(0),
        high=self.dtype(1.)).sample([num_distributions, num_probs], seed=42)
    probs = unnormalized_probs / tf.reduce_sum(
        unnormalized_probs, axis=-1, keepdims=True)

    # chi-squared test is valid as long as `num_samples` is
    # large compared to `num_probs`.
    num_particles = 10000
    num_samples = 2

    sample = self.maybe_compiler(resample_independent)(
        tf.math.log(dist_util.move_dimension(probs,
                                             source_idx=-1,
                                             dest_idx=0)),
        num_particles,
        [num_samples])
    sample = dist_util.move_dimension(sample,
                                      source_idx=0,
                                      dest_idx=-1)
    # TODO(dpiponi): reimplement this test in vectorized form rather than with
    # loops.
    for sample_index in range(num_samples):
      for prob_index in range(num_distributions):
        counts = tf.scatter_nd(
            indices=sample[sample_index, prob_index][:, tf.newaxis],
            updates=tf.ones(num_particles, dtype=tf.int32),
            shape=[num_probs])
        expected_samples = probs[prob_index] * num_particles
        chi2 = tf.reduce_sum(
            (tf.cast(counts, self.dtype) -
             expected_samples)**2 / expected_samples,
            axis=-1)
        self.assertAllLess(
            tfd.Chi2(df=self.dtype(num_probs - 1)).cdf(chi2),
            0.9999)

  def test_categorical_resampler_zero_final_class(self):
    if self.use_xla and tf.executing_eagerly():
      self.skipTest('No need to test XLA under all execution regimes.')

    probs = self.dtype([1.0, 0.0])
    resampled = self.maybe_compiler(resample_independent)(
        tf.math.log(probs), 1000, [], seed=test_util.test_seed())
    self.assertAllClose(resampled, tf.zeros((1000,), dtype=tf.int32))

  def test_systematic_resampler_zero_final_class(self):
    if self.use_xla and tf.executing_eagerly():
      self.skipTest('No need to test XLA under all execution regimes.')

    probs = self.dtype([1.0, 0.0])
    resampled = self.maybe_compiler(resample_systematic)(
        tf.math.log(probs), 1000, [])
    self.assertAllClose(resampled, tf.zeros((1000,), dtype=tf.int32))

  def test_categorical_resampler_large(self):
    if self.use_xla and tf.executing_eagerly():
      self.skipTest('No need to test XLA under all execution regimes.')

    num_probs = 10000
    log_probs = tf.fill([num_probs], -tf.math.log(self.dtype(num_probs)))
    self.evaluate(self.maybe_compiler(resample_independent)(
        log_probs, num_probs, [], seed=test_util.test_seed()))

  def test_systematic_resampler_large(self):
    if self.use_xla and tf.executing_eagerly():
      self.skipTest('No need to test XLA under all execution regimes.')

    num_probs = 10000
    log_probs = tf.fill([num_probs], -tf.math.log(self.dtype(num_probs)))
    self.evaluate(self.maybe_compiler(resample_systematic)(
        log_probs, num_probs, [],
        seed=test_util.test_seed()))

  # TODO(b/153689734): rewrite so as not to use `move_dimension`.
  def test_systematic_resampler_means(self):
    if self.use_xla and tf.executing_eagerly():
      self.skipTest('No need to test XLA under all execution regimes.')

    # Distinct samples with this resampler aren't independent
    # so a chi-squared test is inappropriate.
    # However, because it reduces variance by design, we
    # can, with high probability,  place sharp bounds on the
    # values of the sample means.
    num_distributions = 3
    num_probs = 16
    probs = tfd.Uniform(
        low=self.dtype(0.0),
        high=self.dtype(1.0)).sample([num_distributions, num_probs])
    probs = probs / tf.reduce_sum(probs, axis=-1, keepdims=True)
    num_samples = 10000
    num_particles = 20
    resampled = self.maybe_compiler(resample_systematic)(
        tf.math.log(dist_util.move_dimension(probs,
                                             source_idx=-1,
                                             dest_idx=0)),
        num_particles, [num_samples])
    resampled = dist_util.move_dimension(resampled,
                                         source_idx=0,
                                         dest_idx=-1)
    # TODO(dpiponi): reimplement this test in vectorized form rather than with
    # loops.
    for i in range(num_distributions):
      histogram = tf.reduce_mean(
          _scatter_nd_batch(resampled[:, i, :, tf.newaxis],
                            tf.ones((num_samples, num_particles),
                                    dtype=self.dtype),
                            (num_samples, num_probs),
                            batch_dims=1),
          axis=0)
      means = histogram / num_particles
      means_, probs_ = self.evaluate([means, probs[i]])

      # TODO(dpiponi): it should be possible to compute the exact distribution
      # of these means and choose `atol` in a more principled way.
      self.assertAllClose(means_, probs_, atol=0.01)

  # TODO(b/153689734): rewrite so as not to use `move_dimension`.
  def test_minimum_error_resampler_means(self):
    if self.use_xla and tf.executing_eagerly():
      self.skipTest('No need to test XLA under all execution regimes.')

    # Distinct samples with this resampler aren't independent
    # so a chi-squared test is inappropriate.
    # However, because it reduces variance by design, we
    # can, with high probability,  place sharp bounds on the
    # values of the sample means.
    num_distributions = 3
    num_probs = 8
    probs = tfd.Uniform(
        low=self.dtype(0.0),
        high=self.dtype(1.0)).sample([num_distributions, num_probs],
                                     seed=test_util.test_seed())
    probs = probs / tf.reduce_sum(probs, axis=-1, keepdims=True)
    num_samples = 4
    num_particles = 10000
    resampled = self.maybe_compiler(resample_deterministic_minimum_error)(
        tf.math.log(dist_util.move_dimension(probs,
                                             source_idx=-1,
                                             dest_idx=0)),
        num_particles, [num_samples])
    resampled = dist_util.move_dimension(resampled,
                                         source_idx=0,
                                         dest_idx=-1)
    # TODO(dpiponi): reimplement this test in vectorized form rather than with
    # loops.
    for i in range(num_distributions):
      histogram = tf.reduce_mean(
          _scatter_nd_batch(resampled[:, i, :, tf.newaxis],
                            tf.ones((num_samples, num_particles)),
                            (num_samples, num_probs),
                            batch_dims=1),
          axis=0)
      means = histogram / num_particles
      means_, probs_ = self.evaluate([means, probs[i]])

      self.assertAllClose(means_, probs_, atol=1.0 / num_particles)

  # TODO(b/153689734): rewrite so as not to use `move_dimension`.
  def test_stratified_resampler_means(self):
    if self.use_xla and tf.executing_eagerly():
      self.skipTest('No need to test XLA under all execution regimes.')

    # Distinct samples with this resampler aren't independent
    # so a chi-squared test is inappropriate.
    # However, because it reduces variance by design, we
    # can, with high probability,  place sharp bounds on the
    # values of the sample means.
    num_distributions = 3
    num_probs = 8
    probs = tfd.Uniform(
        low=self.dtype(0.0),
        high=self.dtype(1.0)).sample(
            [num_distributions, num_probs],
            seed=test_util.test_seed())
    probs = probs / tf.reduce_sum(probs, axis=-1, keepdims=True)
    num_samples = 4
    num_particles = 10000
    resampled = self.maybe_compiler(resample_stratified)(
        tf.math.log(dist_util.move_dimension(probs,
                                             source_idx=-1,
                                             dest_idx=0)),
        num_particles, [num_samples])
    resampled = dist_util.move_dimension(resampled,
                                         source_idx=0,
                                         dest_idx=-1)
    # TODO(dpiponi): reimplement this test in vectorized form rather than with
    # loops.
    for i in range(num_distributions):
      histogram = tf.reduce_mean(
          _scatter_nd_batch(resampled[:, i, :, tf.newaxis],
                            tf.ones((num_samples, num_particles)),
                            (num_samples, num_probs),
                            batch_dims=1),
          axis=0)
      means = histogram / num_particles
      means_, probs_ = self.evaluate([means, probs[i]])

      self.assertAllClose(means_, probs_, atol=0.1)

  def test_resample_using_extremal_log_points(self):
    if self.use_xla and tf.executing_eagerly():
      self.skipTest('No need to test XLA under all execution regimes.')

    n = 8

    one = self.dtype(1)
    almost_one = np.nextafter(one, self.dtype(0))
    sample_shape = []
    log_points_zero = tf.math.log(tf.zeros(n, self.dtype))
    log_points_almost_one = tf.fill([n], tf.math.log(almost_one))
    log_probs_beginning = tf.math.log(
        tf.concat([[one], tf.zeros(n - 1, dtype=self.dtype)], axis=0))
    log_probs_end = tf.math.log(tf.concat([tf.zeros(n - 1, dtype=self.dtype),
                                           [one]], axis=0))

    sample_fn = self.maybe_compiler(_resample_using_log_points)
    indices = sample_fn(
        log_probs_beginning, sample_shape, log_points_zero)
    self.assertAllEqual(indices, tf.zeros(n, dtype=tf.float32))

    indices = sample_fn(
        log_probs_end, sample_shape, log_points_zero)
    self.assertAllEqual(indices, tf.fill([n], n - 1))

    indices = sample_fn(
        log_probs_beginning, sample_shape, log_points_almost_one)
    self.assertAllEqual(indices, tf.zeros(n, dtype=tf.float32))

    indices = sample_fn(
        log_probs_end, sample_shape, log_points_almost_one)
    self.assertAllEqual(indices, tf.fill([n], n - 1))

  def maybe_compiler(self, f):
    if self.use_xla:
      return tf.function(f, autograph=False, experimental_compile=True)
    return f  # Do not compile.

class SMCResamplersTestFloat32(_SMCResamplersTest):
  dtype = np.float32
  use_xla = False

class SMCResamplersTestFloat32XLACompiled(_SMCResamplersTest):
  dtype = np.float32
  use_xla = True

class SMCResamplersTestFloat64(_SMCResamplersTest):
  dtype = np.float64
  use_xla = False

del _SMCResamplersTest  # Don't try to run tests from the base class.

if __name__ == '__main__':
  tf.test.main()
<commit_msg>Bump categorical resampler chi2 test threshold to prevent flaking.<commit_after># Copyright 2020 The TensorFlow Probability Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the _License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Tests for weighted resampling methods."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import tensorflow.compat.v2 as tf
import tensorflow_probability as tfp
from tensorflow_probability.python.experimental.mcmc.weighted_resampling import _resample_using_log_points
from tensorflow_probability.python.experimental.mcmc.weighted_resampling import _scatter_nd_batch
from tensorflow_probability.python.experimental.mcmc.weighted_resampling import resample_deterministic_minimum_error
from tensorflow_probability.python.experimental.mcmc.weighted_resampling import resample_independent
from tensorflow_probability.python.experimental.mcmc.weighted_resampling import resample_stratified
from tensorflow_probability.python.experimental.mcmc.weighted_resampling import resample_systematic
from tensorflow_probability.python.internal import distribution_util as dist_util
from tensorflow_probability.python.internal import test_util

tfb = tfp.bijectors
tfd = tfp.distributions

@test_util.test_all_tf_execution_regimes
class _SMCResamplersTest(test_util.TestCase):

  # TODO(b/153689734): rewrite so as not to use `move_dimension`.
  def test_categorical_resampler_chi2(self):
    # Test categorical resampler using chi-squared test.
    if self.use_xla and tf.executing_eagerly():
      self.skipTest('No need to test XLA under all execution regimes.')

    num_probs = 50
    num_distributions = 3
    unnormalized_probs = tfd.Uniform(
        low=self.dtype(0),
        high=self.dtype(1.)).sample([num_distributions, num_probs], seed=42)
    probs = unnormalized_probs / tf.reduce_sum(
        unnormalized_probs, axis=-1, keepdims=True)

    # chi-squared test is valid as long as `num_samples` is
    # large compared to `num_probs`.
    num_particles = 10000
    num_samples = 2

    sample = self.maybe_compiler(resample_independent)(
        tf.math.log(dist_util.move_dimension(probs,
                                             source_idx=-1,
                                             dest_idx=0)),
        num_particles,
        [num_samples])
    sample = dist_util.move_dimension(sample,
                                      source_idx=0,
                                      dest_idx=-1)
    # TODO(dpiponi): reimplement this test in vectorized form rather than with
    # loops.
    for sample_index in range(num_samples):
      for prob_index in range(num_distributions):
        counts = tf.scatter_nd(
            indices=sample[sample_index, prob_index][:, tf.newaxis],
            updates=tf.ones(num_particles, dtype=tf.int32),
            shape=[num_probs])
        expected_samples = probs[prob_index] * num_particles
        chi2 = tf.reduce_sum(
            (tf.cast(counts, self.dtype) -
             expected_samples)**2 / expected_samples,
            axis=-1)
        self.assertAllLess(
            tfd.Chi2(df=self.dtype(num_probs - 1)).cdf(chi2),
            0.99995)

  def test_categorical_resampler_zero_final_class(self):
    if self.use_xla and tf.executing_eagerly():
      self.skipTest('No need to test XLA under all execution regimes.')

    probs = self.dtype([1.0, 0.0])
    resampled = self.maybe_compiler(resample_independent)(
        tf.math.log(probs), 1000, [], seed=test_util.test_seed())
    self.assertAllClose(resampled, tf.zeros((1000,), dtype=tf.int32))

  def test_systematic_resampler_zero_final_class(self):
    if self.use_xla and tf.executing_eagerly():
      self.skipTest('No need to test XLA under all execution regimes.')

    probs = self.dtype([1.0, 0.0])
    resampled = self.maybe_compiler(resample_systematic)(
        tf.math.log(probs), 1000, [])
    self.assertAllClose(resampled, tf.zeros((1000,), dtype=tf.int32))

  def test_categorical_resampler_large(self):
    if self.use_xla and tf.executing_eagerly():
      self.skipTest('No need to test XLA under all execution regimes.')

    num_probs = 10000
    log_probs = tf.fill([num_probs], -tf.math.log(self.dtype(num_probs)))
    self.evaluate(self.maybe_compiler(resample_independent)(
        log_probs, num_probs, [], seed=test_util.test_seed()))

  def test_systematic_resampler_large(self):
    if self.use_xla and tf.executing_eagerly():
      self.skipTest('No need to test XLA under all execution regimes.')

    num_probs = 10000
    log_probs = tf.fill([num_probs], -tf.math.log(self.dtype(num_probs)))
    self.evaluate(self.maybe_compiler(resample_systematic)(
        log_probs, num_probs, [],
        seed=test_util.test_seed()))

  # TODO(b/153689734): rewrite so as not to use `move_dimension`.
  def test_systematic_resampler_means(self):
    if self.use_xla and tf.executing_eagerly():
      self.skipTest('No need to test XLA under all execution regimes.')

    # Distinct samples with this resampler aren't independent
    # so a chi-squared test is inappropriate.
    # However, because it reduces variance by design, we
    # can, with high probability,  place sharp bounds on the
    # values of the sample means.
    num_distributions = 3
    num_probs = 16
    probs = tfd.Uniform(
        low=self.dtype(0.0),
        high=self.dtype(1.0)).sample([num_distributions, num_probs])
    probs = probs / tf.reduce_sum(probs, axis=-1, keepdims=True)
    num_samples = 10000
    num_particles = 20
    resampled = self.maybe_compiler(resample_systematic)(
        tf.math.log(dist_util.move_dimension(probs,
                                             source_idx=-1,
                                             dest_idx=0)),
        num_particles, [num_samples])
    resampled = dist_util.move_dimension(resampled,
                                         source_idx=0,
                                         dest_idx=-1)
    # TODO(dpiponi): reimplement this test in vectorized form rather than with
    # loops.
    for i in range(num_distributions):
      histogram = tf.reduce_mean(
          _scatter_nd_batch(resampled[:, i, :, tf.newaxis],
                            tf.ones((num_samples, num_particles),
                                    dtype=self.dtype),
                            (num_samples, num_probs),
                            batch_dims=1),
          axis=0)
      means = histogram / num_particles
      means_, probs_ = self.evaluate([means, probs[i]])

      # TODO(dpiponi): it should be possible to compute the exact distribution
      # of these means and choose `atol` in a more principled way.
      self.assertAllClose(means_, probs_, atol=0.01)

  # TODO(b/153689734): rewrite so as not to use `move_dimension`.
  def test_minimum_error_resampler_means(self):
    if self.use_xla and tf.executing_eagerly():
      self.skipTest('No need to test XLA under all execution regimes.')

    # Distinct samples with this resampler aren't independent
    # so a chi-squared test is inappropriate.
    # However, because it reduces variance by design, we
    # can, with high probability,  place sharp bounds on the
    # values of the sample means.
    num_distributions = 3
    num_probs = 8
    probs = tfd.Uniform(
        low=self.dtype(0.0),
        high=self.dtype(1.0)).sample([num_distributions, num_probs],
                                     seed=test_util.test_seed())
    probs = probs / tf.reduce_sum(probs, axis=-1, keepdims=True)
    num_samples = 4
    num_particles = 10000
    resampled = self.maybe_compiler(resample_deterministic_minimum_error)(
        tf.math.log(dist_util.move_dimension(probs,
                                             source_idx=-1,
                                             dest_idx=0)),
        num_particles, [num_samples])
    resampled = dist_util.move_dimension(resampled,
                                         source_idx=0,
                                         dest_idx=-1)
    # TODO(dpiponi): reimplement this test in vectorized form rather than with
    # loops.
    for i in range(num_distributions):
      histogram = tf.reduce_mean(
          _scatter_nd_batch(resampled[:, i, :, tf.newaxis],
                            tf.ones((num_samples, num_particles)),
                            (num_samples, num_probs),
                            batch_dims=1),
          axis=0)
      means = histogram / num_particles
      means_, probs_ = self.evaluate([means, probs[i]])

      self.assertAllClose(means_, probs_, atol=1.0 / num_particles)

  # TODO(b/153689734): rewrite so as not to use `move_dimension`.
  def test_stratified_resampler_means(self):
    if self.use_xla and tf.executing_eagerly():
      self.skipTest('No need to test XLA under all execution regimes.')

    # Distinct samples with this resampler aren't independent
    # so a chi-squared test is inappropriate.
    # However, because it reduces variance by design, we
    # can, with high probability,  place sharp bounds on the
    # values of the sample means.
    num_distributions = 3
    num_probs = 8
    probs = tfd.Uniform(
        low=self.dtype(0.0),
        high=self.dtype(1.0)).sample(
            [num_distributions, num_probs],
            seed=test_util.test_seed())
    probs = probs / tf.reduce_sum(probs, axis=-1, keepdims=True)
    num_samples = 4
    num_particles = 10000
    resampled = self.maybe_compiler(resample_stratified)(
        tf.math.log(dist_util.move_dimension(probs,
                                             source_idx=-1,
                                             dest_idx=0)),
        num_particles, [num_samples])
    resampled = dist_util.move_dimension(resampled,
                                         source_idx=0,
                                         dest_idx=-1)
    # TODO(dpiponi): reimplement this test in vectorized form rather than with
    # loops.
    for i in range(num_distributions):
      histogram = tf.reduce_mean(
          _scatter_nd_batch(resampled[:, i, :, tf.newaxis],
                            tf.ones((num_samples, num_particles)),
                            (num_samples, num_probs),
                            batch_dims=1),
          axis=0)
      means = histogram / num_particles
      means_, probs_ = self.evaluate([means, probs[i]])

      self.assertAllClose(means_, probs_, atol=0.1)

  def test_resample_using_extremal_log_points(self):
    if self.use_xla and tf.executing_eagerly():
      self.skipTest('No need to test XLA under all execution regimes.')

    n = 8

    one = self.dtype(1)
    almost_one = np.nextafter(one, self.dtype(0))
    sample_shape = []
    log_points_zero = tf.math.log(tf.zeros(n, self.dtype))
    log_points_almost_one = tf.fill([n], tf.math.log(almost_one))
    log_probs_beginning = tf.math.log(
        tf.concat([[one], tf.zeros(n - 1, dtype=self.dtype)], axis=0))
    log_probs_end = tf.math.log(tf.concat([tf.zeros(n - 1, dtype=self.dtype),
                                           [one]], axis=0))

    sample_fn = self.maybe_compiler(_resample_using_log_points)
    indices = sample_fn(
        log_probs_beginning, sample_shape, log_points_zero)
    self.assertAllEqual(indices, tf.zeros(n, dtype=tf.float32))

    indices = sample_fn(
        log_probs_end, sample_shape, log_points_zero)
    self.assertAllEqual(indices, tf.fill([n], n - 1))

    indices = sample_fn(
        log_probs_beginning, sample_shape, log_points_almost_one)
    self.assertAllEqual(indices, tf.zeros(n, dtype=tf.float32))

    indices = sample_fn(
        log_probs_end, sample_shape, log_points_almost_one)
    self.assertAllEqual(indices, tf.fill([n], n - 1))

  def maybe_compiler(self, f):
    if self.use_xla:
      return tf.function(f, autograph=False, experimental_compile=True)
    return f  # Do not compile.

class SMCResamplersTestFloat32(_SMCResamplersTest):
  dtype = np.float32
  use_xla = False

class SMCResamplersTestFloat32XLACompiled(_SMCResamplersTest):
  dtype = np.float32
  use_xla = True

class SMCResamplersTestFloat64(_SMCResamplersTest):
  dtype = np.float64
  use_xla = False

del _SMCResamplersTest  # Don't try to run tests from the base class.

if __name__ == '__main__':
  tf.test.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python2
import sys, os, json, errno, subprocess, yum

class yumoter:
    def __init__(self, configFile, repobasepath):
        self.urlbasepath = "http://yumoter.gnmedia.net"
        self.repobasepath = repobasepath
        self.reloadConfig(configFile)
        self.yb = yum.YumBase()
        self.yb.setCacheDir(yum.misc.getCacheDir())
        self.yb.repos.disableRepo("*")

    def _getConfig(self, jsonFile):
        fh = open(jsonFile, 'r')
        jsonOutput = json.load(fh)
        fh.close()
        return jsonOutput

    def _mkdir_p(self, path):
        try:
            os.makedirs(path)
        except OSError as exc:
            if exc.errno == errno.EEXIST and os.path.isdir(path):
                pass
            else:
                raise

    def _translateToMajorVer(self, ver):
        # I: 6.4
        # O: 6
        if '.' in ver:
            ver = ver.split('.')[0]
        return ver

    def _getPaths(self):
        for repo in self.repoConfig:
            repopath = []
            # Does this repo have a path for promotion?
            if 'promotionpath' in self.repoConfig[repo]:
                for promopath in self.repoConfig[repo]['promotionpath']:
                    repopath.append("%s/%s/%s" % (self.repobasepath, self.repoConfig[repo]['path'], promopath))
            else:
                # repo does not have a path for promotion
                repopath.append("%s/%s" % (self.repobasepath, self.repoConfig[repo]['path']))
            self.repoConfig[repo]['fullpaths'] = repopath

    def _getURLs(self):
        for repo in self.repoConfig:
            repoURLs = []
            if 'promotionpath' in self.repoConfig[repo]:
                for promopath in self.repoConfig[repo]['promotionpath']:
                    repoURLs.append("%s/%s/%s" % (self.urlbasepath, self.repoConfig[repo]['path'], promopath))
            else:
                repoURLs.append("%s/%s" % (self.urlbasepath, self.repoConfig[repo]['path']))
            self.repoConfig[repo]['fullurls'] = repoURLs

    def _mkPaths(self):
        masterPathList = []
        for repo in self.repoConfig:
            if 'fullpaths' in self.repoConfig[repo]:
                for entry in self.repoConfig[repo]['fullpaths']:
                    masterPathList.append(entry)
        for entry in masterPathList:
            if not os.path.isdir(entry):
                print "creating missing dir: %s" % entry
                self._mkdir_p(entry)

    def _runRsync(self, rsrc, rdst, args):
        # str(rsrc), str(rdst), list(args)
        sysCall = ['rsync'] + args + [rsrc, rdst]
        print 'sysCall', sysCall
        rsyncStdout = []
        rsyncStderr = []
        p = subprocess.Popen(sysCall, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
        for line in iter(p.stdout.readline, ""):
            stdoutLine = line.strip() + '\r\n'
            rsyncStdout.append(stdoutLine)
            sys.stdout.write(stdoutLine)
            sys.stdout.flush()
        for line in iter(p.stderr.readline, ""):
            stderrLine = line.strip() + '\r\n'
            rsyncStderr.append(stderrLine)
            sys.stderr.write(stderrLine)
            sys.stderr.flush()
        return (rsyncStdout, rsyncStderr)
        # TODO check return status please. Stop coding like a 12 year old.

    def _loadRepo(self, reponame, repo):
        print "Adding repo:", reponame, repo
        # if repo is unicode and not a string, this will silently do the wrong thing
        # and all actions against the repo will fail.
        self.yb.add_enable_repo(reponame, baseurls=[str(repo)], mirrorlist=None)

    def _returnNewestByNameArch(self, patternsList):
        pkgs = self.yb.pkgSack.returnNewestByNameArch(patterns=patternsList)
        return pkgs

    def _getDeps(self, pkgObj):
        if type(pkgObj) != list:
            pkgObj = [pkgObj]
        return self.yb.findDeps(pkgObj)

    def _urlToPath(self, url):
        return "%s%s" % (self.repobasepath, url.replace(self.urlbasepath, ''))

    def _urlToRepo(self, url):
        choppedurl = url.replace("%s/" % self.urlbasepath, '')
        for tmprepo in self.repoConfig:
            if choppedurl.startswith(self.repoConfig[tmprepo]['path']):
                return tmprepo
        print "ERROR: _urlToRepo could not evaluate which repo this url came from"
        sys.exit(1)

    def _repoIsPromoted(self, repo):
        if 'promotionpath' not in self.repoConfig[repo].keys():
            return False
        return True

    def _urlToPromoPath(self, url):
        # takes a url of an rpm
        # returns path of promoted dir
        # url = 'http://yumoter.gnmedia.net/epel/6/wildwest/tmux-1.6-3.el6.x86_64.rpm'
        repo = self._urlToRepo(url)
        choppedurl = url.replace("%s/" % self.urlbasepath, '')
        # choppedurl = epel/6/wildwest/tmux-1.6-3.el6.x86_64.rpm
        # check to see if this repo is even promoted
        if not self._repoIsPromoted(repo):
            print "ERROR: _urlToPromoPath was called on a repo which isn't promoted."
            sys.exit(1)
        # determine current env
        currenv = os.path.basename(os.path.dirname(choppedurl))
        newenv = self.repoConfig[repo]['promotionpath'][self.repoConfig[repo]['promotionpath'].index(currenv)+1]
        result = self._urlToPath(url.replace("/%s/" % currenv, "/%s/" % newenv ))
        return result

    def promotePkg(self, pkg):
        oldpath = self._urlToPath(pkg.remote_url)
        newpath = self._urlToPromoPath(pkg.remote_url)
        print "promoting %s -> %s" % (oldpath, newpath)

    def promotePkgs(self, pkgList):
        pass

    def loadRepos(self, osVer, env):
        # this should load all the repos for osVer in env
        # Should use an internal method to load one repo
        # by url.
        loadrepos = []
        shortenv = self._translateToMajorVer(env)
        for repo in self.repoConfig:
            if '.' in self.repoConfig[repo]['osver']:
                # This is a repo sensative to minor versions
                if osVer == self.repoConfig[repo]['osver']:
                    loadrepos.append(repo)
            else:
                # This repo cares only about major versions
                if self._translateToMajorVer(osVer) == self.repoConfig[repo]['osver']:
                    loadrepos.append(repo)
        for repo in loadrepos:
            if len(self.repoConfig[repo]['fullurls']) == 1:
                # This repo only has one env. Repo is not promoted.
                # Load the only URL you can
                self._loadRepo(repo, self.repoConfig[repo]['fullurls'][0])
            else:
                self._loadRepo(repo, self.repoConfig[repo]['fullurls'][self.repoConfig[repo]['promotionpath'].index(env)])

    def getDeps(self, pkgObj):
        if type(pkgObj) != list:
            pkgObj = [pkgObj]
        depsDict = self._getDeps(pkgObj)
        resultDict = {}
        for origPkg in depsDict:
            for dep in depsDict[origPkg]:
                suggestedDep = self.yb.bestPackagesFromList(depsDict[origPkg][dep])
                if len(suggestedDep) > 1:
                    print "WARNING: found multiple suggested dependencies"
                    print suggestedDep
                if origPkg not in resultDict:
                    resultDict[origPkg] = []
                if suggestedDep[0] not in resultDict[origPkg]:
                    resultDict[origPkg].append(suggestedDep[0])
        return resultDict

    def syncRepos(self):
        outputList = []
        for repo in self.repoConfig:
            # Only repos with upstream set need to be synced.
            if 'upstream' in self.repoConfig[repo]:
                # If the dst dir doesn't exist, create it.
                if not os.path.isdir(self.repoConfig[repo]['fullpaths'][0]):
                    self._mkPaths()
                outputList.append(self._runRsync(self.repoConfig[repo]['upstream'], self.repoConfig[repo]['fullpaths'][0], ['-av', '--progress']))
        return outputList

    def reloadConfig(self, jsonFile):
        self.repoConfig = self._getConfig(jsonFile)
        self._getPaths()
        self._getURLs()

'''
    def repoSearch(self, pkgName, repos):

        pkgs = yb.pkgSack.returnNewestByNameArch(patterns=pkgName)
        for pkg in pkgs:
            print "%s: %s" % (pkg, pkg.summary)
'''<commit_msg>dont promote pkgs without promotepaths<commit_after>#!/usr/bin/env python2
import sys, os, json, errno, subprocess, yum

class yumoter:
    def __init__(self, configFile, repobasepath):
        self.urlbasepath = "http://yumoter.gnmedia.net"
        self.repobasepath = repobasepath
        self.reloadConfig(configFile)
        self.yb = yum.YumBase()
        self.yb.setCacheDir(yum.misc.getCacheDir())
        self.yb.repos.disableRepo("*")

    def _getConfig(self, jsonFile):
        fh = open(jsonFile, 'r')
        jsonOutput = json.load(fh)
        fh.close()
        return jsonOutput

    def _mkdir_p(self, path):
        try:
            os.makedirs(path)
        except OSError as exc:
            if exc.errno == errno.EEXIST and os.path.isdir(path):
                pass
            else:
                raise

    def _translateToMajorVer(self, ver):
        # I: 6.4
        # O: 6
        if '.' in ver:
            ver = ver.split('.')[0]
        return ver

    def _getPaths(self):
        for repo in self.repoConfig:
            repopath = []
            # Does this repo have a path for promotion?
            if 'promotionpath' in self.repoConfig[repo]:
                for promopath in self.repoConfig[repo]['promotionpath']:
                    repopath.append("%s/%s/%s" % (self.repobasepath, self.repoConfig[repo]['path'], promopath))
            else:
                # repo does not have a path for promotion
                repopath.append("%s/%s" % (self.repobasepath, self.repoConfig[repo]['path']))
            self.repoConfig[repo]['fullpaths'] = repopath

    def _getURLs(self):
        for repo in self.repoConfig:
            repoURLs = []
            if 'promotionpath' in self.repoConfig[repo]:
                for promopath in self.repoConfig[repo]['promotionpath']:
                    repoURLs.append("%s/%s/%s" % (self.urlbasepath, self.repoConfig[repo]['path'], promopath))
            else:
                repoURLs.append("%s/%s" % (self.urlbasepath, self.repoConfig[repo]['path']))
            self.repoConfig[repo]['fullurls'] = repoURLs

    def _mkPaths(self):
        masterPathList = []
        for repo in self.repoConfig:
            if 'fullpaths' in self.repoConfig[repo]:
                for entry in self.repoConfig[repo]['fullpaths']:
                    masterPathList.append(entry)
        for entry in masterPathList:
            if not os.path.isdir(entry):
                print "creating missing dir: %s" % entry
                self._mkdir_p(entry)

    def _runRsync(self, rsrc, rdst, args):
        # str(rsrc), str(rdst), list(args)
        sysCall = ['rsync'] + args + [rsrc, rdst]
        print 'sysCall', sysCall
        rsyncStdout = []
        rsyncStderr = []
        p = subprocess.Popen(sysCall, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
        for line in iter(p.stdout.readline, ""):
            stdoutLine = line.strip() + '\r\n'
            rsyncStdout.append(stdoutLine)
            sys.stdout.write(stdoutLine)
            sys.stdout.flush()
        for line in iter(p.stderr.readline, ""):
            stderrLine = line.strip() + '\r\n'
            rsyncStderr.append(stderrLine)
            sys.stderr.write(stderrLine)
            sys.stderr.flush()
        return (rsyncStdout, rsyncStderr)
        # TODO check return status please. Stop coding like a 12 year old.

    def _loadRepo(self, reponame, repo):
        print "Adding repo:", reponame, repo
        # if repo is unicode and not a string, this will silently do the wrong thing
        # and all actions against the repo will fail.
        self.yb.add_enable_repo(reponame, baseurls=[str(repo)], mirrorlist=None)

    def _returnNewestByNameArch(self, patternsList):
        pkgs = self.yb.pkgSack.returnNewestByNameArch(patterns=patternsList)
        return pkgs

    def _getDeps(self, pkgObj):
        if type(pkgObj) != list:
            pkgObj = [pkgObj]
        return self.yb.findDeps(pkgObj)

    def _urlToPath(self, url):
        return "%s%s" % (self.repobasepath, url.replace(self.urlbasepath, ''))

    def _urlToRepo(self, url):
        choppedurl = url.replace("%s/" % self.urlbasepath, '')
        for tmprepo in self.repoConfig:
            if choppedurl.startswith(self.repoConfig[tmprepo]['path']):
                return tmprepo
        print "ERROR: _urlToRepo could not evaluate which repo this url came from"
        sys.exit(1)

    def _repoIsPromoted(self, repo):
        if 'promotionpath' not in self.repoConfig[repo].keys():
            return False
        return True

    def _urlToPromoPath(self, url):
        # takes a url of an rpm
        # returns path of promoted dir
        # url = 'http://yumoter.gnmedia.net/epel/6/wildwest/tmux-1.6-3.el6.x86_64.rpm'
        repo = self._urlToRepo(url)
        choppedurl = url.replace("%s/" % self.urlbasepath, '')
        # choppedurl = epel/6/wildwest/tmux-1.6-3.el6.x86_64.rpm
        # check to see if this repo is even promoted
        if not self._repoIsPromoted(repo):
            print "ERROR: _urlToPromoPath was called on a repo which isn't promoted."
            sys.exit(1)
        # determine current env
        currenv = os.path.basename(os.path.dirname(choppedurl))
        newenv = self.repoConfig[repo]['promotionpath'][self.repoConfig[repo]['promotionpath'].index(currenv)+1]
        result = self._urlToPath(url.replace("/%s/" % currenv, "/%s/" % newenv ))
        return result

    def promotePkg(self, pkg):
        if self._repoIsPromoted(self._urlToRepo(pkg.remote_url)):
            oldpath = self._urlToPath(pkg.remote_url)
            newpath = self._urlToPromoPath(pkg.remote_url)
            print "promoting %s -> %s" % (oldpath, newpath)
        else:
            print "skipping %s because it is not in a promoted repo"

    def promotePkgs(self, pkgList):
        pass

    def loadRepos(self, osVer, env):
        # this should load all the repos for osVer in env
        # Should use an internal method to load one repo
        # by url.
        loadrepos = []
        shortenv = self._translateToMajorVer(env)
        for repo in self.repoConfig:
            if '.' in self.repoConfig[repo]['osver']:
                # This is a repo sensative to minor versions
                if osVer == self.repoConfig[repo]['osver']:
                    loadrepos.append(repo)
            else:
                # This repo cares only about major versions
                if self._translateToMajorVer(osVer) == self.repoConfig[repo]['osver']:
                    loadrepos.append(repo)
        for repo in loadrepos:
            if len(self.repoConfig[repo]['fullurls']) == 1:
                # This repo only has one env. Repo is not promoted.
                # Load the only URL you can
                self._loadRepo(repo, self.repoConfig[repo]['fullurls'][0])
            else:
                self._loadRepo(repo, self.repoConfig[repo]['fullurls'][self.repoConfig[repo]['promotionpath'].index(env)])

    def getDeps(self, pkgObj):
        if type(pkgObj) != list:
            pkgObj = [pkgObj]
        depsDict = self._getDeps(pkgObj)
        resultDict = {}
        for origPkg in depsDict:
            for dep in depsDict[origPkg]:
                suggestedDep = self.yb.bestPackagesFromList(depsDict[origPkg][dep])
                if len(suggestedDep) > 1:
                    print "WARNING: found multiple suggested dependencies"
                    print suggestedDep
                if origPkg not in resultDict:
                    resultDict[origPkg] = []
                if suggestedDep[0] not in resultDict[origPkg]:
                    resultDict[origPkg].append(suggestedDep[0])
        return resultDict

    def syncRepos(self):
        outputList = []
        for repo in self.repoConfig:
            # Only repos with upstream set need to be synced.
            if 'upstream' in self.repoConfig[repo]:
                # If the dst dir doesn't exist, create it.
                if not os.path.isdir(self.repoConfig[repo]['fullpaths'][0]):
                    self._mkPaths()
                outputList.append(self._runRsync(self.repoConfig[repo]['upstream'], self.repoConfig[repo]['fullpaths'][0], ['-av', '--progress']))
        return outputList

    def reloadConfig(self, jsonFile):
        self.repoConfig = self._getConfig(jsonFile)
        self._getPaths()
        self._getURLs()

'''
    def repoSearch(self, pkgName, repos):

        pkgs = yb.pkgSack.returnNewestByNameArch(patterns=pkgName)
        for pkg in pkgs:
            print "%s: %s" % (pkg, pkg.summary)
'''<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import sys

from osv import fields, osv
import tools
from tools.translate import _

class project_gtd_context(osv.osv):
    _name = "project.gtd.context"
    _description = "Context"
    _columns = {
        'name': fields.char('Context', size=64, required=True, select=1, translate=1),
        'sequence': fields.integer('Sequence', help="Gives the sequence order when displaying a list of contexts."),
    }
    _defaults = {
        'sequence': 1
    }
    _order = "sequence, name"

project_gtd_context()

class project_gtd_timebox(osv.osv):
    _name = "project.gtd.timebox"
    _order = "sequence"
    _columns = {
        'name': fields.char('Timebox', size=64, required=True, select=1, translate=1),
        'sequence': fields.integer('Sequence', help="Gives the sequence order when displaying a list of timebox."),
        'icon': fields.selection(tools.icons, 'Icon', size=64),
    }

project_gtd_timebox()

class project_task(osv.osv):
    _inherit = "project.task"
    _columns = {
        'timebox_id': fields.many2one('project.gtd.timebox', "Timebox",help="Time-laps during which task has to be treated"),
        'context_id': fields.many2one('project.gtd.context', "Context",help="The context place where user has to treat task"),
     }

    def copy_data(self, cr, uid, id, default=None, context=None):
        if context is None:
            context = {}
        if not default:
            default = {}
        default['timebox_id'] = False
        default['context_id'] = False
        return super(project_task,self).copy_data(cr, uid, id, default, context)

    def _get_context(self,cr, uid, context=None):
        ids = self.pool.get('project.gtd.context').search(cr, uid, [], context=context)
        return ids and ids[0] or False

    _defaults = {
        'context_id': _get_context
    }
    def next_timebox(self, cr, uid, ids, *args):
        timebox_obj = self.pool.get('project.gtd.timebox')
        timebox_ids = timebox_obj.search(cr,uid,[])
        if not timebox_ids: return True
        for task in self.browse(cr,uid,ids):
            timebox = task.timebox_id.id
            if not timebox:
                self.write(cr, uid, task.id, {'timebox_id': timebox_ids[0]})
            elif timebox_ids.index(timebox) != len(timebox_ids)-1:
                index = timebox_ids.index(timebox)
                self.write(cr, uid, task.id, {'timebox_id': timebox_ids[index+1]})
        return True

    def prev_timebox(self, cr, uid, ids, *args):
        timebox_obj = self.pool.get('project.gtd.timebox')
        timebox_ids = timebox_obj.search(cr,uid,[])
        for task in self.browse(cr,uid,ids):
            timebox = task.timebox_id.id
            if timebox:
                if timebox_ids.index(timebox):
                    index = timebox_ids.index(timebox)
                    self.write(cr, uid, task.id, {'timebox_id': timebox_ids[index - 1]})
                else:
                    self.write(cr, uid, task.id, {'timebox_id': False})
        return True

    def fields_view_get(self, cr, uid, view_id=None, view_type='form', context=None, toolbar=False, submenu=False):
        if not context: context = {}
        res = super(project_task,self).fields_view_get(cr, uid, view_id, view_type, context, toolbar=toolbar, submenu=submenu)
        search_extended = False
        timebox_obj = self.pool.get('project.gtd.timebox')
        if (res['type'] == 'search') and context.get('gtd', False):
            tt = timebox_obj.browse(cr, uid, timebox_obj.search(cr,uid,[]), context=context)
            search_extended =''
            for time in tt:
                if time.icon:
                    icon = time.icon
                else :
                    icon=""
                search_extended += '''<filter domain="[('timebox_id','=', ''' + str(time.id) + ''')]" icon="''' + icon + '''" string="''' + time.name + '''" context="{'user_invisible': True}"/>\n'''
            search_extended +='''<separator orientation="vertical"/>'''

            res['arch'] = res['arch'].replace('<separator name="gtdsep"/>', search_extended)

        return res

project_task()

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>[MERGE] lp: 944761<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import sys

from osv import fields, osv
import tools
from tools.translate import _

class project_gtd_context(osv.osv):
    _name = "project.gtd.context"
    _description = "Context"
    _columns = {
        'name': fields.char('Context', size=64, required=True, select=1, translate=1),
        'sequence': fields.integer('Sequence', help="Gives the sequence order when displaying a list of contexts."),
    }
    _defaults = {
        'sequence': 1
    }
    _order = "sequence, name"

project_gtd_context()

class project_gtd_timebox(osv.osv):
    _name = "project.gtd.timebox"
    _order = "sequence"
    _columns = {
        'name': fields.char('Timebox', size=64, required=True, select=1, translate=1),
        'sequence': fields.integer('Sequence', help="Gives the sequence order when displaying a list of timebox."),
        'icon': fields.selection(tools.icons, 'Icon', size=64),
    }

project_gtd_timebox()

class project_task(osv.osv):
    _inherit = "project.task"
    _columns = {
        'timebox_id': fields.many2one('project.gtd.timebox', "Timebox",help="Time-laps during which task has to be treated"),
        'context_id': fields.many2one('project.gtd.context', "Context",help="The context place where user has to treat task"),
     }

    def copy_data(self, cr, uid, id, default=None, context=None):
        if context is None:
            context = {}
        if not default:
            default = {}
        default['timebox_id'] = False
        default['context_id'] = False
        return super(project_task,self).copy_data(cr, uid, id, default, context)

    def _get_context(self,cr, uid, context=None):
        ids = self.pool.get('project.gtd.context').search(cr, uid, [], context=context)
        return ids and ids[0] or False

    _defaults = {
        'context_id': _get_context
    }
    def next_timebox(self, cr, uid, ids, *args):
        timebox_obj = self.pool.get('project.gtd.timebox')
        timebox_ids = timebox_obj.search(cr,uid,[])
        if not timebox_ids: return True
        for task in self.browse(cr,uid,ids):
            timebox = task.timebox_id.id
            if not timebox:
                self.write(cr, uid, task.id, {'timebox_id': timebox_ids[0]})
            elif timebox_ids.index(timebox) != len(timebox_ids)-1:
                index = timebox_ids.index(timebox)
                self.write(cr, uid, task.id, {'timebox_id': timebox_ids[index+1]})
        return True

    def prev_timebox(self, cr, uid, ids, *args):
        timebox_obj = self.pool.get('project.gtd.timebox')
        timebox_ids = timebox_obj.search(cr,uid,[])
        for task in self.browse(cr,uid,ids):
            timebox = task.timebox_id.id
            if timebox:
                if timebox_ids.index(timebox):
                    index = timebox_ids.index(timebox)
                    self.write(cr, uid, task.id, {'timebox_id': timebox_ids[index - 1]})
                else:
                    self.write(cr, uid, task.id, {'timebox_id': False})
        return True

    def fields_view_get(self, cr, uid, view_id=None, view_type='form', context=None, toolbar=False, submenu=False):
        if not context: context = {}
        res = super(project_task,self).fields_view_get(cr, uid, view_id, view_type, context, toolbar=toolbar, submenu=submenu)
        search_extended = False
        timebox_obj = self.pool.get('project.gtd.timebox')
        if (res['type'] == 'search') and context.get('gtd', False):
            tt = timebox_obj.browse(cr, uid, timebox_obj.search(cr,uid,[]), context=context)
            search_extended =''
            for time in tt:
                if time.icon:
                    icon = time.icon
                else :
                    icon=""
                search_extended += '''<filter domain="[('timebox_id','=', ''' + str(time.id) + ''')]" icon="''' + icon + '''" string="''' + time.name + '''" context="{'user_invisible': True}"/>\n'''
            search_extended +='''<separator orientation="vertical"/>'''

            res['arch'] = tools.ustr(res['arch']).replace('<separator name="gtdsep"/>', search_extended)

        return res

project_task()

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>


<commit_before>#!/usr/bin/python3
#
#  Copyright (C) 2019 Angus King
#
#  zoompan.py - This file is used by SIREN.
#
#  This is free software: you can redistribute it and/or modify
#  it under the terms of the GNU Affero General Public License as
#  published by the Free Software Foundation, either version 3 of
#  the License, or (at your option) any later version.
#
#  SIREN is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU Affero General Public License for more details.
#
#  You should have received a copy of the GNU Affero General
#  Public License along with SIREN.  If not, see
#  <http://www.gnu.org/licenses/>.
#
# Based on: https://gist.github.com/tacaswell/3144287

the_days = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]

class ZoomPanX():
    def __init__(self):
        self.base_xlim = None
        self.tbar = None
        self.cur_xlim = None
        self.xpress = None
        self.month = None
        self.week = None
        self.keys = ''

    def zoom_pan(self, ax, base_scale=2.):
        def zoom(event):
            if event.inaxes != ax:
                return
            if self.base_xlim is None:
                self.base_xlim = ax.get_xlim()
            # get the current x and y limits
            cur_xlim = ax.get_xlim()
            cur_ylim = ax.get_ylim()
            xdata = event.xdata # get event x location
            ydata = event.ydata # get event y location
            if event.button == 'up':
                # deal with zoom in
                scale_factor = 1 / base_scale
            elif event.button == 'down':
                # deal with zoom out
                scale_factor = base_scale
            else:
                # deal with something that should never happen
                scale_factor = 1
                print('(56)', event.button)
            # set new limits
            # Get distance from the cursor to the edge of the figure frame
            x_left = xdata - cur_xlim[0]
            x_right = cur_xlim[1] - xdata
            y_top = ydata - cur_ylim[0]
            y_bottom = cur_ylim[1] - ydata
            ax.set_xlim([xdata - x_left*scale_factor,
                        xdata + x_right*scale_factor])
            ax.figure.canvas.draw() # force re-draw

        def onPress(event):
            if self.base_xlim is None:
                self.base_xlim = ax.get_xlim()
            if self.tbar._active is not None:
                return
            if event.button == 3: # reset?
                self.month = None
                self.week = None
                if self.base_xlim is not None:
                    ax.set_xlim(self.base_xlim)
                    ax.figure.canvas.draw()
                    return
            if event.inaxes != ax:
                return
            self.cur_xlim = ax.get_xlim()
            self.xpress = event.xdata

        def onRelease(event):
            self.xpress = None
            ax.figure.canvas.draw()

        def onMotion(event):
            if self.base_xlim is None:
                self.base_xlim = ax.get_xlim()
            if self.xpress is None:
                return
            if event.inaxes != ax:
                return
            dx = event.xdata - self.xpress
            self.cur_xlim -= dx
            ax.set_xlim(self.cur_xlim)
            ax.figure.canvas.draw()

        def onKey(event):
            if event.key.lower() == 'm':
                self.keys = 'm'
                self.week = None
                if self.month is None:
                    self.month = 0
                elif self.month >= 11:
                    self.month = 0
                else:
                    self.month += 1
                ax.set_xlim([self.mth_xlim[self.month], self.mth_xlim[self.month + 1]])
                ax.figure.canvas.draw()
            elif event.key.lower() == 'w':
                self.keys = ''
                if self.week is None:
                    self.week = 0
                else:
                    self.week += 1
                if self.month is None:
                    base = 0
                    if self.week >= 52:
                        self.week = 0
                    strt = self.week * 168 # 24 * 7 hours per week
                else:
                    if self.week >= 5 or \
                      (self.month == 1 and self.week >= 4 and self.mth_xlim[2] == 1416):
                        self.week = 0
                    strt = self.mth_xlim[self.month] + self.week * 168
                ax.set_xlim([strt, strt + 168])
                ax.figure.canvas.draw()
            elif event.key.lower() == 'r':
                self.keys = ''
                self.month = None
                self.week = None
                if self.base_xlim is not None:
                    ax.set_xlim(self.base_xlim)
                    ax.figure.canvas.draw()
                    return
            elif event.key >= '0' and event.key <= '9':
                if self.keys[-2:] == 'm1':
                    self.keys = ''
                    if event.key < '3':
                        self.month = 10 + int(event.key) - 1
                    else:
                        return
                elif self.keys[-1:] == 'm':
                    if event.key == '0':
                        self.month = 0
                    else:
                        self.month = int(event.key) - 1
                    self.keys += event.key
                ax.set_xlim([self.mth_xlim[self.month], self.mth_xlim[self.month + 1]])
                ax.figure.canvas.draw()
        the_days = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
        self.base_xlim = ax.get_xlim() # remember x base
        if self.base_xlim[1] == 8784: # leap year
            the_days[1] = 29
        x = 0
        self.mth_xlim = [x]
        for days in the_days:
            x += days * 24
            self.mth_xlim.append(x)
        fig = ax.get_figure() # get the figure of interest
        self.tbar = fig.canvas.toolbar # get toolbar
        # attach the call back
        fig.canvas.mpl_connect('scroll_event', zoom)
        fig.canvas.mpl_connect('button_press_event', onPress)
        fig.canvas.mpl_connect('button_release_event', onRelease)
        fig.canvas.mpl_connect('motion_notify_event', onMotion)
        fig.canvas.mpl_connect('key_press_event', onKey)
        #return the function
        return zoom
<commit_msg>Move ZoomPanX into seperate file<commit_after>#!/usr/bin/python3
#
#  Copyright (C) 2019 Angus King
#
#  zoompan.py - This file is used by SIREN.
#
#  This is free software: you can redistribute it and/or modify
#  it under the terms of the GNU Affero General Public License as
#  published by the Free Software Foundation, either version 3 of
#  the License, or (at your option) any later version.
#
#  SIREN is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU Affero General Public License for more details.
#
#  You should have received a copy of the GNU Affero General
#  Public License along with SIREN.  If not, see
#  <http://www.gnu.org/licenses/>.
#
# Based on: https://gist.github.com/tacaswell/3144287

the_days = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]

class ZoomPanX():
    def __init__(self):
        self.base_xlim = None
        self.tbar = None
        self.cur_xlim = None
        self.xpress = None
        self.month = None
        self.week = None
        self.keys = ''

    def zoom_pan(self, ax, base_scale=2.):
        def zoom(event):
            if event.inaxes != ax:
                return
            if self.base_xlim is None:
                self.base_xlim = ax.get_xlim()
            # get the current x and y limits
            cur_xlim = ax.get_xlim()
            cur_ylim = ax.get_ylim()
            xdata = event.xdata # get event x location
            ydata = event.ydata # get event y location
            if event.button == 'up':
                # deal with zoom in
                scale_factor = 1 / base_scale
            elif event.button == 'down':
                # deal with zoom out
                scale_factor = base_scale
            else:
                # deal with something that should never happen
                scale_factor = 1
                print('(56)', event.button)
            # set new limits
            # Get distance from the cursor to the edge of the figure frame
            x_left = xdata - cur_xlim[0]
            x_right = cur_xlim[1] - xdata
            y_top = ydata - cur_ylim[0]
            y_bottom = cur_ylim[1] - ydata
            ax.set_xlim([xdata - x_left*scale_factor,
                        xdata + x_right*scale_factor])
            ax.figure.canvas.draw() # force re-draw

        def onPress(event):
            if self.base_xlim is None:
                self.base_xlim = ax.get_xlim()
            if self.tbar._active is not None:
                return
            if event.button == 3: # reset?
                self.month = None
                self.week = None
                if self.base_xlim is not None:
                    ax.set_xlim(self.base_xlim)
                    ax.figure.canvas.draw()
                    return
            if event.inaxes != ax:
                return
            self.cur_xlim = ax.get_xlim()
            self.xpress = event.xdata

        def onRelease(event):
            self.xpress = None
            ax.figure.canvas.draw()

        def onMotion(event):
            if self.base_xlim is None:
                self.base_xlim = ax.get_xlim()
            if self.xpress is None:
                return
            if event.inaxes != ax:
                return
            dx = event.xdata - self.xpress
            self.cur_xlim -= dx
            ax.set_xlim(self.cur_xlim)
            ax.figure.canvas.draw()

        def onKey(event):
            if event.key.lower() == 'r':
                self.keys = ''
                self.month = None
                self.week = None
                if self.base_xlim is not None:
                    ax.set_xlim(self.base_xlim)
                    ax.figure.canvas.draw()
                    return
            if event.key == 'pageup':
                if self.week is not None:
                    self.week -= 1
                    if self.month is None:
                        if self.week < 0:
                            self.week = 52
                        strt = self.week * 168 # 24 * 7 hours per week
                    else:
                        if self.week < 0:
                            if self.month == 1 and self.mth_xlim[2] == 1416:
                                self.week = 3
                            else:
                                self.week = 4
                        strt = self.mth_xlim[self.month] + self.week * 168
                    ax.set_xlim([strt, strt + 168])
                else:
                    if self.month is None or self.month == 0:
                        self.month = 11
                    else:
                        self.month -= 1
                    ax.set_xlim([self.mth_xlim[self.month], self.mth_xlim[self.month + 1]])
                ax.figure.canvas.draw()
            elif event.key == 'pagedown':
                if self.week is not None:
                    self.week += 1
                    if self.month is None:
                        if self.week >= 52:
                            self.week = 0
                        strt = self.week * 168 # 24 * 7 hours per week
                    else:
                        if self.week >= 5 or \
                          (self.month == 1 and self.week >= 4 and self.mth_xlim[2] == 1416):
                            self.week = 0
                        strt = self.mth_xlim[self.month] + self.week * 168
                    ax.set_xlim([strt, strt + 168])
                else:
                    if self.month is None:
                        self.month = 0
                    else:
                        if self.month >= 11:
                            self.month = 0
                        else:
                            self.month += 1
                    ax.set_xlim([self.mth_xlim[self.month], self.mth_xlim[self.month + 1]])
                ax.figure.canvas.draw()
            elif event.key.lower() == 'm':
                self.keys = 'm'
                self.week = None
                if self.month is None:
                    self.month = 0
                elif self.month >= 11:
                    self.month = 0
                else:
                    self.month += 1
                ax.set_xlim([self.mth_xlim[self.month], self.mth_xlim[self.month + 1]])
                ax.figure.canvas.draw()
            elif event.key.lower() == 'w':
                self.keys = ''
                if self.week is None:
                    self.week = 0
                else:
                    self.week += 1
                if self.month is None:
                    if self.week >= 52:
                        self.week = 0
                    strt = self.week * 168 # 24 * 7 hours per week
                else:
                    if self.week >= 5 or \
                      (self.month == 1 and self.week >= 4 and self.mth_xlim[2] == 1416):
                        self.week = 0
                    strt = self.mth_xlim[self.month] + self.week * 168
                ax.set_xlim([strt, strt + 168])
                ax.figure.canvas.draw()
            elif event.key >= '0' and event.key <= '9':
                if self.keys[-2:] == 'm1':
                    self.keys = ''
                    if event.key < '3':
                        self.month = 10 + int(event.key) - 1
                    else:
                        return
                elif self.keys[-1:] == 'm':
                    if event.key == '0':
                        self.month = 0
                    else:
                        self.month = int(event.key) - 1
                    self.keys += event.key
                ax.set_xlim([self.mth_xlim[self.month], self.mth_xlim[self.month + 1]])
                ax.figure.canvas.draw()
        the_days = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
        self.base_xlim = ax.get_xlim() # remember x base
        if self.base_xlim[1] == 8784: # leap year
            the_days[1] = 29
        x = 0
        self.mth_xlim = [x]
        for days in the_days:
            x += days * 24
            self.mth_xlim.append(x)
        fig = ax.get_figure() # get the figure of interest
        self.tbar = fig.canvas.toolbar # get toolbar
        # attach the call back
        fig.canvas.mpl_connect('scroll_event', zoom)
        fig.canvas.mpl_connect('button_press_event', onPress)
        fig.canvas.mpl_connect('button_release_event', onRelease)
        fig.canvas.mpl_connect('motion_notify_event', onMotion)
        fig.canvas.mpl_connect('key_press_event', onKey)
        #return the function
        return zoom
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# This module is part of the GeoTag-X project formatter tool.
# It contains formatters for various components of a project configuration.
#
# Author: Jeremy Othieno (j.othieno@gmail.com)
#
# Copyright (c) 2017 UNITAR/UNOSAT
#
# The MIT License (MIT)
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
# OR OTHER DEALINGS IN THE SOFTWARE.
from geotagx_validator.helper import check_arg_type
import geotagx_validator.project as validator

def format_project_configuration(configuration, validate_configuration=True):
    """Formats the specified project configuration.

    Args:
        configuration (dict): A project configuration to format.

    Raises:
        TypeError: If the configuration argument is not a dictionary, or
            validate_configuration is not a boolean.
        ValueError: If the specified configuration is not a valid project configuration.
    """
    check_arg_type(format_project_configuration, "configuration", configuration, dict)
    check_arg_type(format_project_configuration, "validate_configuration", validate_configuration, bool)

    if validate_configuration:
        valid, message = validator.is_project_configuration(configuration)
        if not valid:
            raise ValueError(message)

    formatters = {
        "name": format_project_name,
        "description": format_project_description,
        "repository": format_project_repository,
    }
    for key in configuration:
        formatter = formatters.get(key)
        if formatter:
            configuration[key] = formatter(configuration[key], False)

    return configuration

def format_project_name(name, validate_name=True):
    raise NotImplementedError

def format_project_description(description, validate_description=True):
    raise NotImplementedError

def format_project_repository(repository, validate_repository=True):
    raise NotImplementedError
<commit_msg>Implement the project configuration formatter functions<commit_after># -*- coding: utf-8 -*-
#
# This module is part of the GeoTag-X project formatter tool.
# It contains formatters for various components of a project configuration.
#
# Author: Jeremy Othieno (j.othieno@gmail.com)
#
# Copyright (c) 2017 UNITAR/UNOSAT
#
# The MIT License (MIT)
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
# OR OTHER DEALINGS IN THE SOFTWARE.
from geotagx_validator.helper import check_arg_type
import geotagx_validator.project as validator

def format_project_configuration(configuration, validate_configuration=True):
    """Formats the specified project configuration.

    Args:
        configuration (dict): A project configuration to format.

    Raises:
        TypeError: If the configuration argument is not a dictionary, or
            validate_configuration is not a boolean.
        ValueError: If the specified configuration is not a valid project configuration.
    """
    check_arg_type(format_project_configuration, "configuration", configuration, dict)
    check_arg_type(format_project_configuration, "validate_configuration", validate_configuration, bool)

    if validate_configuration:
        valid, message = validator.is_project_configuration(configuration)
        if not valid:
            raise ValueError(message)

    formatters = {
        "name": format_project_name,
        "description": format_project_description,
        "repository": format_project_repository,
    }
    for key in configuration:
        formatter = formatters.get(key)
        if formatter:
            configuration[key] = formatter(configuration[key], False)

    return configuration

def format_project_name(name, validate_name=True):
    """Formats the specified project name.

    Formatting a project name will simply remove any leading and trailing whitespace.

    Args:
        name (str): A project name to format.
        validate_name (bool): If set to True, the specified project name will be
            validated before it's processed.

    Returns:
        basestring: The formatted project name.

    Raises:
        TypeError: If the name argument is not a basestring instance, or
            validate_name is not a boolean.
        ValueError: If the specified name is not valid.
    """
    check_arg_type(format_project_name, "name", name, basestring)
    check_arg_type(format_project_name, "validate_name", validate_name, bool)

    if validate_name:
        valid, message = validator.is_project_name(name)
        if not valid:
            raise ValueError(message)

    return name.strip()

def format_project_description(description, validate_description=True):
    """Formats the specified project description.

    Formatting a project description will simply remove any leading and trailing
    whitespace.

    Args:
        description (str): A project description to format.
        validate_description (bool): If set to True, the specified project
            description will be validated before it's processed.

    Returns:
        basestring: The formatted project description.

    Raises:
        TypeError: If the description argument is not a basestring instance, or
            validate_description is not a boolean.
        ValueError: If the specified description is not valid.
    """
    check_arg_type(format_project_description, "description", description, basestring)
    check_arg_type(format_project_description, "validate_description", validate_description, bool)

    if validate_description:
        valid, message = validator.is_project_description(description)
        if not valid:
            raise ValueError(message)

    return description.strip()

def format_project_repository(repository, validate_repository=True):
    """Formats the specified project repository.

    Formatting a project repository will simply remove any leading and trailing
    whitespace.

    Args:
        repository (str): A project repository to format.
        validate_repository (bool): If set to True, the specified project
            repository will be validated before it's processed.

    Returns:
        basestring: The formatted project repository.

    Raises:
        TypeError: If the repository argument is not a basestring instance, or
            validate_repository is not a boolean.
        ValueError: If the specified repository is not valid.
    """
    check_arg_type(format_project_repository, "repository", repository, basestring)
    check_arg_type(format_project_repository, "validate_repository", validate_repository, bool)

    if validate_repository:
        valid, message = validator.is_project_repository(repository)
        if not valid:
            raise ValueError(message)

    return repository.strip()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.shortcuts import render
from django.http import HttpResponse
# Create your views here.
def index(request):
    return HttpResponse("
this is your Info page
")
def detail(request, staff_id):
    return HttpResponse("
Details for staff id: "+ str(staff_id)+"
")
<commit_msg>connect to the database and get each staff name in the index page<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals
from models import Staff
from django.shortcuts import render
from django.http import HttpResponse
# Create your views here.
def index(request):
    #connect to the database. And get all data in Staff
    all_staff = Staff.objects.all()
    html = ''
    for staff in all_staff:
        url = '/info/' + str(staff.id) +"/"
        html += '' + staff.staff_name + '
'

    return HttpResponse(html)
def detail(request, staff_id):
    return HttpResponse("
Details for staff id: "+ str(staff_id)+"
")
<|endoftext|>


<commit_before>#!/usr/bin/python

# Copyright (c) 2008-2011 Mark Eichin <eichin@thok.org>
# See ./LICENSE (MIT style.)

"""take recent twitters and zephyr them to me"""

__version__ = "0.2"
__author__  = "Mark Eichin <eichin@thok.org>"
__license__ = "MIT"

import sys
import os
import getpass
import subprocess
import signal
import tweepy
import time

def get_oauth_info(appname=None):
    """get this user's oauth info"""
    filebase = os.path.expanduser("~/.ztwit_")
    if appname:
        # default path is ztwitgw
        filebase += appname + "_"
    key, secret = file(filebase + "oauth", "r").read().strip().split(":", 1)
    return key, secret

# TODO: write get_verifier_X11

def get_verifier_tty(output_path, appname=None):
    """we don't have a verifier, ask the user to use a browser and get one"""
    consumer_token, consumer_secret = get_oauth_info(appname=appname)
    auth = tweepy.OAuthHandler(consumer_token, consumer_secret)
    redirect_url = auth.get_authorization_url() # tweepy.TweepError
    print "Open this URL in a browser where you're logged in to twitter:"
    print redirect_url
    verifier = raw_input("Enter (cut&paste) the response code: ")
    # use it...
    auth.get_access_token(verifier)
    # hmm, discard the verifier?
    file(output_path, "wb").write(":".join([auth.request_token.key,
                                            auth.request_token.secret,
                                            auth.access_token.key,
                                            auth.access_token.secret,
                                            verifier]))

def get_just_verifier(output_path, appname=None):
    """ask for the verifier *without* having consumer info"""
    auth = tweepy.OAuthHandler("", "")
    # TODO: this can't work unless we first give the user a redirect to the
    #    URL to *get* the response code. and possibly not then?
    verifier = raw_input("Enter (cut&paste) the response code: ")
    # use it...
    auth.get_access_token(verifier)
    # hmm, discard the verifier?
    file(output_path, "wb").write(":".join([auth.request_token.key,
                                            auth.request_token.secret,
                                            auth.access_token.key,
                                            auth.access_token.secret,
                                            verifier]))

def get_oauth_verifier(fallback_mechanism, appname=None):
    """get the request token and verifier, using fallback_mechanism if we don't have one stashed"""
    filebase = os.path.expanduser("~/.ztwit_")
    if appname:
        # default path is ztwitgw
        filebase += appname + "_"
    verifier_file = filebase + "oauth_verifier"
    if not os.path.exists(verifier_file):
        fallback_mechanism(verifier_file, appname=appname)
        if not os.path.exists(verifier_file):
            raise Exception("Fallback Failed")
    rt_key, rt_secret, at_key, at_secret, verifier = file(verifier_file, "r").read().strip().split(":", 4)
    return rt_key, rt_secret, at_key, at_secret, verifier

# do this with a localhost url?

def zwrite(username, body, tag, status_id=None):
    """deliver one twitter message to zephyr"""
    # username... will get encoded when we see one
    body = body.encode("iso-8859-1", "xmlcharrefreplace")
    # example syntax: http://twitter.com/engadget/status/18164103530
    zurl = " http://twitter.com/%s/status/%s" % (username, status_id) if status_id else ""
    zsig = "%s %s%svia ztwitgw%s" % (username, tag, tag and " ", zurl)
    # tag is from codde
    cmd = ["zwrite",
           "-q", # quiet
           "-d", # Don't authenticate
           "-s", zsig,
           "-c", "%s.twitter" % getpass.getuser(),
           "-i", username,
           "-m", body]
    subprocess.check_call(cmd)

def entity_decode(txt):
    """decode simple entities"""
    # TODO: find out what ones twitter considers defined,
    #   or if sgmllib.entitydefs is enough...
    return txt.replace(">", ">").replace("<", "<").replace("&", "&")

# turns out we don't actually see & in practice...
assert entity_decode("-> <3") == "-> <3"

def url_expander(twit, body):
    """expand urls in the body, safely"""
    expcount = 0
    try:
        # https://dev.twitter.com/docs/tweet-entities
        # do media later, stick with urls for now
        for urlblock in twit.entities.get("urls", []):
            if "expanded_url" in urlblock:
                low, high = urlblock["indices"]
                body = body[:low] + urlblock["expanded_url"] + body[high:]
                expcount += 1
        if expcount:
            return body + ("\n[expanded %s urls]" % expcount)
        return body
    except Exception, exc:
        return body + ("[expander failed: %s]" % exc)

def process_new_twits(api, proto=None, tag=""):
    """process new messages, stashing markers"""
    if proto is None:
        proto = api.friends_timeline

    filebase = os.path.expanduser("~/.ztwit_")
    if tag:
        filebase = filebase + tag + "_"
    sincefile = filebase + "since"
    since_id = None
    if os.path.exists(sincefile):
        since_id = file(sincefile, "r").read().strip()
        # if since_id: # allow for truncated file

    # the iterators *have* a prev, but there's no way to "start" at since_id?
    # favorites.json doesn't take an id arg, and it's not like we save anything
    # (other than parsing) by walking up and then down, since the json for the
    # entire set is loaded anyway...
    for twit in reversed(list(tweepy.Cursor(proto, since_id=since_id, include_entities=1).items())):
        # reversed?
        if not twit:
            print "huh? empty twit"
            continue
        # type(twit) == tweepy.models.Status
        # type(twit.author) == tweepy.models.User
        who = twit.author.screen_name
        what = entity_decode(url_expander(twit, twit.text))
        status_id = twit.id_str  # to construct a link
        zwrite(who, what, tag, status_id)
        since_id = status_id
        print "Sent:", since_id
        time.sleep(3)

    # Note that since_id is just an ordering - if I favorite an old tweet (even
    # something that showed up new because it was freshly retweeted) it doesn't
    # show up.  This isn't a new bug, I'm just noticing it...
    newsince = file(sincefile, "w")
    print >> newsince, since_id
    newsince.close()

if __name__ == "__main__":
    signal.alarm(5*60) # been seeing some hangs, give up after a bit
    prog, = sys.argv

    rt_key, rt_secret, at_key, at_secret, verifier = get_oauth_verifier(get_verifier_tty)
    consumer_token, consumer_secret = get_oauth_info()
    auth = tweepy.OAuthHandler(consumer_token, consumer_secret)
    auth.set_request_token(rt_key, rt_secret)
    auth.set_access_token(at_key, at_secret)
    print "ct:", consumer_token
    print "cs:", consumer_secret
    print "rk:", rt_key
    print "rs:", rt_secret
    print "vf:", verifier
    print "ak:", at_key
    print "as:", at_secret
    api = tweepy.API(auth)

    process_new_twits(api)
    process_new_twits(api, proto=api.mentions, tag="reply")
    # replies_url = "http://twitter.com/statuses/replies.json"
    # but that's not in tweepy... try hacking it?
    # hmm, not in http://apiwiki.twitter.com/w/page/22554679/Twitter-API-Documentation either
    process_new_twits(api, proto=api.favorites, tag="favorites")
<commit_msg>expanded_url is actually present, but it can still be None for non-t.co links<commit_after>#!/usr/bin/python

# Copyright (c) 2008-2011 Mark Eichin <eichin@thok.org>
# See ./LICENSE (MIT style.)

"""take recent twitters and zephyr them to me"""

__version__ = "0.2"
__author__  = "Mark Eichin <eichin@thok.org>"
__license__ = "MIT"

import sys
import os
import getpass
import subprocess
import signal
import tweepy
import time

def get_oauth_info(appname=None):
    """get this user's oauth info"""
    filebase = os.path.expanduser("~/.ztwit_")
    if appname:
        # default path is ztwitgw
        filebase += appname + "_"
    key, secret = file(filebase + "oauth", "r").read().strip().split(":", 1)
    return key, secret

# TODO: write get_verifier_X11

def get_verifier_tty(output_path, appname=None):
    """we don't have a verifier, ask the user to use a browser and get one"""
    consumer_token, consumer_secret = get_oauth_info(appname=appname)
    auth = tweepy.OAuthHandler(consumer_token, consumer_secret)
    redirect_url = auth.get_authorization_url() # tweepy.TweepError
    print "Open this URL in a browser where you're logged in to twitter:"
    print redirect_url
    verifier = raw_input("Enter (cut&paste) the response code: ")
    # use it...
    auth.get_access_token(verifier)
    # hmm, discard the verifier?
    file(output_path, "wb").write(":".join([auth.request_token.key,
                                            auth.request_token.secret,
                                            auth.access_token.key,
                                            auth.access_token.secret,
                                            verifier]))

def get_just_verifier(output_path, appname=None):
    """ask for the verifier *without* having consumer info"""
    auth = tweepy.OAuthHandler("", "")
    # TODO: this can't work unless we first give the user a redirect to the
    #    URL to *get* the response code. and possibly not then?
    verifier = raw_input("Enter (cut&paste) the response code: ")
    # use it...
    auth.get_access_token(verifier)
    # hmm, discard the verifier?
    file(output_path, "wb").write(":".join([auth.request_token.key,
                                            auth.request_token.secret,
                                            auth.access_token.key,
                                            auth.access_token.secret,
                                            verifier]))

def get_oauth_verifier(fallback_mechanism, appname=None):
    """get the request token and verifier, using fallback_mechanism if we don't have one stashed"""
    filebase = os.path.expanduser("~/.ztwit_")
    if appname:
        # default path is ztwitgw
        filebase += appname + "_"
    verifier_file = filebase + "oauth_verifier"
    if not os.path.exists(verifier_file):
        fallback_mechanism(verifier_file, appname=appname)
        if not os.path.exists(verifier_file):
            raise Exception("Fallback Failed")
    rt_key, rt_secret, at_key, at_secret, verifier = file(verifier_file, "r").read().strip().split(":", 4)
    return rt_key, rt_secret, at_key, at_secret, verifier

# do this with a localhost url?

def zwrite(username, body, tag, status_id=None):
    """deliver one twitter message to zephyr"""
    # username... will get encoded when we see one
    body = body.encode("iso-8859-1", "xmlcharrefreplace")
    # example syntax: http://twitter.com/engadget/status/18164103530
    zurl = " http://twitter.com/%s/status/%s" % (username, status_id) if status_id else ""
    zsig = "%s %s%svia ztwitgw%s" % (username, tag, tag and " ", zurl)
    # tag is from codde
    cmd = ["zwrite",
           "-q", # quiet
           "-d", # Don't authenticate
           "-s", zsig,
           "-c", "%s.twitter" % getpass.getuser(),
           "-i", username,
           "-m", body]
    subprocess.check_call(cmd)

def entity_decode(txt):
    """decode simple entities"""
    # TODO: find out what ones twitter considers defined,
    #   or if sgmllib.entitydefs is enough...
    return txt.replace(">", ">").replace("<", "<").replace("&", "&")

# turns out we don't actually see & in practice...
assert entity_decode("-> <3") == "-> <3"

def url_expander(twit, body):
    """expand urls in the body, safely"""
    expcount = 0
    try:
        # https://dev.twitter.com/docs/tweet-entities
        # do media later, stick with urls for now
        for urlblock in twit.entities.get("urls", []):
            if urlblock.get("expanded_url"):
                low, high = urlblock["indices"]
                body = body[:low] + urlblock["expanded_url"] + body[high:]
                expcount += 1
        if expcount:
            return body + ("\n[expanded %s urls]" % expcount)
        return body
    except Exception, exc:
        return body + ("[expander failed: %s]" % exc)

def process_new_twits(api, proto=None, tag=""):
    """process new messages, stashing markers"""
    if proto is None:
        proto = api.friends_timeline

    filebase = os.path.expanduser("~/.ztwit_")
    if tag:
        filebase = filebase + tag + "_"
    sincefile = filebase + "since"
    since_id = None
    if os.path.exists(sincefile):
        since_id = file(sincefile, "r").read().strip()
        # if since_id: # allow for truncated file

    # the iterators *have* a prev, but there's no way to "start" at since_id?
    # favorites.json doesn't take an id arg, and it's not like we save anything
    # (other than parsing) by walking up and then down, since the json for the
    # entire set is loaded anyway...
    for twit in reversed(list(tweepy.Cursor(proto, since_id=since_id, include_entities=1).items())):
        # reversed?
        if not twit:
            print "huh? empty twit"
            continue
        # type(twit) == tweepy.models.Status
        # type(twit.author) == tweepy.models.User
        who = twit.author.screen_name
        what = entity_decode(url_expander(twit, twit.text))
        status_id = twit.id_str  # to construct a link
        zwrite(who, what, tag, status_id)
        since_id = status_id
        print "Sent:", since_id
        time.sleep(3)

    # Note that since_id is just an ordering - if I favorite an old tweet (even
    # something that showed up new because it was freshly retweeted) it doesn't
    # show up.  This isn't a new bug, I'm just noticing it...
    newsince = file(sincefile, "w")
    print >> newsince, since_id
    newsince.close()

if __name__ == "__main__":
    signal.alarm(5*60) # been seeing some hangs, give up after a bit
    prog, = sys.argv

    rt_key, rt_secret, at_key, at_secret, verifier = get_oauth_verifier(get_verifier_tty)
    consumer_token, consumer_secret = get_oauth_info()
    auth = tweepy.OAuthHandler(consumer_token, consumer_secret)
    auth.set_request_token(rt_key, rt_secret)
    auth.set_access_token(at_key, at_secret)
    print "ct:", consumer_token
    print "cs:", consumer_secret
    print "rk:", rt_key
    print "rs:", rt_secret
    print "vf:", verifier
    print "ak:", at_key
    print "as:", at_secret
    api = tweepy.API(auth)

    process_new_twits(api)
    process_new_twits(api, proto=api.mentions, tag="reply")
    # replies_url = "http://twitter.com/statuses/replies.json"
    # but that's not in tweepy... try hacking it?
    # hmm, not in http://apiwiki.twitter.com/w/page/22554679/Twitter-API-Documentation either
    process_new_twits(api, proto=api.favorites, tag="favorites")
<|endoftext|>


<commit_before>from django.core.exceptions import ObjectDoesNotExist
from data_collection.base_importers import BaseStationsAddressesImporter
from addressbase.models import Address

class Command(BaseStationsAddressesImporter):
    srid = 27700
    districts_srid = 27700
    stations_filetype = 'shp'
    addresses_filetype = 'shp'
    council_id = 'E07000151'
    addresses_name = 'UPRN_PollingStationRef.shp'
    stations_name = 'DDC Polling Stations.shp'
    elections = [
        'local.northamptonshire.2017-05-04',
        'parl.2017-06-08'
    ]

    def address_record_to_dict(self, record):
        try:
            address = Address.objects.get(pk=record.record[1])
        except ObjectDoesNotExist:
            return None

        return {
            'address': address.address,
            'postcode': address.postcode,
            'polling_station_id': record.record[0],
        }

    def format_address(self, record):
        address_parts = [record[x].strip() for x in range(3, 7)]
        for i, part in enumerate(address_parts):
            if part == b'':
                address_parts[i] = ''
        address = "\n".join(address_parts)
        while "\n\n" in address:
            address = address.replace("\n\n", "\n")
        return address.strip()

    def station_record_to_dict(self, record):
        codes = record[1].split(',')

        # Joe received corrections by email from Daventry:
        # Welford Polling Station – FM (Sulby) and Welford (GA)
        # AC is Althorp Polling District, but they vote at
        # Harlestone Village Institute (Polling Station ID 67)
        if record[0] == 66:
            codes = ['FM', 'GA']
        if record[0] == 67:
            codes.append('AC')

        codes = [code.strip() for code in codes]

        stations = []
        for code in codes:
            stations.append({
                'internal_council_id': code,
                'postcode': record[8].strip(),
                'address': self.format_address(record),
            })
        return stations
<commit_msg>update Daventry import script for parl.2017-06-08<commit_after>from data_collection.management.commands import BaseXpressDemocracyClubCsvImporter

class Command(BaseXpressDemocracyClubCsvImporter):
    council_id = 'E07000151'
    addresses_name = 'parl.2017-06-08/Version 1/Daventry District Council Democracy_Club__08June2017.tsv'
    stations_name = 'parl.2017-06-08/Version 1/Daventry District Council Democracy_Club__08June2017.tsv'
    elections = ['parl.2017-06-08']
    csv_delimiter = '\t'
    csv_encoding = 'windows-1252'
<|endoftext|>


<commit_before>from django.contrib.gis.geos import Point
from data_importers.management.commands import BaseHalaroseCsvImporter

class Command(BaseHalaroseCsvImporter):
    council_id = "CAB"
    addresses_name = "2021-03-23T09:23:18.089254/polling_station_export-2021-03-22.csv"
    stations_name = "2021-03-23T09:23:18.089254/polling_station_export-2021-03-22.csv"
    elections = ["2021-05-06"]
    csv_delimiter = ","

    def station_record_to_dict(self, record):
        rec = super().station_record_to_dict(record)

        if rec["internal_council_id"] == "48-the-c3-centre":
            rec["location"] = Point(0.1572, 52.2003, srid=4326)

        # user issue report #105
        if rec["internal_council_id"] == "36-trinity-old-field-pavilion":
            rec["location"] = Point(0.106029, 52.207284, srid=4326)

        # user issue report #129
        if rec["internal_council_id"] == "2-east-barnwell-community-centre":
            rec["location"] = Point(0.165576, 52.212132, srid=4326)

        # report from council
        if (
            rec["internal_council_id"] == "16-seminar-rooms-3-4-churchill-college"
            or rec["internal_council_id"] == "15-seminar-rooms-3-4-churchill-college"
        ):
            rec["location"] = Point(0.099002518, 52.215464, srid=4326)

        return rec

    def address_record_to_dict(self, record):
        uprn = record.uprn.strip().lstrip("0")

        if uprn in [
            "10090968296",  # FLAT 3, BURTON HOUSE 2A, ROCK ROAD, CAMBRIDGE
            "10090968298",  # FLAT 5, BURTON HOUSE 2A, ROCK ROAD, CAMBRIDGE
            "10090968295",  # FLAT 2, BURTON HOUSE 2A, ROCK ROAD, CAMBRIDGE
            "10090968294",  # FLAT 1, BURTON HOUSE 2A, ROCK ROAD, CAMBRIDGE
            "10090968297",  # FLAT 4, BURTON HOUSE 2A, ROCK ROAD, CAMBRIDGE
            "200004186734",  # STATION LODGE, BARNWELL JUNCTION, NEWMARKET ROAD, CAMBRIDGE
            "200004210208",  # STATION HOUSE, BARNWELL JUNCTION, NEWMARKET ROAD, CAMBRIDGE
        ]:
            return None

        if record.housepostcode in ["CB4 2QA", "CB4 1LD"]:
            return None

        return super().address_record_to_dict(record)
<commit_msg>Import script for Cambridge (2022-05-05) (closes #4391)<commit_after>from django.contrib.gis.geos import Point
from data_importers.management.commands import BaseHalaroseCsvImporter

class Command(BaseHalaroseCsvImporter):
    council_id = "CAB"
    addresses_name = (
        "2022-05-05/2022-03-22T14:45:23.854113/polling_station_export-2022-03-22.csv"
    )
    stations_name = (
        "2022-05-05/2022-03-22T14:45:23.854113/polling_station_export-2022-03-22.csv"
    )
    elections = ["2022-05-05"]

    def station_record_to_dict(self, record):
        rec = super().station_record_to_dict(record)

        if rec["internal_council_id"] == "47-the-c3-centre":
            rec["location"] = Point(0.1572, 52.2003, srid=4326)

        # user issue report #105
        if rec["internal_council_id"] == "35-trinity-old-field-pavilion":
            rec["location"] = Point(0.106029, 52.207284, srid=4326)

        # user issue report #129
        if rec["internal_council_id"] == "2-east-barnwell-community-centre":
            rec["location"] = Point(0.165576, 52.212132, srid=4326)

        # report from council
        if (
            rec["internal_council_id"] == "14-seminar-rooms-3-4-churchill-college"
            or rec["internal_council_id"] == "15-seminar-rooms-3-4-churchill-college"
        ):
            rec["location"] = Point(0.099002518, 52.215464, srid=4326)

        return rec

    def address_record_to_dict(self, record):
        if record.housepostcode in ["CB4 2QA", "CB4 1LD"]:
            return None

        return super().address_record_to_dict(record)
<|endoftext|>