<commit_before># -*- coding: iso8859-1 -*-
#
# Copyright (C) 2005 Edgewall Software
# Copyright (C) 2005 Christopher Lenz <cmlenz@gmx.de>
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.com/license.html.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://projects.edgewall.com/trac/.
#
# Author: Christopher Lenz <cmlenz@gmx.de>

import os
import sys

try:
    import pkg_resources
except ImportError:
    pkg_resources = None

TRAC_META = 'trac_plugin.txt'

__all__ = ['load_components']

def load_components(env):

    loaded_components = []

    def load_module(name):
         if name not in loaded_components:
             try:
                 module = __import__(name)
                 loaded_components.append(name)
                 return module
             except ImportError, e:
                 env.log.error('Component module %s not found',
                               name, exc_info=True)

    plugins_dir = os.path.join(env.path, 'plugins')

    def enable_modules(egg_path, modules):
        """Automatically enable any components provided by plugins loaded from
        the environment plugins directory."""
        try:
            if os.path.samefile(os.path.dirname(egg_path), plugins_dir):
                for module in modules:
                    env.config.setdefault('components', module + '.*',
                                          'enabled')
        except OSError:
            # If there was an error stat-ing a directory, just ignore it
            pass

    # Load components from the environment plugins directory
    if pkg_resources is not None: # But only if setuptools is installed!
        if hasattr(pkg_resources, 'Environment'):
            # setuptools >= 0.6
            pkg_env = pkg_resources.Environment([plugins_dir] + sys.path)
            for name in pkg_env:
                egg = pkg_env[name][0]
                modules = []

                for name in egg.get_entry_map('trac.plugins'):
                    # Load plugins declared via the `trac.plugins` entry point.
                    # This is the only supported option going forward, the
                    # others will be dropped at some point in the future.
                    env.log.debug('Loading plugin %s from %s', name,
                                  egg.location)
                    egg.activate()
                    try:
                        entry_point = egg.get_entry_info('trac.plugins', name)
                        if entry_point.module_name not in loaded_components:
                            entry_point.load()
                            modules.append(entry_point.module_name)
                            loaded_components.append(entry_point.module_name)
                    except ImportError, e:
                        env.log.error('Failed to load plugin %s from %s', name,
                                      egg.location, exc_info=True)

                else:
                    # Support for pre-entry-point plugins
                    if egg.has_metadata('trac_plugin.txt'):
                        env.log.debug('Loading plugin %s from %s', name,
                                      egg.location)
                        egg.activate()
                        for module in egg.get_metadata_lines('trac_plugin.txt'):
                            if load_module(module):
                                modules.append(module)

                if modules:
                    enable_modules(egg.location, modules)

        else:
            # setuptools < 0.6
            distributions = pkg_resources.AvailableDistributions([plugins_dir] \
                                                                 + sys.path)
            for name in distributions:
                egg = distributions[name][0]
                modules = []
                if egg.metadata.has_metadata(TRAC_META):
                    egg.install_on()
                    for module in egg.metadata.get_metadata_lines(TRAC_META):
                        if load_module(module):
                            modules.append(module)

                if modules:
                    enable_modules(egg.path, modules)

    elif os.path.exists(plugins_dir) and os.listdir(plugins_dir):
        env.log.warning('setuptools is required for plugin deployment')

    # Load default components
    from trac.db_default import default_components
    for module in default_components:
        if not module in loaded_components:
            __import__(module)
<commit_msg>`os.path.samefile()` is not implemented on Windows, so use a different approach in `trac.loader`.<commit_after># -*- coding: iso8859-1 -*-
#
# Copyright (C) 2005 Edgewall Software
# Copyright (C) 2005 Christopher Lenz <cmlenz@gmx.de>
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.com/license.html.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://projects.edgewall.com/trac/.
#
# Author: Christopher Lenz <cmlenz@gmx.de>

import os
import sys

try:
    import pkg_resources
except ImportError:
    pkg_resources = None

TRAC_META = 'trac_plugin.txt'

__all__ = ['load_components']

def load_components(env):

    loaded_components = []

    def load_module(name):
         if name not in loaded_components:
             try:
                 module = __import__(name)
                 loaded_components.append(name)
                 return module
             except ImportError, e:
                 env.log.error('Component module %s not found',
                               name, exc_info=True)

    plugins_dir = os.path.join(env.path, 'plugins')

    def enable_modules(egg_path, modules):
        """Automatically enable any components provided by plugins loaded from
        the environment plugins directory."""
        if os.path.dirname(egg_path) == os.path.realpath(plugins_dir):
            for module in modules:
                env.config.setdefault('components', module + '.*', 'enabled')

    # Load components from the environment plugins directory
    if pkg_resources is not None: # But only if setuptools is installed!
        if hasattr(pkg_resources, 'Environment'):
            # setuptools >= 0.6
            pkg_env = pkg_resources.Environment([plugins_dir] + sys.path)
            for name in pkg_env:
                egg = pkg_env[name][0]
                modules = []

                for name in egg.get_entry_map('trac.plugins'):
                    # Load plugins declared via the `trac.plugins` entry point.
                    # This is the only supported option going forward, the
                    # others will be dropped at some point in the future.
                    env.log.debug('Loading plugin %s from %s', name,
                                  egg.location)
                    egg.activate()
                    try:
                        entry_point = egg.get_entry_info('trac.plugins', name)
                        if entry_point.module_name not in loaded_components:
                            entry_point.load()
                            modules.append(entry_point.module_name)
                            loaded_components.append(entry_point.module_name)
                    except ImportError, e:
                        env.log.error('Failed to load plugin %s from %s', name,
                                      egg.location, exc_info=True)

                else:
                    # Support for pre-entry-point plugins
                    if egg.has_metadata('trac_plugin.txt'):
                        env.log.debug('Loading plugin %s from %s', name,
                                      egg.location)
                        egg.activate()
                        for module in egg.get_metadata_lines('trac_plugin.txt'):
                            if load_module(module):
                                modules.append(module)

                if modules:
                    enable_modules(egg.location, modules)

        else:
            # setuptools < 0.6
            distributions = pkg_resources.AvailableDistributions([plugins_dir] \
                                                                 + sys.path)
            for name in distributions:
                egg = distributions[name][0]
                modules = []
                if egg.metadata.has_metadata(TRAC_META):
                    egg.install_on()
                    for module in egg.metadata.get_metadata_lines(TRAC_META):
                        if load_module(module):
                            modules.append(module)

                if modules:
                    enable_modules(egg.path, modules)

    elif os.path.exists(plugins_dir) and os.listdir(plugins_dir):
        env.log.warning('setuptools is required for plugin deployment')

    # Load default components
    from trac.db_default import default_components
    for module in default_components:
        if not module in loaded_components:
            __import__(module)
<|endoftext|>


<commit_before>################################################################
# Cluster reduction of pixel data
#
# Usage: python2 train-agglo.py <training data> [options]
#
# Builds the agglo (Agglommerative clusting object) which
# reduces problem dimension

#Parse command line options and arguments
#Do this before other imports so incorrect usage message appears quickly
import optparse
parser = optparse.OptionParser()

parser.add_option("-o", action="store", type="string", dest="o")
parser.add_option("-n", action="store", type="int", dest="n")
parser.add_option("-m", action="store", type="string", dest="m")
parser.set_defaults(o="out.csv", n=14*14, m="agglo.p")
opts, args = parser.parse_args()

if(len(args) < 1):
    print "Usage: python2 train-agglo.py <training data> [options]"
    raise SystemExit(1)

infile = args[0]
outfile = opts.o
n_clusters = opts.n
model_fn = opts.m

#Other imports
import pickle
import pandas as pd
import numpy as np
from sklearn.feature_extraction.image import grid_to_graph
import sklearn.cluster as cluster

#Full resolution images
fullres = pd.read_csv(infile,header=0)

num_images = len(fullres)
image_data = fullres.iloc[ :num_images, 1:].values
labels = fullres.iloc[ :num_images, 0].values

#Unflatten to proper 2D shape and extract the connectivity graph
images = image_data.reshape(num_images,28,28)
connectivity = grid_to_graph(*images[0].shape)

#Do the clustering in feature space
agglo = cluster.FeatureAgglomeration(connectivity=connectivity, n_clusters=n_clusters)
agglo.fit(image_data)

# Save agglo for future use in workflow.
# Must use this same agglo for test data
barrel=open(model_fn,'wb')
pickle.dump(agglo,barrel)
barrel.close()

#Transform the original data into reduced feature space
image_data_reduced = agglo.transform(image_data)

#Reduced data out to file
out = pd.DataFrame(image_data_reduced)
out.insert(0, 'label', labels)#Add back the target labels
out.to_csv(outfile,index=False)

#
#Display some approx images to get a sense of what has been stripped out
# import matplotlib.pyplot as plt
#Go back to real space from eigenpixel space
# image_data_approx = agglo.inverse_transform(image_data_reduced)
# num_row, num_col  = 5, 5
# #Unflatten to 2D
# images_approx = np.reshape(image_data_approx, images.shape)
# for idx in range(0,num_row*num_col):
#     plt.subplot(num_row, num_col, idx + 1)
#     plt.axis('off')
#     plt.imshow(images_approx[idx], cmap=plt.cm.gray_r, interpolation='nearest')

# plt.show()

<commit_msg>cleanup<commit_after>################################################################
# Cluster reduction of pixel data
#
# Usage: python2 train-agglo.py <training data> [options]
#
# Builds the agglo (Agglommerative clusting object) which
# reduces problem dimension

#Parse command line options and arguments
#Do this before other imports so incorrect usage message appears quickly
import optparse
parser = optparse.OptionParser()

parser.add_option("-o", action="store", type="string", dest="o")
parser.add_option("-n", action="store", type="int", dest="n")
parser.add_option("-a", action="store", type="string", dest="a")
parser.set_defaults(o="out.csv", n=14*14, a="agglo.p")
opts, args = parser.parse_args()

if(len(args) < 1):
    print "Usage: python2 train-agglo.py <training data> [options]"
    raise SystemExit(1)

infile = args[0]
outfile = opts.o
n_clusters = opts.n
agglo_fn = opts.a

#Other imports
import pickle
import pandas as pd
import numpy as np
from sklearn.feature_extraction.image import grid_to_graph
import sklearn.cluster as cluster

#Full resolution images
fullres = pd.read_csv(infile,header=0)

num_images = len(fullres)
image_data = fullres.iloc[ :num_images, 1:].values
labels = fullres.iloc[ :num_images, 0].values

#Unflatten to proper 2D shape and extract the connectivity graph
images = image_data.reshape(num_images,28,28)
connectivity = grid_to_graph(*images[0].shape)

#Do the clustering in feature space
agglo = cluster.FeatureAgglomeration(connectivity=connectivity, n_clusters=n_clusters)
agglo.fit(image_data)

# Save agglo for future use in workflow.
# Must use this same agglo for test data
barrel=open(agglo_fn,'wb')
pickle.dump(agglo,barrel)
barrel.close()

#Transform the original data into reduced feature space
image_data_reduced = agglo.transform(image_data)

#Reduced data out to file
out = pd.DataFrame(image_data_reduced)
out.insert(0, 'label', labels)#Add back the target labels
out.to_csv(outfile,index=False)

#
#Display some approx images to get a sense of what has been stripped out
# import matplotlib.pyplot as plt
#Go back to real space from eigenpixel space
# image_data_approx = agglo.inverse_transform(image_data_reduced)
# num_row, num_col  = 5, 5
# #Unflatten to 2D
# images_approx = np.reshape(image_data_approx, images.shape)
# for idx in range(0,num_row*num_col):
#     plt.subplot(num_row, num_col, idx + 1)
#     plt.axis('off')
#     plt.imshow(images_approx[idx], cmap=plt.cm.gray_r, interpolation='nearest')

# plt.show()

<|endoftext|>


<commit_before>#!/usr/bin/python3
#
# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Train the model."""

from __future__ import division
import os
from absl import app
from absl import flags
import numpy as np
import tensorflow as tf
from absl import logging
from lib import data_provider
from lib import dataset_utils
from lib import tf_utils
from lib import hparams
from lib.ssl_framework import SSLFramework
from lib import networks

_PRINT_SPAN = 300
_CHECK_TRIAL_EARLY_STOP = 100

# Flags for model training
flags.DEFINE_string(
    "hparam_string", None, "String from which we parse hparams."
)
flags.DEFINE_string(
    "primary_dataset_name", "svhn", "Name of dataset containing primary data."
)
flags.DEFINE_string(
    "secondary_dataset_name",
    "svhn",
    "Name of dataset containing secondary data.",
)
flags.DEFINE_integer("label_map_index", 0, "Index of the label map.")
flags.DEFINE_integer(
    "n_labeled", 10000, "Number of labeled examples, or -1 for entire dataset."
)
flags.DEFINE_integer(
    "training_length", 500000, "number of steps to train for."
)
flags.DEFINE_integer("batch_size", 100, "Size of the batch")
flags.DEFINE_string(
    "consistency_model", "mean_teacher", "Which consistency model to use."
)
flags.DEFINE_string(
    "zca_input_file_path",
    "",
    "Path to ZCA input statistics. '' means don't ZCA.",
)

flags.DEFINE_float(
    "unlabeled_data_random_fraction",
    1.0,
    "The fraction of unlabeled data to use during training.",
)
flags.DEFINE_string(
    "labeled_classes_filter",
    "",
    "Comma-delimited list of class numbers from labeled "
    "dataset to use during training. Defaults to all classes.",
)
flags.DEFINE_string(
    "unlabeled_classes_filter",
    "",
    "Comma-delimited list of class numbers from unlabeled "
    "dataset to use during training. Useful for labeled "
    "datasets being used as unlabeled data. Defaults to all "
    "classes.",
)

# Flags for book-keeping
flags.DEFINE_string(
    "root_dir", None, "The overall dir in which we store experiments"
)
flags.mark_flag_as_required("root_dir")

flags.DEFINE_string(
    "experiment_name", "default", "The name of this particular experiment"
)
flags.DEFINE_string(
    "load_checkpoint",
    "",
    "Checkpoint file to start training from (e.g. "
    ".../model.ckpt-354615), or None for random init",
)

FLAGS = flags.FLAGS

def train(hps, result_dir, tuner=None, trial_name=None):
    """Construct model and run main training loop."""
    # Write hyperparameters to text summary
    hparams_dict = hps.values()
    # Create a markdown table from hparams.
    header = "| Key | Value |\n| :--- | :--- |\n"
    keys = sorted(hparams_dict.keys())
    lines = ["| %s | %s |" % (key, str(hparams_dict[key])) for key in keys]
    hparams_table = header + "\n".join(lines) + "\n"

    hparam_summary = tf.summary.text(
        "hparams", tf.constant(hparams_table, name="hparams"), collections=[]
    )

    with tf.Session() as sess:
        writer = tf.summary.FileWriter(result_dir, graph=sess.graph)
        writer.add_summary(hparam_summary.eval())
        writer.close()

    # We need to be able to run on the normal dataset for debugging.
    if FLAGS.n_labeled != -1:
        label_map = "label_map_count_{}_index_{}".format(
            FLAGS.n_labeled, FLAGS.label_map_index
        )
    else:
        label_map = None

    container_name = trial_name or ""

    # Create a separate container for each run so parameters don't stick around
    with tf.container(container_name):

        if label_map:
            label_table = dataset_utils.construct_label_table(
                FLAGS.primary_dataset_name, label_map
            )
        else:
            label_table = None

        labeled_data_filter_fn = make_labeled_data_filter_fn(label_table)
        unlabeled_data_filter_fn = make_unlabeled_data_filter_fn()

        images, labels, _, _, _, _ = data_provider.get_simple_mixed_batch(
            labeled_dataset_name=FLAGS.primary_dataset_name,
            unlabeled_dataset_name=FLAGS.secondary_dataset_name,
            split="train",
            batch_size=FLAGS.batch_size,
            shuffle_buffer_size=1000,
            labeled_data_filter_fn=labeled_data_filter_fn,
            unlabeled_data_filter_fn=unlabeled_data_filter_fn,
        )

        logging.info("Training data tensors constructed.")
        # This is necessary because presently svhn data comes as uint8
        images = tf.cast(images, tf.float32)
        ssl_framework = SSLFramework(
            networks.wide_resnet,
            hps,
            images,
            labels,
            make_train_tensors=True,
            consistency_model=FLAGS.consistency_model,
            zca_input_file_path=FLAGS.zca_input_file_path,
        )
        tf.summary.scalar("n_labeled", FLAGS.n_labeled)
        tf.summary.scalar("batch_size", FLAGS.batch_size)

        logging.info("Model instantiated.")
        init_op = tf.global_variables_initializer()

        saver = tf.train.Saver(keep_checkpoint_every_n_hours=1)

        if FLAGS.load_checkpoint:
            vars_to_load = [
                v for v in tf.all_variables() if "logit" not in v.name
            ]
            finetuning_saver = tf.train.Saver(
                keep_checkpoint_every_n_hours=1, var_list=vars_to_load
            )

        def init_fn(_, sess):
            sess.run(init_op)
            if FLAGS.load_checkpoint:
                logging.info(
                    "Fine tuning from checkpoint: %s", FLAGS.load_checkpoint
                )
                finetuning_saver.restore(sess, FLAGS.load_checkpoint)

        scaffold = tf.train.Scaffold(
            saver=saver, init_op=ssl_framework.global_step_init, init_fn=init_fn
        )
        logging.info("Scaffold created.")
        monitored_sess = tf.train.MonitoredTrainingSession(
            scaffold=scaffold,
            checkpoint_dir=result_dir,
            save_checkpoint_secs=300,
            save_summaries_secs=10,
            save_summaries_steps=None,
            config=tf.ConfigProto(
                allow_soft_placement=True, log_device_placement=False
            ),
            max_wait_secs=300,
        )
        logging.info("MonitoredTrainingSession initialized.")
        trainable_params = np.sum(
            [
                np.prod(v.get_shape().as_list())
                for v in tf.trainable_variables()
            ]
        )
        logging.info("Trainable parameters: %s", str(trainable_params))

        def should_stop_early():
            if tuner and tuner.should_trial_stop():
                logging.info(
                    "Got tuner.should_trial_stop(). Stopping trial early."
                )
                return True
            else:
                return False

        with monitored_sess as sess:

            while True:
                _, step, values_to_log = sess.run(
                    [
                        ssl_framework.train_op,
                        ssl_framework.global_step,
                        ssl_framework.scalars_to_log,
                    ],
                    feed_dict={ssl_framework.is_training: True},
                )

                if step % _PRINT_SPAN == 0:
                    logging.info(
                        "step %d: %r",
                        step,
                        dict((k, v) for k, v in values_to_log.items()),
                    )
                if step >= FLAGS.training_length:
                    break
                # Don't call should_stop_early() too frequently
                if step % _CHECK_TRIAL_EARLY_STOP == 0 and should_stop_early():
                    break

def make_labeled_data_filter_fn(label_table):
    """Make filter for certain classes of labeled data."""
    class_filter = tf_utils.filter_fn_from_comma_delimited(
        FLAGS.labeled_classes_filter
    )
    if label_table:
        return lambda _, label, fkey: class_filter(label) & label_table.lookup(
            fkey
        )
    else:
        return lambda _, label, fkey: class_filter(label)

def make_unlabeled_data_filter_fn():
    """Make filter for certain classes and a random fraction of unlabeled
    data."""
    class_filter = tf_utils.filter_fn_from_comma_delimited(
        FLAGS.unlabeled_classes_filter
    )

    def random_frac_filter(fkey):
        return tf_utils.hash_float(fkey) < FLAGS.unlabeled_data_random_fraction

    return lambda _, label, fkey: class_filter(label) & random_frac_filter(
        fkey
    )

def main(_):
    result_dir = os.path.join(FLAGS.root_dir, FLAGS.experiment_name)
    hps = hparams.get_hparams(
        FLAGS.primary_dataset_name, FLAGS.consistency_model
    )
    if FLAGS.hparam_string:
        hps.parse(FLAGS.hparam_string)
    train(hps, result_dir)

if __name__ == "__main__":
    app.run(main)
<commit_msg>Use -1 as default for n_labeled<commit_after>#!/usr/bin/python3
#
# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Train the model."""

from __future__ import division
import os
from absl import app
from absl import flags
import numpy as np
import tensorflow as tf
from absl import logging
from lib import data_provider
from lib import dataset_utils
from lib import tf_utils
from lib import hparams
from lib.ssl_framework import SSLFramework
from lib import networks

_PRINT_SPAN = 300
_CHECK_TRIAL_EARLY_STOP = 100

# Flags for model training
flags.DEFINE_string(
    "hparam_string", None, "String from which we parse hparams."
)
flags.DEFINE_string(
    "primary_dataset_name", "svhn", "Name of dataset containing primary data."
)
flags.DEFINE_string(
    "secondary_dataset_name",
    "svhn",
    "Name of dataset containing secondary data.",
)
flags.DEFINE_integer("label_map_index", 0, "Index of the label map.")
flags.DEFINE_integer(
    "n_labeled", -1, "Number of labeled examples, or -1 for entire dataset."
)
flags.DEFINE_integer(
    "training_length", 500000, "number of steps to train for."
)
flags.DEFINE_integer("batch_size", 100, "Size of the batch")
flags.DEFINE_string(
    "consistency_model", "mean_teacher", "Which consistency model to use."
)
flags.DEFINE_string(
    "zca_input_file_path",
    "",
    "Path to ZCA input statistics. '' means don't ZCA.",
)

flags.DEFINE_float(
    "unlabeled_data_random_fraction",
    1.0,
    "The fraction of unlabeled data to use during training.",
)
flags.DEFINE_string(
    "labeled_classes_filter",
    "",
    "Comma-delimited list of class numbers from labeled "
    "dataset to use during training. Defaults to all classes.",
)
flags.DEFINE_string(
    "unlabeled_classes_filter",
    "",
    "Comma-delimited list of class numbers from unlabeled "
    "dataset to use during training. Useful for labeled "
    "datasets being used as unlabeled data. Defaults to all "
    "classes.",
)

# Flags for book-keeping
flags.DEFINE_string(
    "root_dir", None, "The overall dir in which we store experiments"
)
flags.mark_flag_as_required("root_dir")

flags.DEFINE_string(
    "experiment_name", "default", "The name of this particular experiment"
)
flags.DEFINE_string(
    "load_checkpoint",
    "",
    "Checkpoint file to start training from (e.g. "
    ".../model.ckpt-354615), or None for random init",
)

FLAGS = flags.FLAGS

def train(hps, result_dir, tuner=None, trial_name=None):
    """Construct model and run main training loop."""
    # Write hyperparameters to text summary
    hparams_dict = hps.values()
    # Create a markdown table from hparams.
    header = "| Key | Value |\n| :--- | :--- |\n"
    keys = sorted(hparams_dict.keys())
    lines = ["| %s | %s |" % (key, str(hparams_dict[key])) for key in keys]
    hparams_table = header + "\n".join(lines) + "\n"

    hparam_summary = tf.summary.text(
        "hparams", tf.constant(hparams_table, name="hparams"), collections=[]
    )

    with tf.Session() as sess:
        writer = tf.summary.FileWriter(result_dir, graph=sess.graph)
        writer.add_summary(hparam_summary.eval())
        writer.close()

    # We need to be able to run on the normal dataset for debugging.
    if FLAGS.n_labeled != -1:
        label_map = "label_map_count_{}_index_{}".format(
            FLAGS.n_labeled, FLAGS.label_map_index
        )
    else:
        label_map = None

    container_name = trial_name or ""

    # Create a separate container for each run so parameters don't stick around
    with tf.container(container_name):

        if label_map:
            label_table = dataset_utils.construct_label_table(
                FLAGS.primary_dataset_name, label_map
            )
        else:
            label_table = None

        labeled_data_filter_fn = make_labeled_data_filter_fn(label_table)
        unlabeled_data_filter_fn = make_unlabeled_data_filter_fn()

        images, labels, _, _, _, _ = data_provider.get_simple_mixed_batch(
            labeled_dataset_name=FLAGS.primary_dataset_name,
            unlabeled_dataset_name=FLAGS.secondary_dataset_name,
            split="train",
            batch_size=FLAGS.batch_size,
            shuffle_buffer_size=1000,
            labeled_data_filter_fn=labeled_data_filter_fn,
            unlabeled_data_filter_fn=unlabeled_data_filter_fn,
        )

        logging.info("Training data tensors constructed.")
        # This is necessary because presently svhn data comes as uint8
        images = tf.cast(images, tf.float32)
        ssl_framework = SSLFramework(
            networks.wide_resnet,
            hps,
            images,
            labels,
            make_train_tensors=True,
            consistency_model=FLAGS.consistency_model,
            zca_input_file_path=FLAGS.zca_input_file_path,
        )
        tf.summary.scalar("n_labeled", FLAGS.n_labeled)
        tf.summary.scalar("batch_size", FLAGS.batch_size)

        logging.info("Model instantiated.")
        init_op = tf.global_variables_initializer()

        saver = tf.train.Saver(keep_checkpoint_every_n_hours=1)

        if FLAGS.load_checkpoint:
            vars_to_load = [
                v for v in tf.all_variables() if "logit" not in v.name
            ]
            finetuning_saver = tf.train.Saver(
                keep_checkpoint_every_n_hours=1, var_list=vars_to_load
            )

        def init_fn(_, sess):
            sess.run(init_op)
            if FLAGS.load_checkpoint:
                logging.info(
                    "Fine tuning from checkpoint: %s", FLAGS.load_checkpoint
                )
                finetuning_saver.restore(sess, FLAGS.load_checkpoint)

        scaffold = tf.train.Scaffold(
            saver=saver, init_op=ssl_framework.global_step_init, init_fn=init_fn
        )
        logging.info("Scaffold created.")
        monitored_sess = tf.train.MonitoredTrainingSession(
            scaffold=scaffold,
            checkpoint_dir=result_dir,
            save_checkpoint_secs=300,
            save_summaries_secs=10,
            save_summaries_steps=None,
            config=tf.ConfigProto(
                allow_soft_placement=True, log_device_placement=False
            ),
            max_wait_secs=300,
        )
        logging.info("MonitoredTrainingSession initialized.")
        trainable_params = np.sum(
            [
                np.prod(v.get_shape().as_list())
                for v in tf.trainable_variables()
            ]
        )
        logging.info("Trainable parameters: %s", str(trainable_params))

        def should_stop_early():
            if tuner and tuner.should_trial_stop():
                logging.info(
                    "Got tuner.should_trial_stop(). Stopping trial early."
                )
                return True
            else:
                return False

        with monitored_sess as sess:

            while True:
                _, step, values_to_log = sess.run(
                    [
                        ssl_framework.train_op,
                        ssl_framework.global_step,
                        ssl_framework.scalars_to_log,
                    ],
                    feed_dict={ssl_framework.is_training: True},
                )

                if step % _PRINT_SPAN == 0:
                    logging.info(
                        "step %d: %r",
                        step,
                        dict((k, v) for k, v in values_to_log.items()),
                    )
                if step >= FLAGS.training_length:
                    break
                # Don't call should_stop_early() too frequently
                if step % _CHECK_TRIAL_EARLY_STOP == 0 and should_stop_early():
                    break

def make_labeled_data_filter_fn(label_table):
    """Make filter for certain classes of labeled data."""
    class_filter = tf_utils.filter_fn_from_comma_delimited(
        FLAGS.labeled_classes_filter
    )
    if label_table:
        return lambda _, label, fkey: class_filter(label) & label_table.lookup(
            fkey
        )
    else:
        return lambda _, label, fkey: class_filter(label)

def make_unlabeled_data_filter_fn():
    """Make filter for certain classes and a random fraction of unlabeled
    data."""
    class_filter = tf_utils.filter_fn_from_comma_delimited(
        FLAGS.unlabeled_classes_filter
    )

    def random_frac_filter(fkey):
        return tf_utils.hash_float(fkey) < FLAGS.unlabeled_data_random_fraction

    return lambda _, label, fkey: class_filter(label) & random_frac_filter(
        fkey
    )

def main(_):
    result_dir = os.path.join(FLAGS.root_dir, FLAGS.experiment_name)
    hps = hparams.get_hparams(
        FLAGS.primary_dataset_name, FLAGS.consistency_model
    )
    if FLAGS.hparam_string:
        hps.parse(FLAGS.hparam_string)
    train(hps, result_dir)

if __name__ == "__main__":
    app.run(main)
<|endoftext|>


<commit_before># coding: utf-8
from __future__ import print_function
from __future__ import unicode_literals
import re
from collections import namedtuple

DEBUG = False

# these are operator types. This is annoying. I'd love an enum
FROM_KEY = "from"
TO_KEY = "to"

RawTransaction = namedtuple(
    "Transaction",
    ["first_party", "operator", "second_party", "value", "notes"])

class Transaction(object):

    """A transaction represents a debt relationship between two parties"""
    debtor = None
    creditor = None
    value = None
    currency = None
    notes = None

    raw_text = None
    raw_transaction = None

    @property
    def parsed(self):
        return True if self.raw_transaction else False

    def __init__(self, input_str):
        super(Transaction, self).__init__()
        self.raw_text = input_str
        self.raw_transaction = self.componentize_transaction(
            input_str,
            [self.componentize1, self.componentize2]
        )
        self.parse_components(self.raw_transaction)

    def __repr__(self):
        if self.parsed:
            return "%s %s %s %s %s" % (
                self.debtor, self.creditor, self.value,
                self.currency, self.notes)
        return "No Transaction"

    def obligation_to_user(self, user):
        if user == self.creditor:
            return self.value, self.debtor
        if user == self.debtor:
            return -self.value, self.creditor
        return 0, None

    def componentize_transaction(self, text, funcs):
        subbed = re.sub(r'<@(\w+)>', r'@\1 ', text)
        if DEBUG:
            print(text)
            print(subbed)
        for f in funcs:
            result = f(subbed)
            if result:
                return result

    def componentize1(self, text):
        parsed = re.search(
            r'@(\w+).?\s*(\S*)\s*@(\w+)\s+\$?([0-9\.]+)\$?(.*)', text)
        if parsed:
            return RawTransaction(
                parsed.group(1), parsed.group(2), parsed.group(3),
                parsed.group(4), parsed.group(5))

    def componentize2(self, text):
        parsed = re.search(r'@(\w+).?\s*(\S*)\s*(\w+)\s(.*?)([0-9\.]+)', text)
        if parsed:
            return RawTransaction(
                parsed.group(1), parsed.group(2), parsed.group(3),
                parsed.group(5), parsed.group(4))

    def parse_components(self, raw_transaction):
        if raw_transaction:
            if DEBUG:
                print(raw_transaction)
            operator = self.parse_operator(raw_transaction.operator)
            if operator == TO_KEY:
                self.debtor = raw_transaction.first_party
                self.creditor = raw_transaction.second_party
            elif operator == FROM_KEY:
                self.debtor = raw_transaction.second_party
                self.creditor = raw_transaction.first_party

            self.operator = raw_transaction.operator
            self.value = float(raw_transaction.value)
            if raw_transaction.notes != "":
                self.notes = raw_transaction.notes

    def parse_operator(self, operator_text):
        if operator_text == "->":
            return TO_KEY
        if operator_text == "-<":
            return FROM_KEY

def test():
    tests = [
        '<@U024H4SR1> -> <@U024H5LFB> $15 (thai food)',
        '<@U024H5KK7> -> <@U024H5LFB> 20$',
        '<@U024H5KK7> -> <@U024H5LFB> $10 sandwich place',
        '<@U024H5KK7> -< <@U024H5LFB> $10 sandwich place',
        '<@U024H4SR1|will> set the channel topic: trying to incur \
        more debt to make the debt-bot more exciting',
    ]
    for t in tests:
        print(Transaction(t).obligation_to_user('U024H5LFB'))

if __name__ == '__main__':
    test()
<commit_msg>better printing in transaction.py<commit_after># coding: utf-8
from __future__ import print_function
from __future__ import unicode_literals
import re
from collections import namedtuple

DEBUG = False

# these are operator types. This is annoying. I'd love an enum
FROM_KEY = "from"
TO_KEY = "to"

RawTransaction = namedtuple(
    "Transaction",
    ["first_party", "operator", "second_party", "value", "notes"])

class Transaction(object):

    """A transaction represents a debt relationship between two parties"""
    debtor = None
    creditor = None
    value = None
    currency = None
    notes = None

    raw_text = None
    raw_transaction = None

    @property
    def parsed(self):
        return all([self.debtor, self.creditor, self.value])

    def __init__(self, input_str):
        super(Transaction, self).__init__()
        self.raw_text = input_str
        self.raw_transaction = self.componentize_transaction(
            input_str,
            [self.componentize1, self.componentize2]
        )
        self.parse_components(self.raw_transaction)

    def __repr__(self):
        msg = "debtor: {} creditor: {} value: {} notes: {}".format(
            self.debtor or '',
            self.creditor or '',
            self.value or '',
            self.currency or '',
            self.notes or '')

        if not self.parsed:
            msg = 'FAILED TO PARSE: \n' + msg
        return msg

    def obligation_to_user(self, user):
        if user == self.creditor:
            return self.value, self.debtor
        if user == self.debtor:
            return -self.value, self.creditor
        return 0, None

    def componentize_transaction(self, text, funcs):
        subbed = re.sub(r'<@(\w+)>', r'@\1 ', text)
        if DEBUG:
            print(text)
            print(subbed)
        for f in funcs:
            result = f(subbed)
            if result:
                return result

    def componentize1(self, text):
        parsed = re.search(
            r'@(\w+).?\s*(\S*)\s*@(\w+)\s+\$?([0-9\.]+)\$?(.*)', text)
        if parsed:
            return RawTransaction(
                parsed.group(1), parsed.group(2), parsed.group(3),
                parsed.group(4), parsed.group(5))

    def componentize2(self, text):
        parsed = re.search(r'@(\w+).?\s*(\S*)\s*(\w+)\s(.*?)([0-9\.]+)', text)
        if parsed:
            return RawTransaction(
                parsed.group(1), parsed.group(2), parsed.group(3),
                parsed.group(5), parsed.group(4))

    def parse_components(self, raw_transaction):
        if raw_transaction:
            if DEBUG:
                print(raw_transaction)
            operator = self.parse_operator(raw_transaction.operator)
            if operator == TO_KEY:
                self.debtor = raw_transaction.first_party
                self.creditor = raw_transaction.second_party
            elif operator == FROM_KEY:
                self.debtor = raw_transaction.second_party
                self.creditor = raw_transaction.first_party

            self.operator = raw_transaction.operator
            self.value = float(raw_transaction.value)
            if raw_transaction.notes != "":
                self.notes = raw_transaction.notes

    def parse_operator(self, operator_text):
        if operator_text == "->":
            return TO_KEY
        if operator_text == "-<":
            return FROM_KEY

def test():
    tests = [
        '<@U024H4SR1> -> <@U024H5LFB> $15 (thai food)',
        '<@U024H5KK7> -> <@U024H5LFB> 20$',
        '<@U024H5KK7> -> <@U024H5LFB> $10 sandwich place',
        '<@U024H5KK7> -< <@U024H5LFB> $10 sandwich place',
        '<@U024H4SR1> -> <@U024H5LFB> $7',
        '<@U024H4SR1|will> set the channel topic: trying to incur \
        more debt to make the debt-bot more exciting',
    ]
    for t in tests:
        transaction = Transaction(t)
        print(transaction)
        print(transaction.obligation_to_user('U024H5LFB'))

if __name__ == '__main__':
    test()
<|endoftext|>


<commit_before>"""
This module tests the logic responsible for deciding what the display will do with the run it is set to show, namely
the display mode.
"""

from display_adapter import runmode_config
from display_adapter.display_driver.display_modes import DisplayMode, RunMode, ScreensaverMode

class TestDisplayMode(object):
    """
    This class tests the super-class mode, to test shared functionality of all the display modes.
    """

    def test_init(self):
        """
        Test the creation of the DisplayMode object.
        This object is not expected to act as a class, but to be
        inherited by other classes.
        """
        dm = DisplayMode("-*-\n-*-\n-*-")

        assert dm
        # Assert the display mode's game engine has access to the right pattern?
        assert DisplayMode._game_engine
        assert DisplayMode._game_engine.get_current_generation(output=True) == "-*-\n-*-\n-*-"

    def test_is_active(self):
        """
        This method tests the shared functionality of the is_active class.
        """
        dm = DisplayMode("-*-\n-*-\n-*-")

        assert dm.is_active()

    def test_is_active_fails(self):
        """
        This method tests the shared functionality of the is_active class.
        """
        dm = DisplayMode("---\n---\n---")

        assert dm.is_active() == False

    def test_get_display_pattern(self):
        """
        This method tests the ability of the display mode to correctly communicate with the game engine and return
        the correct next generation.
        """
        dm = DisplayMode("-*-\n-*-\n-*-")

        assert dm.get_display_pattern() == "-*-\n-*-\n-*-"
        assert dm.get_display_pattern() == "***\n***\n***"

class TestRunMode(object):
    """
    This class tests the display's Run mode in order to test the working logic of the display when showing a player's
    pattern.
    """

    def test_init(self):
        """
        Tests initialisation of the display's Run mode.
        """
        rm = RunMode("---\n---\n---")

        assert rm
        assert hasattr(rm, "_pattern")

    def test_is_active(self):
        """
        Ensuring that the run is still active, especially in it's presentation state.
        """
        rm = RunMode("-*-\n-*-\n-*-")

        for _ in range(0, runmode_config["iterations"]):
            for _ in range(0, runmode_config["full_frames"]):
                assert rm.is_active()
                rm.get_display_pattern()

            for _ in range(0, runmode_config["pattern_frames"]):
                assert rm.is_active()
                rm.get_display_pattern()

        assert rm.is_active()
        rm.get_display_pattern()
        assert rm.is_active()

    def test_is_active_fails(self):
        """
        Ensuring that the run is not active after the mode leaves it's presentation state.
        """
        rm = RunMode("---\n---\n---")

        for _ in range(0, runmode_config["iterations"]):
            for _ in range(0, runmode_config["full_frames"]):
                assert rm.is_active()
                rm.get_display_pattern()

            for _ in range(0, runmode_config["pattern_frames"]):
                assert rm.is_active()
                rm.get_display_pattern()

        assert not rm.is_active()

    def test_get_display_pattern(self):
        """
        Ensuring that the mode is, under all circumstances, returning the correct pattern to be displayed.
        The circumstances are as follows:
            - Two patterns during presentation state; all lights on and pattern lights on.
            - Once presentation state has finished, the player's pattern at the beginning of their cycle.
        """
        input = "-*-\n-*-\n-*-"
        rm = RunMode(input)

        for _ in range(0, runmode_config["iterations"]):
            for _ in range(0, runmode_config["full_frames"]):
                rm.get_display_pattern() == "***\n***\n***"

            for _ in range(0, runmode_config["pattern_frames"]):
                rm.get_display_pattern() == input

        assert rm.get_display_pattern() == input

class TestScreensaverMode(object):
    """
    This class tests the display's Screensaver mode in order to test the working logic of the display when not showing
    a player's pattern.
    """

    def test_init(self):
        """
        Tests initialisation of the display's Run mode.
        """
        sm = ScreensaverMode("---\n---\n---", "---\n---\n---")

        assert sm
        assert sm._previous_pattern == "---\n---\n---"

    def test_is_active(self):
        """
        Ensuring that the run is still active, especially in it's clear state.
        """
        input = "-*-\n-*-\n-*-"
        previous = "---\n---\n---"
        sm = ScreensaverMode(input, previous)

        # Testing the 3 frame pause
        for _ in range(0, 3):
            assert sm.is_active()
            sm.get_display_pattern()

        for i, _ in enumerate(previous.split("\n")):
            assert sm.is_active()
            sm.get_display_pattern()
            assert sm.is_active()

        sm.get_display_pattern()
        assert sm.is_active()

    def test_is_active_fails(self):
        """
        Ensuring that the run is not active after the mode leaves it's clear state.
        """
        input = "---\n---\n---"
        previous = "---\n---\n---"
        sm = ScreensaverMode(input, previous)

        # Testing the 3 frame pause
        for _ in range(0, 3):
            assert sm.is_active()
            sm.get_display_pattern()

        for i, _ in enumerate(previous.split("\n")):
            assert sm.is_active()
            sm.get_display_pattern()

        assert not sm.is_active()

    def test_get_display_pattern(self):
        """
        Ensuring that the mode is, under all circumstances, returning the correct pattern to be displayed.
        The circumstances are as follows:
            - A pause of the frame on the display when it entered screensaver mode.
            - The correct number of clear frames when running in clear state.
            - Once clear state has finished, the correct frames for a screensaver pattern.
        """
        input = "-*-\n-*-\n-*-"
        previous = "---\n---\n---"
        sm = ScreensaverMode(input, previous)

        # Testing the 3 frame pause
        for _ in range(0, 3):
            assert sm.get_display_pattern() == previous

        for i, _ in enumerate(previous.split("\n")):
            assert sm.get_display_pattern()

        assert sm.get_display_pattern() == input
<commit_msg>Code documentation added. Written by Michael.<commit_after>"""
This module contains the testing logic for the different display modes available to the display driver. The tests
should make sure the different modes correctly handle the data given to them.
"""

from display_adapter import runmode_config
from display_adapter.display_driver.display_modes import DisplayMode, RunMode, ScreensaverMode

class TestDisplayMode(object):
    """
    This class tests shared functionality of all display modes, making sure their backbone is correctly implemented.
    """

    def test_init(self):
        """
        Tests the initialisation of a DisplayMode object. This is to check shared object data is correctly set - the
        object itself is not meant to be used.
        """
        dm = DisplayMode("-*-\n-*-\n-*-")

        # Assert the display mode has been initialised.
        assert dm
        # Assert the display mode's game engine has been initialised correctly.
        assert DisplayMode._game_engine
        # Assert the display mode's game engine has access to the right pattern.
        assert DisplayMode._game_engine.get_current_generation(output=True) == "-*-\n-*-\n-*-"

    def test_is_active(self):
        """
        This method tests the functionality of the over-arching is_active method. The expected result of this test
        is for the display mode to be active.
        """
        dm = DisplayMode("-*-\n-*-\n-*-")

        # Assert that the display mode is active.
        assert dm.is_active()

    def test_is_active_fails(self):
        """
        This method tests the functionality of the over-arching is_active method. The expected result of this test
        is for the display mode to be inactive.
        """
        dm = DisplayMode("---\n---\n---")

        # Assert that the display mode is inactive.
        assert dm.is_active() == False

    def test_get_display_pattern(self):
        """
        This method tests the ability of the display mode to communicate with the game engine. The expected result of
        this test is for the get_display_pattern method to first of all return its given pattern, and then the correctly
        calculated next generation of the Game of life.
        """
        dm = DisplayMode("-*-\n-*-\n-*-")

        # Assert the display mode can return its given pattern.
        assert dm.get_display_pattern() == "-*-\n-*-\n-*-"
        # Assert the display mode can use the game engine to return the next calculated pattern.
        assert dm.get_display_pattern() == "***\n***\n***"

class TestRunMode(object):
    """
    This class tests the display's 'run mode' in order to test the working logic of the display when showing a player's
    pattern.
    """

    def test_init(self):
        """
        Tests initialisation of the display's Run mode. The expected result of this test is to make sure the correct
        data has been saved on initialisation.
        """
        rm = RunMode("---\n---\n---")

        # Assert the 'run mode' has been initialised.
        assert rm
        # Assert that the mode has been initialised with the right pattern.
        assert hasattr(rm, "_pattern")

    def test_is_active(self):
        """
        This method tests that 'run mode' is still active, especially in it's presentation state. The expected result
        of this test is for the check to show the mode is still active.
        """
        rm = RunMode("-*-\n-*-\n-*-")

        for _ in range(0, runmode_config["iterations"]):
            # For each iteration of the mode's presentation state.
            for _ in range(0, runmode_config["full_frames"]):
                # For each time presentation state shows a full light display.

                # Assert the mode is still active.
                assert rm.is_active()
                rm.get_display_pattern()

            for _ in range(0, runmode_config["pattern_frames"]):
                # For each time presentation state shows the user's pattern

                # Assert the mode is still active
                assert rm.is_active()
                rm.get_display_pattern()

        # Once presentation state, assert the mode is still active.
        assert rm.is_active()
        rm.get_display_pattern()
        # Assert the mode is still active once the user's pattern has started running.
        assert rm.is_active()

    def test_is_active_fails(self):
        """
        This method tests that 'run mode' activity does change. The expected result of this test is that the
        is_active method returns inactivity.
        """
        rm = RunMode("---\n---\n---")

        for _ in range(0, runmode_config["iterations"]):
            # For each iteration of the mode's presentation state
            for _ in range(0, runmode_config["full_frames"]):
                # For each time presentation state shows a full light display.

                # Assert the mode is active.
                assert rm.is_active()
                rm.get_display_pattern()

            for _ in range(0, runmode_config["pattern_frames"]):
                # For each time presentation state shows the user's pattern.

                # Assert the mode is still active.
                assert rm.is_active()
                rm.get_display_pattern()

        # Assert the mode is no longer active.
        assert not rm.is_active()

    def test_get_display_pattern(self):
        """
        This method tests that the mode is, under all circumstances, returning the correct pattern to be displayed.
        The circumstances are as follows:
            - Two patterns during presentation state; a full light display, and the cells of the user's pattern.
            - Once presentation state has finished, the player's pattern as it plays out on the display.
        """
        input = "-*-\n-*-\n-*-"
        rm = RunMode(input)

        for _ in range(0, runmode_config["iterations"]):
            # For each iteration of the mode's presentation state
            for _ in range(0, runmode_config["full_frames"]):
                # For each time the presentation state shows a full light display
                # Assert that the presentation state is showing a full light display
                assert rm.get_display_pattern() == "***\n***\n***"

            for _ in range(0, runmode_config["pattern_frames"]):
                # For each time the presentation state shows the user's pattern.
                # Assert that the presentation state is showing the user's pattern
                assert rm.get_display_pattern() == input

        # Assert that the mode is showing the user's pattern
        assert rm.get_display_pattern() == input

class TestScreensaverMode(object):
    """
    This class tests the display's 'sceensaver mode' in order to test the working logic of the display when not showing
    a player's pattern.
    """

    def test_init(self):
        """
        This method tests the initialisation of the display's screensaver mode. The expected result of this tests is
        to make sure the correct data has been saved on initialisation.
        """
        sm = ScreensaverMode("---\n---\n---", "---\n---\n---")

        # Assert the mode has been initialised.
        assert sm
        # Assert the mode's previous pattern has been correctly set.
        assert sm._previous_pattern == "---\n---\n---"

    def test_is_active(self):
        """
        This method tests that 'screensaver mode' is still active, especially in its clearing state. The expected
        result of this tests is for the check to show the mode is still active.
        """
        input = "-*-\n-*-\n-*-"
        previous = "---\n---\n---"
        sm = ScreensaverMode(input, previous)

        for _ in range(0, 3):
            # For each frame of the pause

            # Assert the mode is still active.
            assert sm.is_active()
            sm.get_display_pattern()

        for i, _ in enumerate(previous.split("\n")):
            # For each cell in the previous pattern

            # Assert the mode is still active.
            assert sm.is_active()
            # Calculate the next pattern to be displayed.
            sm.get_display_pattern()
            # Assert the mode is still active.
            assert sm.is_active()

        # Calculate the next pattern to be displayed.
        sm.get_display_pattern()
        # Assert the mode is still active.
        assert sm.is_active()

    def test_is_active_fails(self):
        """
        This method tests that 'screensaver mode' activity does change. The expected result of the test is that the
        is_active method returns inactivity.
        """
        input = "---\n---\n---"
        previous = "---\n---\n---"
        sm = ScreensaverMode(input, previous)

        for _ in range(0, 3):
            # For each frame of the pause

            # Assert the mode is still active.
            assert sm.is_active()
            sm.get_display_pattern()

        for i, _ in enumerate(previous.split("\n")):
            # For each cell in the previous pattern.

            # Assert the mode is still active.
            assert sm.is_active()
            # Calculate the next pattern to be displayed.
            sm.get_display_pattern()

        # Assert that the mode is no longer active.
        assert not sm.is_active()

    def test_get_display_pattern(self):
        """
        This method tests that the mode is, under all circumstances, returning the correct pattern to be displayed.
        The circumstances are as follows:
            - A pause on the frame the display was showing when it entered screensaver mode.
            - The correct number of clearing frames when running in clearing state.
            - Once clearing state has finished, the correct frames for a screensaver pattern.
        """
        input = "-*-\n-*-\n-*-"
        previous = "---\n---\n---"
        sm = ScreensaverMode(input, previous)

        for _ in range(0, 3):
            # For each frame of the pause.

            # Assert the mode is returning the correct pattern at this point.
            assert sm.get_display_pattern() == previous

        for i, _ in enumerate(previous.split("\n")):
            # For each cell of the previous pattern

            # Assert that the mode is still calculating a pattern at this point.
            assert sm.get_display_pattern()

        # Assert that the mode is returning the input pattern at this point.
        assert sm.get_display_pattern() == input
<|endoftext|>


<commit_before>#! /usr/bin/env python
#*******************************************************************************
# ALMA - Atacama Large Millimiter Array
# (c) National Research Council of Canada, 2007
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307  USA
#
# "@(#) $Id: TestLogLevelsComp.py,v 1.1 2007/12/14 16:53:32 eallaert Exp $"
#
# who       when      what
# --------  --------  ----------------------------------------------
# arne  2007-11-14  created
#

#--REGULAR IMPORTS-------------------------------------------------------------

#--CORBA STUBS-----------------------------------------------------------------
import contLogTest__POA
import contLogTest
#--ACS Imports-----------------------------------------------------------------
from Acspy.Servants.ContainerServices  import ContainerServices
from Acspy.Servants.ComponentLifecycle import ComponentLifecycle
from Acspy.Servants.ACSComponent       import ACSComponent
#--GLOBALS---------------------------------------------------------------------

#------------------------------------------------------------------------------
class TestLogLevelsComp(contLogTest__POA.TestLogLevelsComp,  #CORBA stubs for IDL interface
                ACSComponent,  #Base IDL interface
                ContainerServices,  #Developer niceties
                ComponentLifecycle):  #HLA stuff
    '''
    Simple component implementation provided as a reference for developers.
    '''
    def __init__(self):
        '''
        Just call superclass constructors here.
        '''
        ACSComponent.__init__(self)
        ContainerServices.__init__(self)
        return
    #------------------------------------------------------------------------------
    #--Override ComponentLifecycle methods-----------------------------------------
    #------------------------------------------------------------------------------
    def initialize(self):
        '''
        Override this method inherited from ComponentLifecycle
        '''
        self.getLogger().logTrace("called...")

    #------------------------------------------------------------------------------
    def cleanUp(self):
        '''
        Override this method inherited from ComponentLifecycle
        '''
        self.getLogger().logInfo("called...")
    #------------------------------------------------------------------------------
    #--Implementation of IDL methods-----------------------------------------------
    #------------------------------------------------------------------------------
    def getLevels(self):
        '''
        Python implementation of IDL method.
        LongSeq getLevels();
        '''
        self.getLogger().logInfo("called...")
        return [1, 2, 3, 4, 5]

    #------------------------------------------------------------------------------
    def logDummyMessages(self, levels):
        '''
        Python implementation of IDL method.
        void logDummyMessages(in LongSeq levels);
        '''
        self.getLogger().logInfo("called...")

#------------------------------------------------------------------------------
#--Main defined only for generic testing---------------------------------------
#------------------------------------------------------------------------------
if __name__ == "__main__":
    print "Creating an object"
    g = TestLogLevelsComp()
    print "Done..."

#
# ___oOo___
<commit_msg>Initial implementations of the TestLogLevelsComp interface in the Python test component.  Inspiration was taken from the C++ test component.<commit_after>#! /usr/bin/env python
#*******************************************************************************
# ALMA - Atacama Large Millimiter Array
# (c) National Research Council of Canada, 2007
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307  USA
#
# "@(#) $Id: TestLogLevelsComp.py,v 1.2 2007/12/20 22:50:33 agrimstrup Exp $"
#
# who       when      what
# --------  --------  ----------------------------------------------
# arne  2007-11-14  created
#

#--REGULAR IMPORTS-------------------------------------------------------------

#--CORBA STUBS-----------------------------------------------------------------
import contLogTest__POA
import contLogTest
#--ACS Imports-----------------------------------------------------------------
from Acspy.Servants.ContainerServices  import ContainerServices
from Acspy.Servants.ComponentLifecycle import ComponentLifecycle
from Acspy.Servants.ACSComponent       import ACSComponent
from Acspy.Common.Log                  import getLevelName
#--GLOBALS---------------------------------------------------------------------

#------------------------------------------------------------------------------
class TestLogLevelsComp(contLogTest__POA.TestLogLevelsComp,  #CORBA stubs for IDL interface
                ACSComponent,  #Base IDL interface
                ContainerServices,  #Developer niceties
                ComponentLifecycle):  #HLA stuff
    '''
    Simple component implementation provided as a reference for developers.
    '''
    def __init__(self):
        '''
        Just call superclass constructors here.
        '''
        ACSComponent.__init__(self)
        ContainerServices.__init__(self)
        return
    #------------------------------------------------------------------------------
    #--Override ComponentLifecycle methods-----------------------------------------
    #------------------------------------------------------------------------------
    def initialize(self):
        '''
        Override this method inherited from ComponentLifecycle
        '''
        self.getLogger().logTrace("TestLogLevelsComp.TestLogLevelsComp")

    #------------------------------------------------------------------------------
    def cleanUp(self):
        '''
        Override this method inherited from ComponentLifecycle
        '''
        self.getLogger().logInfo("Destroying " + self.name + "...")
    #------------------------------------------------------------------------------
    #--Implementation of IDL methods-----------------------------------------------
    #------------------------------------------------------------------------------
    def getLevels(self):
        '''
        Python implementation of IDL method.
        LongSeq getLevels();
        '''
        mylogger = self.getLogger()
        mylogger.logInfo("called...")
        levels = mylogger.getLevels()

        return [2, 2, min(levels.minLogLevel, levels.minLogLevelLocal), levels.minLogLevel, levels.minLogLevelLocal]

    #------------------------------------------------------------------------------
    def logDummyMessages(self, levels):
        '''
        Python implementation of IDL method.
        void logDummyMessages(in LongSeq levels);
        '''
        mylogger = self.getLogger()
        for l in levels[:-1]:
            mylogger.logAtLevel(l, "dummy log message for core level %d/%s" % (l, getLevelName(l)))
        mylogger.logAtLevel(levels[-1], "===last log message===")

#------------------------------------------------------------------------------
#--Main defined only for generic testing---------------------------------------
#------------------------------------------------------------------------------
if __name__ == "__main__":
    print "Creating an object"
    g = TestLogLevelsComp()
    print "Done..."

#
# ___oOo___
<|endoftext|>


<commit_before>"""
doctr

A tool to automatically deploy docs to GitHub pages from Travis CI.

The doctr command is two commands in one. To use, first run

doctr configure

on your local machine. This will prompt for your GitHub credentials and the
name of the repo you want to deploy docs for. This will generate a secure key,
which you should insert into your .travis.yml.

Then, on Travis, for the build where you build your docs, add

    - doctr deploy

to the end of the build to deploy the docs to GitHub pages.  This will only
run on the master branch, and won't run on pull requests.

For more information, see https://drdoctr.github.io/doctr/docs/
"""

import sys
import os
import argparse
import shlex
import subprocess

from textwrap import dedent

from .local import (generate_GitHub_token, encrypt_variable, encrypt_file,
    upload_GitHub_deploy_key, generate_ssh_key, check_repo_exists, GitHub_login)
from .travis import (setup_GitHub_push, commit_docs, push_docs,
    get_current_repo, sync_from_log, find_sphinx_build_dir, run)
from . import __version__

def get_parser():
    # This uses RawTextHelpFormatter so that the description (the docstring of
    # this module) is formatted correctly. Unfortunately, that means that
    # parser help is not text wrapped (but all other help is).
    parser = argparse.ArgumentParser(description=__doc__,
        formatter_class=argparse.RawTextHelpFormatter, epilog="""
Run --help on the subcommands like 'doctr deploy --help' to see the
options available.
        """,
        )
    parser.add_argument('-V', '--version', action='version', version='doctr ' + __version__)

    subcommand = parser.add_subparsers(title='subcommand', dest='subcommand')
    deploy_parser = subcommand.add_parser('deploy', help="""Deploy the docs to GitHub from Travis.""")
    deploy_parser.set_defaults(func=deploy)
    deploy_parser.add_argument('deploy_directory', type=str, nargs='?',
        help="""Directory to deploy the html documentation to on gh-pages.""")
    deploy_parser.add_argument('--force', action='store_true', help="""Run the deploy command even
    if we do not appear to be on Travis.""")
    deploy_parser.add_argument('--token', action='store_true', default=False,
        help="""Push to GitHub using a personal access token. Use this if you
        used 'doctr configure --token'.""")
    deploy_parser.add_argument('--key-path', default='github_deploy_key.enc',
        help="""Path of the encrypted GitHub deploy key. The default is %(default)r.""")
    deploy_parser.add_argument('--built-docs', default=None,
        help="""Location of the built html documentation to be deployed to
        gh-pages. If not specified, Doctr will try to automatically detect build location""")
    deploy_parser.add_argument('--tmp-dir', default=None,
        help=argparse.SUPPRESS)
    deploy_parser.add_argument('--deploy-repo', default=None, help="""Repo to
        deploy the docs to. By default, it deploys to the repo Doctr is run from.""")
    deploy_parser.add_argument('--no-require-master', dest='require_master', action='store_false',
        default=True, help="""Allow docs to be pushed from a branch other than master""")
    deploy_parser.add_argument('--command', default=None, help="""Command to
        be run before committing and pushing. If the command creates
        additional files that should be deployed, they should be added to the
        index.""")
    deploy_parser.add_argument('--no-sync', dest='sync', action='store_false',
        default=True, help="""Don't sync any files. This is generally used in
        conjunction with the --command flag, for instance, if the command syncs
        the files for you. Any files you wish to commit should be added to the
        index.""")
    deploy_parser.add_argument('--no-push', dest='push', action='store_false',
        default=True, help="Run all the steps except the last push step."
        "Useful for debugging")
    deploy_parser.add_argument('--gh-pages-docs', default='docs',
        help="""!!DEPRECATED!! Directory to deploy the html documentation to on gh-pages.
        The default is %(default)r. The deploy directory should be passed as
        the first argument to 'doctr deploy'. This flag is kept for backwards
        compatibility.""")

    configure_parser = subcommand.add_parser('configure', help="Configure doctr. This command should be run locally (not on Travis).")
    configure_parser.set_defaults(func=configure)
    configure_parser.add_argument('--force', action='store_true', help="""Run the configure command even
    if we appear to be on Travis.""")
    configure_parser.add_argument('--token', action="store_true", default=False,
        help="""Generate a personal access token to push to GitHub. The default is to use a
        deploy key. WARNING: This will grant read/write access to all the
        public repositories for the user. This option is not recommended
        unless you are using a separate GitHub user for deploying.""")
    configure_parser.add_argument("--no-upload-key", action="store_false", default=True,
        dest="upload_key", help="""Don't automatically upload the deploy key to GitHub. If you select this
        option, you will not be prompted for your GitHub credentials, so this option is not compatible with
        private repositories.""")
    configure_parser.add_argument('--key-path', default='github_deploy_key',
        help="""Path to save the encrypted GitHub deploy key. The default is %(default)r.
    The .enc extension is added to the file automatically.""")

    return parser

def process_args(parser):
    args = parser.parse_args()

    if not args.subcommand:
        parser.print_usage()
        parser.exit(1)

    try:
        return args.func(args, parser)
    except RuntimeError as e:
        sys.exit("Error: " + e.args[0])

def on_travis():
    return os.environ.get("TRAVIS_JOB_NUMBER", '')

def deploy(args, parser):
    if not args.force and not on_travis():
        parser.error("doctr does not appear to be running on Travis. Use "
            "doctr deploy --force to run anyway.")

    if args.tmp_dir:
        parser.error("The --tmp-dir flag has been removed (doctr no longer uses a temporary directory when deploying).")

    build_repo = get_current_repo()
    deploy_repo = args.deploy_repo or build_repo

    current_commit = subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode('utf-8').strip()
    try:
        can_push = setup_GitHub_push(deploy_repo, auth_type='token' if args.token else
                             'deploy_key', full_key_path=args.key_path,
                             require_master=args.require_master)

        if args.sync:
            built_docs = args.built_docs or find_sphinx_build_dir()

            log_file = os.path.join(args.gh_pages_docs, '.doctr-files')

            print("Moving built docs into place")
            added, removed = sync_from_log(src=built_docs,
                dst=args.gh_pages_docs, log_file=log_file)

        else:
            added, removed = [], []

        if args.command:
            run(shlex.split(args.command))

        changes = commit_docs(added=added, removed=removed)
        if changes:
            if can_push and args.push:
                push_docs()
            else:
                print("Don't have permission to push. Not trying.")
        else:
            print("The docs have not changed. Not updating")
    finally:
        subprocess.run(['git', 'checkout', current_commit])

class IncrementingInt:
    def __init__(self, i=0):
        self.i = i

    def __repr__(self):
        ret = repr(self.i)
        self.i += 1
        return ret

    __str__ = __repr__

def configure(args, parser):
    if not args.force and on_travis():
        parser.error("doctr appears to be running on Travis. Use "
            "doctr configure --force to run anyway.")

    if args.upload_key:
        login_kwargs = GitHub_login()
    else:
        login_kwargs = {'auth': None, 'headers': None}

    build_repo = input("What repo do you want to build the docs for (org/reponame, like 'drdoctr/doctr')? ")
    is_private = check_repo_exists(build_repo, service='github', **login_kwargs)
    check_repo_exists(build_repo, service='travis')

    deploy_repo = input("What repo do you want to deploy the docs to? [{build_repo}] ".format(build_repo=build_repo))
    if not deploy_repo:
        deploy_repo = build_repo

    if deploy_repo != build_repo:
        check_repo_exists(deploy_repo, service='github', **login_kwargs)

    N = IncrementingInt(1)

    header = "\n================== You should now do the following ==================\n"

    if args.token:
        token = generate_GitHub_token(**login_kwargs)['token']
        encrypted_variable = encrypt_variable("GH_TOKEN={token}".format(token=token).encode('utf-8'),
            build_repo=build_repo, is_private=is_private, **login_kwargs)
        print(dedent("""
        A personal access token for doctr has been created.

        You can go to https://github.com/settings/tokens to revoke it."""))

        print(header)
    else:
        ssh_key = generate_ssh_key("doctr deploy key for {deploy_repo}".format(deploy_repo=deploy_repo), keypath=args.key_path)
        key = encrypt_file(args.key_path, delete=True)
        encrypted_variable = encrypt_variable(b"DOCTR_DEPLOY_ENCRYPTION_KEY=" + key, build_repo=build_repo, is_private=is_private, **login_kwargs)

        deploy_keys_url = 'https://github.com/{deploy_repo}/settings/keys'.format(deploy_repo=deploy_repo)

        if args.upload_key:

            upload_GitHub_deploy_key(deploy_repo, ssh_key, **login_kwargs)

            print(dedent("""
            The deploy key has been added for {deploy_repo}.

            You can go to {deploy_keys_url} to revoke the deploy key.\
            """.format(deploy_repo=deploy_repo, deploy_keys_url=deploy_keys_url, keypath=args.key_path)))
            print(header)
        else:
            print(header)
            print(dedent("""\
            {N}. Go to {deploy_keys_url}
            and add the following as a new key:

            {ssh_key}
            Be sure to allow write access for the key.
            """.format(ssh_key=ssh_key, deploy_keys_url=deploy_keys_url, N=N)))

        print(dedent("""\
        {N}. Commit the file {keypath}.enc.
        """.format(keypath=args.key_path, N=N)))

    options = ''
    if args.key_path != 'github_deploy_key':
        options += ' --key-path {keypath}.enc'.format(keypath=args.key_path)
    if deploy_repo != build_repo:
        options += ' --deploy-repo {deploy_repo}'.format(deploy_repo=deploy_repo)

    print(dedent("""\
    {N}. Add

        script:
          - set -e
          - # Command to build your docs
          - pip install doctr
          - doctr deploy{options} {deploy_directory}

    to the docs build of your .travis.yml.  The 'set -e' prevents doctr from
    running when the docs build fails. Use the 'script' section so that if
    doctr fails it causes the build to fail.
    """.format(options=options, N=N)))

    print(dedent("""\
    {N}. Put

        env:
          global:
            - secure: "{encrypted_variable}"

    in your .travis.yml.
    """.format(encrypted_variable=encrypted_variable.decode('utf-8'), N=N)))

def main():
    return process_args(get_parser())

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>check that only one deploy directory is specified<commit_after>"""
doctr

A tool to automatically deploy docs to GitHub pages from Travis CI.

The doctr command is two commands in one. To use, first run

doctr configure

on your local machine. This will prompt for your GitHub credentials and the
name of the repo you want to deploy docs for. This will generate a secure key,
which you should insert into your .travis.yml.

Then, on Travis, for the build where you build your docs, add

    - doctr deploy

to the end of the build to deploy the docs to GitHub pages.  This will only
run on the master branch, and won't run on pull requests.

For more information, see https://drdoctr.github.io/doctr/docs/
"""

import sys
import os
import argparse
import shlex
import subprocess

from textwrap import dedent

from .local import (generate_GitHub_token, encrypt_variable, encrypt_file,
    upload_GitHub_deploy_key, generate_ssh_key, check_repo_exists, GitHub_login)
from .travis import (setup_GitHub_push, commit_docs, push_docs,
    get_current_repo, sync_from_log, find_sphinx_build_dir, run)
from . import __version__

def get_parser():
    # This uses RawTextHelpFormatter so that the description (the docstring of
    # this module) is formatted correctly. Unfortunately, that means that
    # parser help is not text wrapped (but all other help is).
    parser = argparse.ArgumentParser(description=__doc__,
        formatter_class=argparse.RawTextHelpFormatter, epilog="""
Run --help on the subcommands like 'doctr deploy --help' to see the
options available.
        """,
        )
    parser.add_argument('-V', '--version', action='version', version='doctr ' + __version__)

    subcommand = parser.add_subparsers(title='subcommand', dest='subcommand')
    deploy_parser = subcommand.add_parser('deploy', help="""Deploy the docs to GitHub from Travis.""")
    deploy_parser.set_defaults(func=deploy)
    deploy_parser.add_argument('deploy_directory', type=str, nargs='?',
        help="""Directory to deploy the html documentation to on gh-pages.""")
    deploy_parser.add_argument('--force', action='store_true', help="""Run the deploy command even
    if we do not appear to be on Travis.""")
    deploy_parser.add_argument('--token', action='store_true', default=False,
        help="""Push to GitHub using a personal access token. Use this if you
        used 'doctr configure --token'.""")
    deploy_parser.add_argument('--key-path', default='github_deploy_key.enc',
        help="""Path of the encrypted GitHub deploy key. The default is %(default)r.""")
    deploy_parser.add_argument('--built-docs', default=None,
        help="""Location of the built html documentation to be deployed to
        gh-pages. If not specified, Doctr will try to automatically detect build location""")
    deploy_parser.add_argument('--tmp-dir', default=None,
        help=argparse.SUPPRESS)
    deploy_parser.add_argument('--deploy-repo', default=None, help="""Repo to
        deploy the docs to. By default, it deploys to the repo Doctr is run from.""")
    deploy_parser.add_argument('--no-require-master', dest='require_master', action='store_false',
        default=True, help="""Allow docs to be pushed from a branch other than master""")
    deploy_parser.add_argument('--command', default=None, help="""Command to
        be run before committing and pushing. If the command creates
        additional files that should be deployed, they should be added to the
        index.""")
    deploy_parser.add_argument('--no-sync', dest='sync', action='store_false',
        default=True, help="""Don't sync any files. This is generally used in
        conjunction with the --command flag, for instance, if the command syncs
        the files for you. Any files you wish to commit should be added to the
        index.""")
    deploy_parser.add_argument('--no-push', dest='push', action='store_false',
        default=True, help="Run all the steps except the last push step."
        "Useful for debugging")
    deploy_parser.add_argument('--gh-pages-docs', default=None,
        help="""!!DEPRECATED!! Directory to deploy the html documentation to on gh-pages.
        The default is %(default)r. The deploy directory should be passed as
        the first argument to 'doctr deploy'. This flag is kept for backwards
        compatibility.""")

    configure_parser = subcommand.add_parser('configure', help="Configure doctr. This command should be run locally (not on Travis).")
    configure_parser.set_defaults(func=configure)
    configure_parser.add_argument('--force', action='store_true', help="""Run the configure command even
    if we appear to be on Travis.""")
    configure_parser.add_argument('--token', action="store_true", default=False,
        help="""Generate a personal access token to push to GitHub. The default is to use a
        deploy key. WARNING: This will grant read/write access to all the
        public repositories for the user. This option is not recommended
        unless you are using a separate GitHub user for deploying.""")
    configure_parser.add_argument("--no-upload-key", action="store_false", default=True,
        dest="upload_key", help="""Don't automatically upload the deploy key to GitHub. If you select this
        option, you will not be prompted for your GitHub credentials, so this option is not compatible with
        private repositories.""")
    configure_parser.add_argument('--key-path', default='github_deploy_key',
        help="""Path to save the encrypted GitHub deploy key. The default is %(default)r.
    The .enc extension is added to the file automatically.""")

    return parser

def process_args(parser):
    args = parser.parse_args()

    if not args.subcommand:
        parser.print_usage()
        parser.exit(1)

    try:
        return args.func(args, parser)
    except RuntimeError as e:
        sys.exit("Error: " + e.args[0])

def on_travis():
    return os.environ.get("TRAVIS_JOB_NUMBER", '')

def deploy(args, parser):
    if not args.force and not on_travis():
        parser.error("doctr does not appear to be running on Travis. Use "
            "doctr deploy --force to run anyway.")

    if args.tmp_dir:
        parser.error("The --tmp-dir flag has been removed (doctr no longer uses a temporary directory when deploying).")

    if args.gh_pages_docs:
        print("The --gh-pages-docs flag is deprecated and will be removed in the next release. Instead pass the deploy directory as an argument, e.g. `doctr deploy .`")

    if args.gh_pages_docs and args.deploy_directory:
        parser.error("The --gh-pages-docs flag is deprecated. Specify the directory to deploy to using `doctr deploy <dir>`")

    if not args.gh_pages_docs and not args.deploy_directory:
        parser.error("No deploy directory specified. Specify the directory to deploy to using `doctr deploy <dir>`")

    deploy_dir = args.gh_pages_docs or args.deploy_directory

    build_repo = get_current_repo()
    deploy_repo = args.deploy_repo or build_repo

    current_commit = subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode('utf-8').strip()
    try:
        can_push = setup_GitHub_push(deploy_repo, auth_type='token' if args.token else
                             'deploy_key', full_key_path=args.key_path,
                             require_master=args.require_master)

        if args.sync:
            built_docs = args.built_docs or find_sphinx_build_dir()

            log_file = os.path.join(deploy_dir, '.doctr-files')

            print("Moving built docs into place")
            added, removed = sync_from_log(src=built_docs,
                dst=deploy_dir, log_file=log_file)

        else:
            added, removed = [], []

        if args.command:
            run(shlex.split(args.command))

        changes = commit_docs(added=added, removed=removed)
        if changes:
            if can_push and args.push:
                push_docs()
            else:
                print("Don't have permission to push. Not trying.")
        else:
            print("The docs have not changed. Not updating")
    finally:
        subprocess.run(['git', 'checkout', current_commit])

class IncrementingInt:
    def __init__(self, i=0):
        self.i = i

    def __repr__(self):
        ret = repr(self.i)
        self.i += 1
        return ret

    __str__ = __repr__

def configure(args, parser):
    if not args.force and on_travis():
        parser.error("doctr appears to be running on Travis. Use "
            "doctr configure --force to run anyway.")

    if args.upload_key:
        login_kwargs = GitHub_login()
    else:
        login_kwargs = {'auth': None, 'headers': None}

    build_repo = input("What repo do you want to build the docs for (org/reponame, like 'drdoctr/doctr')? ")
    is_private = check_repo_exists(build_repo, service='github', **login_kwargs)
    check_repo_exists(build_repo, service='travis')

    deploy_repo = input("What repo do you want to deploy the docs to? [{build_repo}] ".format(build_repo=build_repo))
    if not deploy_repo:
        deploy_repo = build_repo

    if deploy_repo != build_repo:
        check_repo_exists(deploy_repo, service='github', **login_kwargs)

    N = IncrementingInt(1)

    header = "\n================== You should now do the following ==================\n"

    if args.token:
        token = generate_GitHub_token(**login_kwargs)['token']
        encrypted_variable = encrypt_variable("GH_TOKEN={token}".format(token=token).encode('utf-8'),
            build_repo=build_repo, is_private=is_private, **login_kwargs)
        print(dedent("""
        A personal access token for doctr has been created.

        You can go to https://github.com/settings/tokens to revoke it."""))

        print(header)
    else:
        ssh_key = generate_ssh_key("doctr deploy key for {deploy_repo}".format(deploy_repo=deploy_repo), keypath=args.key_path)
        key = encrypt_file(args.key_path, delete=True)
        encrypted_variable = encrypt_variable(b"DOCTR_DEPLOY_ENCRYPTION_KEY=" + key, build_repo=build_repo, is_private=is_private, **login_kwargs)

        deploy_keys_url = 'https://github.com/{deploy_repo}/settings/keys'.format(deploy_repo=deploy_repo)

        if args.upload_key:

            upload_GitHub_deploy_key(deploy_repo, ssh_key, **login_kwargs)

            print(dedent("""
            The deploy key has been added for {deploy_repo}.

            You can go to {deploy_keys_url} to revoke the deploy key.\
            """.format(deploy_repo=deploy_repo, deploy_keys_url=deploy_keys_url, keypath=args.key_path)))
            print(header)
        else:
            print(header)
            print(dedent("""\
            {N}. Go to {deploy_keys_url}
            and add the following as a new key:

            {ssh_key}
            Be sure to allow write access for the key.
            """.format(ssh_key=ssh_key, deploy_keys_url=deploy_keys_url, N=N)))

        print(dedent("""\
        {N}. Commit the file {keypath}.enc.
        """.format(keypath=args.key_path, N=N)))

    options = ''
    if args.key_path != 'github_deploy_key':
        options += ' --key-path {keypath}.enc'.format(keypath=args.key_path)
    if deploy_repo != build_repo:
        options += ' --deploy-repo {deploy_repo}'.format(deploy_repo=deploy_repo)

    print(dedent("""\
    {N}. Add

        script:
          - set -e
          - # Command to build your docs
          - pip install doctr
          - doctr deploy{options} {deploy_directory}

    to the docs build of your .travis.yml.  The 'set -e' prevents doctr from
    running when the docs build fails. Use the 'script' section so that if
    doctr fails it causes the build to fail.
    """.format(options=options, N=N)))

    print(dedent("""\
    {N}. Put

        env:
          global:
            - secure: "{encrypted_variable}"

    in your .travis.yml.
    """.format(encrypted_variable=encrypted_variable.decode('utf-8'), N=N)))

def main():
    return process_args(get_parser())

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>