<commit_before>import os
import uuid
import leveldb
import ujson as json

from shutil import rmtree
from threading import Thread, Event

from .env import Environment
from .constants import FAILURE_STATUS, SUCCESS_STATUS,\
                       WARNING_STATUS, OS_ERROR, DATABASE_ERROR
from .utils.snippets import from_bytes_to_mo
from .utils.patterns import enum

class Ocd(Thread):
    """Sometimes, you just want your program to have some
    obsessive compulsive disorder

    Source : http://pastebin.com/xNV7hx8h"""
    def __init__(self, interval, function, iterations=0, args=[], kwargs={}):
        Thread.__init__(self)
        self.interval = interval
        self.function = function
        self.iterations = iterations
        self.args = args
        self.kwargs = kwargs
        self.finished = Event()

    def run(self):
        count = 0
        while not self.finished.is_set() and (self.iterations <= 0 or count < self.iterations):
            self.finished.wait(self.interval)
            if not self.finished.is_set():
                self.function(*self.args, **self.kwargs)
                count += 1

    def cancel(self):
        self.finished.set()

class DatabaseOptions(dict):
    def __init__(self, *args, **kwargs):
        self['create_if_missing'] = True
        self['error_if_exists'] = False
        self['paranoid_checks'] = False
        self['block_cache_size'] = 8 * (2 << 20)
        self['write_buffer_size'] = 2 * (2 << 20)
        self['block_size'] = 4096
        self['max_open_files'] = 1000

        for key, value in kwargs.iteritems():
            if key in self:
                self[key] = value

class DatabasesHandler(dict):
    STATUSES = enum('MOUNTED', 'UNMOUNTED')

    def __init__(self, store, dest, *args, **kwargs):
        self.env = Environment()
        self.index = dict().fromkeys('name_to_uid')

        self.index['name_to_uid'] = {}
        self['reverse_name_index'] = {}
        self['paths_index'] = {}
        self.dest = dest
        self.store = store

        self._global_cache_size = None

        self.load()

    @property
    def global_cache_size(self):
        store_datas = self.extract_store_datas()
        max_caches = [int(db["options"]["block_cache_size"]) for db
                      in store_datas.itervalues()]

        return sum([from_bytes_to_mo(x) for x in max_caches])

    def _disposable_cache(self, new_cache_size):
        next_cache_size = self.global_cache_size + from_bytes_to_mo(new_cache_size)
        ratio = int(self.env["global"]["max_cache_size"]) - next_cache_size

        # Both values are in
        if ratio < 0:
            return (False, ratio)
        return (True, ratio)

    def extract_store_datas(self):
        """Retrieves database store from file

        If file doesn't exist, or is invalid json,
        and empty store is returned.

        Return
        ------
        store_datas, dict
        """
        try:
            store_datas = json.load(open(self.store, 'r'))
        except (IOError, ValueError):
            store_datas = {}

        return store_datas

    def load(self):
        """Loads databases from store file"""
        store_datas = self.extract_store_datas()

        for db_name, db_desc in store_datas.iteritems():
            self.index['name_to_uid'].update({db_name: db_desc['uid']})
            self.update({
                db_desc['uid']: {
                    'connector': leveldb.LevelDB(db_desc['path']),
                    'name': db_name,
                    'path': db_desc['path'],
                    'status': self.STATUSES.UNMOUNTED,
                    'ref_count': 0,
                }
            })

        # Always bootstrap 'default'
        if 'default' not in self.index['name_to_uid']:
            self.add('default')

    def store_update(self, db_name, db_desc):
        """Updates the database store file db_name
        key, with db_desc value"""
        store_datas = self.extract_store_datas()

        store_datas.update({db_name: db_desc})
        json.dump(store_datas, open(self.store, 'w'))

    def store_remove(self, db_name):
        """Removes a database from store file"""
        store_datas = self.extract_store_datas()
        store_datas.pop(db_name)
        json.dump(store_datas, open(self.store, 'w'))

    def mount(self, db_name):
        db_uid = self.index['name_to_uid'][db_name] if db_name in self.index['name_to_uid'] else None

        if self[db_uid]['status'] == self.STATUSES.UNMOUNTED:
            db_path = self[db_uid]['path']

            self[db_uid]['status'] = self.STATUSES.MOUNTED
            self[db_uid]['connector'] = leveldb.LevelDB(db_path)
        else:
            return (FAILURE_STATUS,
                    [DATABASE_ERROR, "Database %r already mounted" % db_name])

        return SUCCESS_STATUS, None

    def umount(self, db_name):
        db_uid = self.index['name_to_uid'][db_name] if db_name in self.index['name_to_uid'] else None

        if self[db_uid]['status'] == self.STATUSES.MOUNTED:
            self[db_uid]['status'] = self.STATUSES.UNMOUNTED
            del self[db_uid]['connector']
            self[db_uid]['connector'] = None
        else:
            return (FAILURE_STATUS,
                    [DATABASE_ERROR, "Database %r already unmounted" % db_name])

        return SUCCESS_STATUS, None

    def add(self, db_name, db_options=None):
        """Adds a db to the DatabasesHandler object, and sync it
        to the store file"""
        db_options = db_options or DatabaseOptions()
        cache_status, ratio = self._disposable_cache(db_options["block_cache_size"])
        if not cache_status:
            return (FAILURE_STATUS,
                    [DATABASE_ERROR,
                     "Not enough disposable cache memory "
                     "%d Mo missing" % ratio])

        db_name_is_path = db_name.startswith('.') or ('/' in db_name)
        is_abspath = lambda: not db_name.startswith('.') and ('/' in db_name)

        # Handle case when a db is a path
        if db_name_is_path:
            if not is_abspath():
                return (FAILURE_STATUS,
                        [DATABASE_ERROR, "Canno't create database from relative path"])
            try:
                new_db_path = db_name
                if not os.path.exists(new_db_path):
                    os.mkdir(new_db_path)
            except OSError as e:
                return (FAILURE_STATUS,
                        [OS_ERROR, e.strerror])
        else:
            new_db_path = os.path.join(self.dest, db_name)

        path = new_db_path
        uid = str(uuid.uuid4())
        options = db_options
        self.store_update(db_name, {
            'path': path,
            'uid': uid,
            'options': options,
        })

        self.index['name_to_uid'].update({db_name: uid})
        self.update({
            uid: {
                'connector': leveldb.LevelDB(path, **options),
                'name': db_name,
                'path': path,
                'status': self.STATUSES.MOUNTED,
                'ref_count': 0,
            },
        })

        return SUCCESS_STATUS, None

    def drop(self, db_name):
        """Drops a db from the DatabasesHandler, and sync it
        to store file"""
        db_uid = self.index['name_to_uid'].pop(db_name)
        db_path = self[db_uid]['path']

        self.pop(db_uid)
        self.store_remove(db_name)

        try:
            rmtree(db_path)
        except OSError:
            return (FAILURE_STATUS,
                    [DATABASE_ERROR,
                    "Cannot drop db : %s, files not found"])

        return SUCCESS_STATUS, None

    def exists(self, db_name):
        """Checks if a database exists on disk"""
        db_uid = self.index['name_to_uid'][db_name] if db_name in self.index['name_to_uid'] else None

        if db_uid:
            if os.path.exists(self[db_uid]['path']):
                return True
            else:
                self.drop(db_name)

        return False

    def list(self):
        """Lists all the DatabasesHandler known databases"""
        return [db_name for db_name
                in [key for key
                    in self.index['name_to_uid'].iterkeys()]
                if self.exists(db_name)]
<commit_msg>Update #91: Mount default at server startup<commit_after>import os
import uuid
import leveldb
import ujson as json

from shutil import rmtree
from threading import Thread, Event

from .env import Environment
from .constants import FAILURE_STATUS, SUCCESS_STATUS,\
                       WARNING_STATUS, OS_ERROR, DATABASE_ERROR
from .utils.snippets import from_bytes_to_mo
from .utils.patterns import enum

class Ocd(Thread):
    """Sometimes, you just want your program to have some
    obsessive compulsive disorder

    Source : http://pastebin.com/xNV7hx8h"""
    def __init__(self, interval, function, iterations=0, args=[], kwargs={}):
        Thread.__init__(self)
        self.interval = interval
        self.function = function
        self.iterations = iterations
        self.args = args
        self.kwargs = kwargs
        self.finished = Event()

    def run(self):
        count = 0
        while not self.finished.is_set() and (self.iterations <= 0 or count < self.iterations):
            self.finished.wait(self.interval)
            if not self.finished.is_set():
                self.function(*self.args, **self.kwargs)
                count += 1

    def cancel(self):
        self.finished.set()

class DatabaseOptions(dict):
    def __init__(self, *args, **kwargs):
        self['create_if_missing'] = True
        self['error_if_exists'] = False
        self['paranoid_checks'] = False
        self['block_cache_size'] = 8 * (2 << 20)
        self['write_buffer_size'] = 2 * (2 << 20)
        self['block_size'] = 4096
        self['max_open_files'] = 1000

        for key, value in kwargs.iteritems():
            if key in self:
                self[key] = value

class DatabasesHandler(dict):
    STATUSES = enum('MOUNTED', 'UNMOUNTED')

    def __init__(self, store, dest, *args, **kwargs):
        self.env = Environment()
        self.index = dict().fromkeys('name_to_uid')

        self.index['name_to_uid'] = {}
        self['reverse_name_index'] = {}
        self['paths_index'] = {}
        self.dest = dest
        self.store = store

        self._global_cache_size = None

        self.load()
        self.mount('default')  # Always mount default

    @property
    def global_cache_size(self):
        store_datas = self.extract_store_datas()
        max_caches = [int(db["options"]["block_cache_size"]) for db
                      in store_datas.itervalues()]

        return sum([from_bytes_to_mo(x) for x in max_caches])

    def _disposable_cache(self, new_cache_size):
        next_cache_size = self.global_cache_size + from_bytes_to_mo(new_cache_size)
        ratio = int(self.env["global"]["max_cache_size"]) - next_cache_size

        # Both values are in
        if ratio < 0:
            return (False, ratio)
        return (True, ratio)

    def extract_store_datas(self):
        """Retrieves database store from file

        If file doesn't exist, or is invalid json,
        and empty store is returned.

        Return
        ------
        store_datas, dict
        """
        try:
            store_datas = json.load(open(self.store, 'r'))
        except (IOError, ValueError):
            store_datas = {}

        return store_datas

    def load(self):
        """Loads databases from store file"""
        store_datas = self.extract_store_datas()

        for db_name, db_desc in store_datas.iteritems():
            self.index['name_to_uid'].update({db_name: db_desc['uid']})
            self.update({
                db_desc['uid']: {
                    'connector': None,
                    'name': db_name,
                    'path': db_desc['path'],
                    'status': self.STATUSES.UNMOUNTED,
                    'ref_count': 0,
                }
            })

        # Always bootstrap 'default'
        if 'default' not in self.index['name_to_uid']:
            self.add('default')

    def store_update(self, db_name, db_desc):
        """Updates the database store file db_name
        key, with db_desc value"""
        store_datas = self.extract_store_datas()

        store_datas.update({db_name: db_desc})
        json.dump(store_datas, open(self.store, 'w'))

    def store_remove(self, db_name):
        """Removes a database from store file"""
        store_datas = self.extract_store_datas()
        store_datas.pop(db_name)
        json.dump(store_datas, open(self.store, 'w'))

    def mount(self, db_name):
        db_uid = self.index['name_to_uid'][db_name] if db_name in self.index['name_to_uid'] else None

        if self[db_uid]['status'] == self.STATUSES.UNMOUNTED:
            db_path = self[db_uid]['path']

            self[db_uid]['status'] = self.STATUSES.MOUNTED
            self[db_uid]['connector'] = leveldb.LevelDB(db_path)
        else:
            return (FAILURE_STATUS,
                    [DATABASE_ERROR, "Database %r already mounted" % db_name])

        return SUCCESS_STATUS, None

    def umount(self, db_name):
        db_uid = self.index['name_to_uid'][db_name] if db_name in self.index['name_to_uid'] else None

        if self[db_uid]['status'] == self.STATUSES.MOUNTED:
            self[db_uid]['status'] = self.STATUSES.UNMOUNTED
            del self[db_uid]['connector']
            self[db_uid]['connector'] = None
        else:
            return (FAILURE_STATUS,
                    [DATABASE_ERROR, "Database %r already unmounted" % db_name])

        return SUCCESS_STATUS, None

    def add(self, db_name, db_options=None):
        """Adds a db to the DatabasesHandler object, and sync it
        to the store file"""
        db_options = db_options or DatabaseOptions()
        cache_status, ratio = self._disposable_cache(db_options["block_cache_size"])
        if not cache_status:
            return (FAILURE_STATUS,
                    [DATABASE_ERROR,
                     "Not enough disposable cache memory "
                     "%d Mo missing" % ratio])

        db_name_is_path = db_name.startswith('.') or ('/' in db_name)
        is_abspath = lambda: not db_name.startswith('.') and ('/' in db_name)

        # Handle case when a db is a path
        if db_name_is_path:
            if not is_abspath():
                return (FAILURE_STATUS,
                        [DATABASE_ERROR, "Canno't create database from relative path"])
            try:
                new_db_path = db_name
                if not os.path.exists(new_db_path):
                    os.mkdir(new_db_path)
            except OSError as e:
                return (FAILURE_STATUS,
                        [OS_ERROR, e.strerror])
        else:
            new_db_path = os.path.join(self.dest, db_name)

        path = new_db_path
        uid = str(uuid.uuid4())
        options = db_options
        self.store_update(db_name, {
            'path': path,
            'uid': uid,
            'options': options,
        })

        self.index['name_to_uid'].update({db_name: uid})
        self.update({
            uid: {
                'connector': leveldb.LevelDB(path, **options),
                'name': db_name,
                'path': path,
                'status': self.STATUSES.MOUNTED,
                'ref_count': 0,
            },
        })

        return SUCCESS_STATUS, None

    def drop(self, db_name):
        """Drops a db from the DatabasesHandler, and sync it
        to store file"""
        db_uid = self.index['name_to_uid'].pop(db_name)
        db_path = self[db_uid]['path']

        self.pop(db_uid)
        self.store_remove(db_name)

        try:
            rmtree(db_path)
        except OSError:
            return (FAILURE_STATUS,
                    [DATABASE_ERROR,
                    "Cannot drop db : %s, files not found"])

        return SUCCESS_STATUS, None

    def exists(self, db_name):
        """Checks if a database exists on disk"""
        db_uid = self.index['name_to_uid'][db_name] if db_name in self.index['name_to_uid'] else None

        if db_uid:
            if os.path.exists(self[db_uid]['path']):
                return True
            else:
                self.drop(db_name)

        return False

    def list(self):
        """Lists all the DatabasesHandler known databases"""
        return [db_name for db_name
                in [key for key
                    in self.index['name_to_uid'].iterkeys()]
                if self.exists(db_name)]
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
import numpy as np
import tensorflow as tf
from network import Network
from custom_lstm import CustomBasicLSTMCell

class PolicyVNetwork(Network):

    def __init__(self, conf):
        """ Set up remaining layers, objective and loss functions, gradient
        compute and apply ops, network parameter synchronization ops, and
        summary ops. """

        super(PolicyVNetwork, self).__init__(conf)

        self.beta = conf['args'].entropy_regularisation_strength
        self.use_recurrent = conf['args'].alg_type == 'a3c-lstm'

        with tf.name_scope(self.name):

            self.critic_target_ph = tf.placeholder(
                'float32', [None], name='target')
            self.adv_actor_ph = tf.placeholder("float", [None], name='advantage')

            if self.use_recurrent:
                layer_name = 'lstm_layer'
                self.hidden_state_size = 256
                with tf.variable_scope(self.name+'/'+layer_name) as vs:
                    self.lstm_cell = CustomBasicLSTMCell(self.hidden_state_size, forget_bias=1.0)

                    self.step_size = tf.placeholder(tf.float32, [1], name='step_size')
                    self.initial_lstm_state = tf.placeholder(
                        tf.float32, [1, 2*self.hidden_state_size], name='initital_state')

                    o3_reshaped = tf.reshape(self.o3, [1,-1,256])
                    lstm_outputs, self.lstm_state = tf.nn.dynamic_rnn(
                        self.lstm_cell,
                        o3_reshaped,
                        initial_state=self.initial_lstm_state,
                        sequence_length=self.step_size,
                        time_major=False,
                        scope=vs)

                    self.ox = tf.reshape(lstm_outputs, [-1,256])

                    # Get all LSTM trainable params
                    self.lstm_trainable_variables = [v for v in
                        tf.trainable_variables() if v.name.startswith(vs.name)]
            else:
                if self.arch == 'NIPS':
                    self.ox = self.o3
                else: #NATURE
                    self.ox = self.o4

            # Final actor layer
            layer_name = 'softmax_policy4'
            self.wpi, self.bpi, self.output_layer_pi, self.log_output_layer_pi = self._softmax_and_log_softmax(
                layer_name, self.ox, self.num_actions)

            # Avoiding log(0) by adding a very small quantity (1e-30) to output.
            # self.log_output_layer_pi = tf.log(
            #     tf.maximum(self.output_layer_pi, tf.constant(1e-30)),
            #     name=layer_name+'_log_policy')

            # Entropy: sum_a (-p_a ln p_a)
            self.output_layer_entropy = tf.reduce_sum(
                - 1.0 * tf.mul(
                    self.output_layer_pi,
                    self.log_output_layer_pi
                ), reduction_indices=1)

            # Final critic layer
            self.wv, self.bv, self.output_layer_v = self._fc(
                'fc_value4', self.ox, 1, activation='linear')

            if self.arch == 'NIPS':
                self.params = [self.w1, self.b1, self.w2, self.b2, self.w3,
                    self.b3, self.wpi, self.bpi, self.wv, self.bv]
            else: #NATURE
                self.params = [self.w1, self.b1, self.w2, self.b2, self.w3,
                    self.b3, self.w4, self.b4, self.wpi, self.bpi, self.wv, self.bv]

            if self.use_recurrent:
                self.params += self.lstm_trainable_variables

            # Advantage critic
            self.adv_critic = tf.sub(self.critic_target_ph, tf.reshape(self.output_layer_v, [-1]))

            # Actor objective
            # Multiply the output of the network by a one hot vector, 1 for the
            # executed action. This will make the non-regularised objective
            # term for non-selected actions to be zero.
            log_output_selected_action = tf.reduce_sum(
                tf.mul(self.log_output_layer_pi, self.selected_action_ph),
                reduction_indices=1
            )
            actor_objective_advantage_term = tf.mul(
                log_output_selected_action, self.adv_actor_ph
            )
            actor_objective_entropy_term = self.beta * self.output_layer_entropy
            self.actor_objective = -tf.reduce_sum(
                actor_objective_advantage_term
                + actor_objective_entropy_term
            )

            # Critic loss
            if self.clip_loss_delta > 0:
                quadratic_part = tf.minimum(tf.abs(self.adv_critic),
                    tf.constant(self.clip_loss_delta))
                linear_part = tf.sub(tf.abs(self.adv_critic), quadratic_part)
                #OBS! For the standard L2 loss, we should multiply by 0.5. However, the authors of the paper
                # recommend multiplying the gradients of the V function by 0.5. Thus the 0.5
                self.critic_loss = tf.mul(tf.constant(0.5), tf.nn.l2_loss(quadratic_part) + \
                    self.clip_loss_delta * linear_part)
            else:
                #OBS! For the standard L2 loss, we should multiply by 0.5. However, the authors of the paper
                # recommend multiplying the gradients of the V function by 0.5. Thus the 0.5
                self.critic_loss = tf.mul(tf.constant(0.5), tf.nn.l2_loss(self.adv_critic))

            self.loss = self.actor_objective + self.critic_loss

            # Optimizer
            grads = tf.gradients(self.loss, self.params)

            # This is not really an operation, but a list of gradient Tensors.
            # When calling run() on it, the value of those Tensors
            # (i.e., of the gradients) will be calculated
            if self.clip_norm_type == 'ignore':
                # Unclipped gradients
                self.get_gradients = grads
            elif self.clip_norm_type == 'global':
                # Clip network grads by network norm
                self.get_gradients = tf.clip_by_global_norm(
                            grads, self.clip_norm)[0]
            elif self.clip_norm_type == 'local':
                # Clip layer grads by layer norm
                self.get_gradients = [tf.clip_by_norm(
                            g, self.clip_norm) for g in grads]

            # Placeholders for shared memory vars
            self.params_ph = []
            for p in self.params:
                self.params_ph.append(tf.placeholder(tf.float32,
                    shape=p.get_shape(),
                    name="shared_memory_for_{}".format(
                        (p.name.split("/", 1)[1]).replace(":", "_"))))

            # Ops to sync net with shared memory vars
            self.sync_with_shared_memory = []
            for i in xrange(len(self.params)):
                self.sync_with_shared_memory.append(
                    self.params[i].assign(self.params_ph[i]))

#This is still experimental
class SequencePolicyVNetwork(Network):

    def __init__(self, conf):
        """ Set up remaining layers, objective and loss functions, gradient
        compute and apply ops, network parameter synchronization ops, and
        summary ops. """

        super(SequencePolicyVNetwork, self).__init__(conf)

        self.beta = conf['args'].entropy_regularisation_strength

        self.max_local_steps = conf['args'].max_local_steps
        self.use_recurrent = conf['args'].alg_type == 'a3c-lstm'

        with tf.name_scope(self.name):

            self.critic_target_ph = tf.placeholder(
                'float32', [None], name='target')
            self.adv_actor_ph = tf.placeholder("float", [None], name='advantage')

            if self.arch == 'NIPS':
                self.ox = self.o3
            else: #NATURE
                self.ox = self.o4

            with tf.variable_scope(self.name+'/lstm_decoder') as vs:
                self.max_seq_length = 30
                self.decoder_seq_lengths = tf.placeholder(tf.float32, [None], name='decoder_seq_lengths')
                self.action_outputs = tf.placeholder(tf.float32, [None, self.max_seq_length, self.num_actions+1], name='action_outputs')
                self.action_inputs = tf.placeholder(tf.float32, [None, self.max_seq_length, self.num_actions+1], name='action_inputs')

                self.decoder_hidden_state_size = 256
                self.decoder_lstm_cell = CustomBasicLSTMCell(self.decoder_hidden_state_size, forget_bias=1.0)

                self.network_state = tf.concat(1, [
                    tf.fill(tf.shape(self.ox), 0.0), self.ox
                ])
                self.decoder_initial_state = tf.placeholder(tf.float32, [None, 2*self.decoder_hidden_state_size], name='decoder_initial_state')

                self.modify_state = tf.placeholder(tf.bool, name='modify_state')
                initial_state_op = tf.cond(
                    self.modify_state,
                    lambda: self.decoder_initial_state,
                    lambda: self.network_state,
                    name='decode_initial_state_conditional')
                decoder_outputs, self.decoder_state = tf.nn.dynamic_rnn(
                    self.decoder_lstm_cell,
                    self.action_inputs,
                    initial_state=initial_state_op,
                    sequence_length=self.decoder_seq_lengths,
                    time_major=False,
                    scope=vs)

                self.decoder_trainable_variables = [
                    v for v in tf.trainable_variables()
                    if v.name.startswith(vs.name)
                ]

            fan_in = self.decoder_hidden_state_size
            fan_out = self.num_actions+1
            d = np.sqrt(6. / (fan_in + fan_out))
            initial = tf.random_uniform([fan_in, fan_out], minval=-d, maxval=d)
            self.W_pi = tf.Variable(initial, name='W_pi', dtype='float32')
            self.b_pi = tf.Variable(tf.zeros(fan_out, name='b_pi', dtype='float32'))

            logits = tf.einsum('ijk,kl->ijl', decoder_outputs, self.W_pi) + self.b_pi
            self.action_probs = tf.nn.softmax(logits)
            log_action_probs = tf.nn.log_softmax(logits)

            sequence_probs = tf.reduce_prod(tf.reduce_sum(self.action_probs * self.action_outputs, 2), 1)
            log_sequence_probs = tf.reduce_sum(tf.reduce_sum(log_action_probs * self.action_outputs, 2), 1)

            # ∏a_i * ∑ log a_i
            # self.output_layer_entropy = - tf.reduce_sum(sequence_probs * log_sequence_probs) / tf.stop_gradient(1e-30 + tf.reduce_sum(sequence_probs))

            self.output_layer_entropy = - tf.reduce_mean(tf.stop_gradient(1 + log_sequence_probs) * log_sequence_probs)
            self.entropy = - tf.reduce_mean(log_sequence_probs)

            # Final critic layer
            self.wv, self.bv, self.output_layer_v = self._fc(
                'fc_value4', self.ox, 1, activation='linear')

            # Advantage critic
            self.adv_critic = self.critic_target_ph - tf.reshape(self.output_layer_v, [-1])

            self.actor_advantage_term = tf.reduce_sum(log_sequence_probs[:self.max_local_steps] * self.adv_actor_ph)
            self.actor_entropy_term = self.beta * self.output_layer_entropy
            self.actor_objective = - (
                self.actor_advantage_term + self.actor_entropy_term
            )

            # Critic loss
            if self.clip_loss_delta > 0:
                quadratic_part = tf.minimum(tf.abs(self.adv_critic),
                    tf.constant(self.clip_loss_delta))
                linear_part = tf.sub(tf.abs(self.adv_critic), quadratic_part)
                #OBS! For the standard L2 loss, we should multiply by 0.5. However, the authors of the paper
                # recommend multiplying the gradients of the V function by 0.5. Thus the 0.5
                self.critic_loss = tf.mul(tf.constant(0.5), tf.nn.l2_loss(quadratic_part) + \
                    self.clip_loss_delta * linear_part)
            else:
                #OBS! For the standard L2 loss, we should multiply by 0.5. However, the authors of the paper
                # recommend multiplying the gradients of the V function by 0.5. Thus the 0.5
                self.critic_loss = tf.mul(tf.constant(0.5), tf.nn.l2_loss(self.adv_critic))

            self.loss = self.actor_objective + self.critic_loss

            if self.arch == 'NIPS':
                self.params = [self.w1, self.b1, self.w2, self.b2, self.w3,
                    self.b3, self.W_pi, self.b_pi, self.wv, self.bv]
            else: #NATURE
                self.params = [self.w1, self.b1, self.w2, self.b2, self.w3,
                    self.b3, self.w4, self.b4, self.W_pi, self.b_pi, self.wv, self.bv]

            self.params += self.decoder_trainable_variables

            # Optimizer
            grads = tf.gradients(self.loss, self.params)

            # This is not really an operation, but a list of gradient Tensors.
            # When calling run() on it, the value of those Tensors
            # (i.e., of the gradients) will be calculated
            if self.clip_norm_type == 'ignore':
                self.get_gradients = grads
            elif self.clip_norm_type == 'global':
                self.get_gradients = tf.clip_by_global_norm(grads, self.clip_norm)[0]
            elif self.clip_norm_type == 'avg':
                self.get_gradients = tf.clip_by_average_norm(grads, self.clip_norm)[0]
            elif self.clip_norm_type == 'local':
                self.get_gradients = [
                    tf.clip_by_norm(g, self.clip_norm) for g in grads
                ]

            # Placeholders for shared memory vars
            self.params_ph = []
            for p in self.params:
                self.params_ph.append(tf.placeholder(tf.float32,
                    shape=p.get_shape(),
                    name="shared_memory_for_{}".format(
                        (p.name.split("/", 1)[1]).replace(":", "_"))))

            # Ops to sync net with shared memory vars
            self.sync_with_shared_memory = []
            for i in xrange(len(self.params)):
                self.sync_with_shared_memory.append(
                    self.params[i].assign(self.params_ph[i]))

<commit_msg>remove commented out line<commit_after># -*- encoding: utf-8 -*-
import numpy as np
import tensorflow as tf
from network import Network
from custom_lstm import CustomBasicLSTMCell

class PolicyVNetwork(Network):

    def __init__(self, conf):
        """ Set up remaining layers, objective and loss functions, gradient
        compute and apply ops, network parameter synchronization ops, and
        summary ops. """

        super(PolicyVNetwork, self).__init__(conf)

        self.beta = conf['args'].entropy_regularisation_strength
        self.use_recurrent = conf['args'].alg_type == 'a3c-lstm'

        with tf.name_scope(self.name):

            self.critic_target_ph = tf.placeholder(
                'float32', [None], name='target')
            self.adv_actor_ph = tf.placeholder("float", [None], name='advantage')

            if self.use_recurrent:
                layer_name = 'lstm_layer'
                self.hidden_state_size = 256
                with tf.variable_scope(self.name+'/'+layer_name) as vs:
                    self.lstm_cell = CustomBasicLSTMCell(self.hidden_state_size, forget_bias=1.0)

                    self.step_size = tf.placeholder(tf.float32, [1], name='step_size')
                    self.initial_lstm_state = tf.placeholder(
                        tf.float32, [1, 2*self.hidden_state_size], name='initital_state')

                    o3_reshaped = tf.reshape(self.o3, [1,-1,256])
                    lstm_outputs, self.lstm_state = tf.nn.dynamic_rnn(
                        self.lstm_cell,
                        o3_reshaped,
                        initial_state=self.initial_lstm_state,
                        sequence_length=self.step_size,
                        time_major=False,
                        scope=vs)

                    self.ox = tf.reshape(lstm_outputs, [-1,256])

                    # Get all LSTM trainable params
                    self.lstm_trainable_variables = [v for v in
                        tf.trainable_variables() if v.name.startswith(vs.name)]
            else:
                if self.arch == 'NIPS':
                    self.ox = self.o3
                else: #NATURE
                    self.ox = self.o4

            # Final actor layer
            layer_name = 'softmax_policy4'
            self.wpi, self.bpi, self.output_layer_pi, self.log_output_layer_pi = self._softmax_and_log_softmax(
                layer_name, self.ox, self.num_actions)

            # Avoiding log(0) by adding a very small quantity (1e-30) to output.
            # self.log_output_layer_pi = tf.log(
            #     tf.maximum(self.output_layer_pi, tf.constant(1e-30)),
            #     name=layer_name+'_log_policy')

            # Entropy: sum_a (-p_a ln p_a)
            self.output_layer_entropy = tf.reduce_sum(
                - 1.0 * tf.mul(
                    self.output_layer_pi,
                    self.log_output_layer_pi
                ), reduction_indices=1)

            # Final critic layer
            self.wv, self.bv, self.output_layer_v = self._fc(
                'fc_value4', self.ox, 1, activation='linear')

            if self.arch == 'NIPS':
                self.params = [self.w1, self.b1, self.w2, self.b2, self.w3,
                    self.b3, self.wpi, self.bpi, self.wv, self.bv]
            else: #NATURE
                self.params = [self.w1, self.b1, self.w2, self.b2, self.w3,
                    self.b3, self.w4, self.b4, self.wpi, self.bpi, self.wv, self.bv]

            if self.use_recurrent:
                self.params += self.lstm_trainable_variables

            # Advantage critic
            self.adv_critic = tf.sub(self.critic_target_ph, tf.reshape(self.output_layer_v, [-1]))

            # Actor objective
            # Multiply the output of the network by a one hot vector, 1 for the
            # executed action. This will make the non-regularised objective
            # term for non-selected actions to be zero.
            log_output_selected_action = tf.reduce_sum(
                tf.mul(self.log_output_layer_pi, self.selected_action_ph),
                reduction_indices=1
            )
            actor_objective_advantage_term = tf.mul(
                log_output_selected_action, self.adv_actor_ph
            )
            actor_objective_entropy_term = self.beta * self.output_layer_entropy
            self.actor_objective = -tf.reduce_sum(
                actor_objective_advantage_term
                + actor_objective_entropy_term
            )

            # Critic loss
            if self.clip_loss_delta > 0:
                quadratic_part = tf.minimum(tf.abs(self.adv_critic),
                    tf.constant(self.clip_loss_delta))
                linear_part = tf.sub(tf.abs(self.adv_critic), quadratic_part)
                #OBS! For the standard L2 loss, we should multiply by 0.5. However, the authors of the paper
                # recommend multiplying the gradients of the V function by 0.5. Thus the 0.5
                self.critic_loss = tf.mul(tf.constant(0.5), tf.nn.l2_loss(quadratic_part) + \
                    self.clip_loss_delta * linear_part)
            else:
                #OBS! For the standard L2 loss, we should multiply by 0.5. However, the authors of the paper
                # recommend multiplying the gradients of the V function by 0.5. Thus the 0.5
                self.critic_loss = tf.mul(tf.constant(0.5), tf.nn.l2_loss(self.adv_critic))

            self.loss = self.actor_objective + self.critic_loss

            # Optimizer
            grads = tf.gradients(self.loss, self.params)

            # This is not really an operation, but a list of gradient Tensors.
            # When calling run() on it, the value of those Tensors
            # (i.e., of the gradients) will be calculated
            if self.clip_norm_type == 'ignore':
                # Unclipped gradients
                self.get_gradients = grads
            elif self.clip_norm_type == 'global':
                # Clip network grads by network norm
                self.get_gradients = tf.clip_by_global_norm(
                            grads, self.clip_norm)[0]
            elif self.clip_norm_type == 'local':
                # Clip layer grads by layer norm
                self.get_gradients = [tf.clip_by_norm(
                            g, self.clip_norm) for g in grads]

            # Placeholders for shared memory vars
            self.params_ph = []
            for p in self.params:
                self.params_ph.append(tf.placeholder(tf.float32,
                    shape=p.get_shape(),
                    name="shared_memory_for_{}".format(
                        (p.name.split("/", 1)[1]).replace(":", "_"))))

            # Ops to sync net with shared memory vars
            self.sync_with_shared_memory = []
            for i in xrange(len(self.params)):
                self.sync_with_shared_memory.append(
                    self.params[i].assign(self.params_ph[i]))

#This is still experimental
class SequencePolicyVNetwork(Network):

    def __init__(self, conf):
        """ Set up remaining layers, objective and loss functions, gradient
        compute and apply ops, network parameter synchronization ops, and
        summary ops. """

        super(SequencePolicyVNetwork, self).__init__(conf)

        self.beta = conf['args'].entropy_regularisation_strength

        self.max_local_steps = conf['args'].max_local_steps
        self.use_recurrent = conf['args'].alg_type == 'a3c-lstm'

        with tf.name_scope(self.name):

            self.critic_target_ph = tf.placeholder(
                'float32', [None], name='target')
            self.adv_actor_ph = tf.placeholder("float", [None], name='advantage')

            if self.arch == 'NIPS':
                self.ox = self.o3
            else: #NATURE
                self.ox = self.o4

            with tf.variable_scope(self.name+'/lstm_decoder') as vs:
                self.max_seq_length = 30
                self.decoder_seq_lengths = tf.placeholder(tf.float32, [None], name='decoder_seq_lengths')
                self.action_outputs = tf.placeholder(tf.float32, [None, self.max_seq_length, self.num_actions+1], name='action_outputs')
                self.action_inputs = tf.placeholder(tf.float32, [None, self.max_seq_length, self.num_actions+1], name='action_inputs')

                self.decoder_hidden_state_size = 256
                self.decoder_lstm_cell = CustomBasicLSTMCell(self.decoder_hidden_state_size, forget_bias=1.0)

                self.network_state = tf.concat(1, [
                    tf.fill(tf.shape(self.ox), 0.0), self.ox
                ])
                self.decoder_initial_state = tf.placeholder(tf.float32, [None, 2*self.decoder_hidden_state_size], name='decoder_initial_state')

                self.modify_state = tf.placeholder(tf.bool, name='modify_state')
                initial_state_op = tf.cond(
                    self.modify_state,
                    lambda: self.decoder_initial_state,
                    lambda: self.network_state,
                    name='decode_initial_state_conditional')
                decoder_outputs, self.decoder_state = tf.nn.dynamic_rnn(
                    self.decoder_lstm_cell,
                    self.action_inputs,
                    initial_state=initial_state_op,
                    sequence_length=self.decoder_seq_lengths,
                    time_major=False,
                    scope=vs)

                self.decoder_trainable_variables = [
                    v for v in tf.trainable_variables()
                    if v.name.startswith(vs.name)
                ]

            fan_in = self.decoder_hidden_state_size
            fan_out = self.num_actions+1
            d = np.sqrt(6. / (fan_in + fan_out))
            initial = tf.random_uniform([fan_in, fan_out], minval=-d, maxval=d)
            self.W_pi = tf.Variable(initial, name='W_pi', dtype='float32')
            self.b_pi = tf.Variable(tf.zeros(fan_out, name='b_pi', dtype='float32'))

            logits = tf.einsum('ijk,kl->ijl', decoder_outputs, self.W_pi) + self.b_pi
            self.action_probs = tf.nn.softmax(logits)
            log_action_probs = tf.nn.log_softmax(logits)

            sequence_probs = tf.reduce_prod(tf.reduce_sum(self.action_probs * self.action_outputs, 2), 1)
            log_sequence_probs = tf.reduce_sum(tf.reduce_sum(log_action_probs * self.action_outputs, 2), 1)

            # ∏a_i * ∑ log a_i
            self.output_layer_entropy = - tf.reduce_mean(tf.stop_gradient(1 + log_sequence_probs) * log_sequence_probs)
            self.entropy = - tf.reduce_mean(log_sequence_probs)

            # Final critic layer
            self.wv, self.bv, self.output_layer_v = self._fc(
                'fc_value4', self.ox, 1, activation='linear')

            # Advantage critic
            self.adv_critic = self.critic_target_ph - tf.reshape(self.output_layer_v, [-1])

            self.actor_advantage_term = tf.reduce_sum(log_sequence_probs[:self.max_local_steps] * self.adv_actor_ph)
            self.actor_entropy_term = self.beta * self.output_layer_entropy
            self.actor_objective = - (
                self.actor_advantage_term + self.actor_entropy_term
            )

            # Critic loss
            if self.clip_loss_delta > 0:
                quadratic_part = tf.minimum(tf.abs(self.adv_critic),
                    tf.constant(self.clip_loss_delta))
                linear_part = tf.sub(tf.abs(self.adv_critic), quadratic_part)
                #OBS! For the standard L2 loss, we should multiply by 0.5. However, the authors of the paper
                # recommend multiplying the gradients of the V function by 0.5. Thus the 0.5
                self.critic_loss = tf.mul(tf.constant(0.5), tf.nn.l2_loss(quadratic_part) + \
                    self.clip_loss_delta * linear_part)
            else:
                #OBS! For the standard L2 loss, we should multiply by 0.5. However, the authors of the paper
                # recommend multiplying the gradients of the V function by 0.5. Thus the 0.5
                self.critic_loss = tf.mul(tf.constant(0.5), tf.nn.l2_loss(self.adv_critic))

            self.loss = self.actor_objective + self.critic_loss

            if self.arch == 'NIPS':
                self.params = [self.w1, self.b1, self.w2, self.b2, self.w3,
                    self.b3, self.W_pi, self.b_pi, self.wv, self.bv]
            else: #NATURE
                self.params = [self.w1, self.b1, self.w2, self.b2, self.w3,
                    self.b3, self.w4, self.b4, self.W_pi, self.b_pi, self.wv, self.bv]

            self.params += self.decoder_trainable_variables

            # Optimizer
            grads = tf.gradients(self.loss, self.params)

            # This is not really an operation, but a list of gradient Tensors.
            # When calling run() on it, the value of those Tensors
            # (i.e., of the gradients) will be calculated
            if self.clip_norm_type == 'ignore':
                self.get_gradients = grads
            elif self.clip_norm_type == 'global':
                self.get_gradients = tf.clip_by_global_norm(grads, self.clip_norm)[0]
            elif self.clip_norm_type == 'avg':
                self.get_gradients = tf.clip_by_average_norm(grads, self.clip_norm)[0]
            elif self.clip_norm_type == 'local':
                self.get_gradients = [
                    tf.clip_by_norm(g, self.clip_norm) for g in grads
                ]

            # Placeholders for shared memory vars
            self.params_ph = []
            for p in self.params:
                self.params_ph.append(tf.placeholder(tf.float32,
                    shape=p.get_shape(),
                    name="shared_memory_for_{}".format(
                        (p.name.split("/", 1)[1]).replace(":", "_"))))

            # Ops to sync net with shared memory vars
            self.sync_with_shared_memory = []
            for i in xrange(len(self.params)):
                self.sync_with_shared_memory.append(
                    self.params[i].assign(self.params_ph[i]))

<|endoftext|>


<commit_before># encoding: utf-8
from __future__ import absolute_import
from __future__ import print_function

from contextlib import contextmanager
import os
import subprocess
import sys

if sys.version_info > (2, 7):
    from unittest import TestCase
else:
    from unittest2 import TestCase

if sys.version_info >= (3, 0):
    basestring = str

from PIL import Image

from needle.diff import ImageDiff
from needle.driver import NeedleWebDriver, NeedleWebElement

def _object_filename(obj):
    return os.path.abspath(sys.modules[type(obj).__module__].__file__)

class NeedleTestCase(TestCase):
    """
    A `unittest2 <http://www.voidspace.org.uk/python/articles/unittest2.shtml>`_
    test case which provides tools for testing CSS with Selenium.
    """
    #: An instance of :py:class:`~needle.driver.NeedleWebDriver`, created when
    #: each test is run.
    driver = None

    driver_command_executor = 'http://127.0.0.1:4444/wd/hub'
    driver_desired_capabilities = {
        'browserName': os.environ.get('NEEDLE_BROWSER', 'firefox'),
    }
    driver_browser_profile = None

    capture = False

    viewport_width = 1024
    viewport_height = 768

    # TODO: More robust env handling:
    use_perceptualdiff = os.environ.get('NEEDLE_USE_PERCEPTUALDIFF', False)
    perceptualdiff_path = 'perceptualdiff'
    perceptualdiff_output_png = True

    @classmethod
    def setUpClass(cls):
        if os.environ.get('NEEDLE_CAPTURE'):
            cls.capture = True
        cls.driver = cls.get_web_driver()
        cls.driver.set_window_position(0, 0)
        cls.set_viewport_size(cls.viewport_width, cls.viewport_height)
        super(NeedleTestCase, cls).setUpClass()

    @classmethod
    def tearDownClass(cls):
        super(NeedleTestCase, cls).tearDownClass()
        cls.driver.quit()

    @classmethod
    def get_web_driver(cls):
        return NeedleWebDriver(cls.driver_command_executor,
                               cls.driver_desired_capabilities,
                               cls.driver_browser_profile)

    def __init__(self, *args, **kwargs):
        super(NeedleTestCase, self).__init__(*args, **kwargs)
        # TODO: should output directory be timestamped?
        self.output_directory = os.environ.get('NEEDLE_OUTPUT_DIR', os.path.realpath(os.path.join(os.getcwd(), 'screenshots')))
        # TODO: Should baseline be a top-level peer to output_directory?
        self.baseline_directory = os.environ.get('NEEDLE_BASELINE_DIR', os.path.realpath(os.path.join(os.getcwd(), 'screenshots', 'baseline')))

        for i in (self.baseline_directory, self.output_directory):
            if not os.path.exists(i):
                print('Creating %s' % i, file=sys.stderr)
                os.makedirs(i)

    @classmethod
    def set_viewport_size(cls, viewport_width, viewport_height):
        cls.driver.set_window_size(viewport_width, viewport_height)

        # Measure the difference between the actual document width and the
        # desired viewport width so we can account for scrollbars:
        measured = cls.driver.execute_script("return {width: document.body.clientWidth, height: document.body.clientHeight};")
        delta = viewport_width - measured['width']

        cls.driver.set_window_size(viewport_width + delta, viewport_height)

    def assertScreenshot(self, element_or_selector, file, threshold=0.1):
        """assert-style variant of compareScreenshot context manager

        compareScreenshot() can be considerably more efficient for recording baselines by avoiding the need
        to load pages before checking whether we're actually going to save them. This function allows you
        to continue using normal unittest-style assertions if you don't need the efficiency benefits
        """

        with self.compareScreenshot(element_or_selector, file, threshold=threshold):
            pass

    @contextmanager
    def compareScreenshot(self, element_or_selector, file, threshold=0.1):
        """
        Assert that a screenshot of an element is the same as a screenshot on disk,
        within a given threshold.

        :param element_or_selector:
            Either a CSS selector as a string or a
            :py:class:`~needle.driver.NeedleWebElement` object that represents
            the element to capture.
        :param file:
            If a string, then assumed to be the filename for the screenshot,
            which will be appended with ``.png``. Otherwise assumed to be
            a file object for the baseline image.
        :param threshold:
            The threshold for triggering a test failure.
        """

        yield  # To allow using this method as a context manager

        if not isinstance(element_or_selector, NeedleWebElement):
            element = self.driver.find_element_by_css_selector(element_or_selector)
        else:
            element = element_or_selector

        if isinstance(file, basestring):
            baseline_file = os.path.join(self.baseline_directory, '%s.png' % file)
            output_file = os.path.join(self.output_directory, '%s.png' % file)

            if self.capture and os.path.exists(baseline_file):
                self.skipTest('Not capturing %s, image already exists. If you '
                              'want to capture this element again, delete %s'
                              % (file, baseline_file))

            if self.capture:
                element.get_screenshot().save(baseline_file)
                return
            else:
                baseline_image = Image.open(baseline_file)
                fresh_screenshot = element.get_screenshot()
                fresh_screenshot.save(output_file)

                if self.use_perceptualdiff:
                    # TODO: figure out how best to convert threshold distances to pixel counts
                    diff_ppm = output_file.replace(".png", ".diff.ppm")
                    cmd = "%s -output %s %s %s" % (self.perceptualdiff_path, diff_ppm, baseline_file, output_file)
                    process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                    perceptualdiff_stdout, _ = process.communicate()
                    if process.returncode == 0:
                        # No differences found
                        return
                    else:
                        if os.path.exists(diff_ppm):
                            if self.perceptualdiff_output_png:
                                # Convert the .ppm output to .png
                                diff_png = diff_ppm.replace("diff.ppm", "diff.png")
                                Image.open(diff_ppm).save(diff_png)
                                os.remove(diff_ppm)
                                diff_file_msg = ' (See %s)' % diff_png
                            else:
                                diff_file_msg = ' (See %s)' % diff_ppm
                        else:
                            diff_file_msg = ''
                        raise AssertionError("The saved screenshot for '%s' did not match "
                                             "the screenshot captured%s:\n%s"
                                             % (file, diff_file_msg, perceptualdiff_stdout))

        else:
            baseline_image = Image.open(file).convert('RGB')
            fresh_screenshot = element.get_screenshot()

        diff = ImageDiff(fresh_screenshot, baseline_image)
        distance = abs(diff.get_distance())
        if distance > threshold:
            raise AssertionError("The given screenshot did not match the "
                                 "screenshot captured (by a distance of %.2f)"
                                 % distance)
<commit_msg>Improved an error message.<commit_after># encoding: utf-8
from __future__ import absolute_import
from __future__ import print_function

from contextlib import contextmanager
import os
import subprocess
import sys

if sys.version_info > (2, 7):
    from unittest import TestCase
else:
    from unittest2 import TestCase

if sys.version_info >= (3, 0):
    basestring = str

from PIL import Image

from needle.diff import ImageDiff
from needle.driver import NeedleWebDriver, NeedleWebElement

def _object_filename(obj):
    return os.path.abspath(sys.modules[type(obj).__module__].__file__)

class NeedleTestCase(TestCase):
    """
    A `unittest2 <http://www.voidspace.org.uk/python/articles/unittest2.shtml>`_
    test case which provides tools for testing CSS with Selenium.
    """
    #: An instance of :py:class:`~needle.driver.NeedleWebDriver`, created when
    #: each test is run.
    driver = None

    driver_command_executor = 'http://127.0.0.1:4444/wd/hub'
    driver_desired_capabilities = {
        'browserName': os.environ.get('NEEDLE_BROWSER', 'firefox'),
    }
    driver_browser_profile = None

    capture = False

    viewport_width = 1024
    viewport_height = 768

    # TODO: More robust env handling:
    use_perceptualdiff = os.environ.get('NEEDLE_USE_PERCEPTUALDIFF', False)
    perceptualdiff_path = 'perceptualdiff'
    perceptualdiff_output_png = True

    @classmethod
    def setUpClass(cls):
        if os.environ.get('NEEDLE_CAPTURE'):
            cls.capture = True
        cls.driver = cls.get_web_driver()
        cls.driver.set_window_position(0, 0)
        cls.set_viewport_size(cls.viewport_width, cls.viewport_height)
        super(NeedleTestCase, cls).setUpClass()

    @classmethod
    def tearDownClass(cls):
        super(NeedleTestCase, cls).tearDownClass()
        cls.driver.quit()

    @classmethod
    def get_web_driver(cls):
        return NeedleWebDriver(cls.driver_command_executor,
                               cls.driver_desired_capabilities,
                               cls.driver_browser_profile)

    def __init__(self, *args, **kwargs):
        super(NeedleTestCase, self).__init__(*args, **kwargs)
        # TODO: should output directory be timestamped?
        self.output_directory = os.environ.get('NEEDLE_OUTPUT_DIR', os.path.realpath(os.path.join(os.getcwd(), 'screenshots')))
        # TODO: Should baseline be a top-level peer to output_directory?
        self.baseline_directory = os.environ.get('NEEDLE_BASELINE_DIR', os.path.realpath(os.path.join(os.getcwd(), 'screenshots', 'baseline')))

        for i in (self.baseline_directory, self.output_directory):
            if not os.path.exists(i):
                print('Creating %s' % i, file=sys.stderr)
                os.makedirs(i)

    @classmethod
    def set_viewport_size(cls, viewport_width, viewport_height):
        cls.driver.set_window_size(viewport_width, viewport_height)

        # Measure the difference between the actual document width and the
        # desired viewport width so we can account for scrollbars:
        measured = cls.driver.execute_script("return {width: document.body.clientWidth, height: document.body.clientHeight};")
        delta = viewport_width - measured['width']

        cls.driver.set_window_size(viewport_width + delta, viewport_height)

    def assertScreenshot(self, element_or_selector, file, threshold=0.1):
        """assert-style variant of compareScreenshot context manager

        compareScreenshot() can be considerably more efficient for recording baselines by avoiding the need
        to load pages before checking whether we're actually going to save them. This function allows you
        to continue using normal unittest-style assertions if you don't need the efficiency benefits
        """

        with self.compareScreenshot(element_or_selector, file, threshold=threshold):
            pass

    @contextmanager
    def compareScreenshot(self, element_or_selector, file, threshold=0.1):
        """
        Assert that a screenshot of an element is the same as a screenshot on disk,
        within a given threshold.

        :param element_or_selector:
            Either a CSS selector as a string or a
            :py:class:`~needle.driver.NeedleWebElement` object that represents
            the element to capture.
        :param file:
            If a string, then assumed to be the filename for the screenshot,
            which will be appended with ``.png``. Otherwise assumed to be
            a file object for the baseline image.
        :param threshold:
            The threshold for triggering a test failure.
        """

        yield  # To allow using this method as a context manager

        if not isinstance(element_or_selector, NeedleWebElement):
            element = self.driver.find_element_by_css_selector(element_or_selector)
        else:
            element = element_or_selector

        if isinstance(file, basestring):
            baseline_file = os.path.join(self.baseline_directory, '%s.png' % file)
            output_file = os.path.join(self.output_directory, '%s.png' % file)

            if self.capture and os.path.exists(baseline_file):
                self.skipTest('Not capturing %s, image already exists. If you '
                              'want to capture this element again, delete %s'
                              % (file, baseline_file))

            if self.capture:
                element.get_screenshot().save(baseline_file)
                return
            else:
                if not os.path.exists(baseline_file):
                    raise IOError('The baseline screenshot %s does not exist. '
                                  'You might want to re-run this test in baseline-saving mode.'
                                  % baseline_file)

                baseline_image = Image.open(baseline_file)

                fresh_screenshot = element.get_screenshot()
                fresh_screenshot.save(output_file)

                if self.use_perceptualdiff:
                    # TODO: figure out how best to convert threshold distances to pixel counts
                    diff_ppm = output_file.replace(".png", ".diff.ppm")
                    cmd = "%s -output %s %s %s" % (self.perceptualdiff_path, diff_ppm, baseline_file, output_file)
                    process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                    perceptualdiff_stdout, _ = process.communicate()
                    if process.returncode == 0:
                        # No differences found
                        return
                    else:
                        if os.path.exists(diff_ppm):
                            if self.perceptualdiff_output_png:
                                # Convert the .ppm output to .png
                                diff_png = diff_ppm.replace("diff.ppm", "diff.png")
                                Image.open(diff_ppm).save(diff_png)
                                os.remove(diff_ppm)
                                diff_file_msg = ' (See %s)' % diff_png
                            else:
                                diff_file_msg = ' (See %s)' % diff_ppm
                        else:
                            diff_file_msg = ''
                        raise AssertionError("The saved screenshot for '%s' did not match "
                                             "the screenshot captured%s:\n%s"
                                             % (file, diff_file_msg, perceptualdiff_stdout))

        else:
            baseline_image = Image.open(file).convert('RGB')
            fresh_screenshot = element.get_screenshot()

        diff = ImageDiff(fresh_screenshot, baseline_image)
        distance = abs(diff.get_distance())
        if distance > threshold:
            raise AssertionError("The given screenshot did not match the "
                                 "screenshot captured (by a distance of %.2f)"
                                 % distance)
<|endoftext|>


<commit_before>from typing import Dict, Mapping, List

explored = None
topological_order = None
current_label = None

def topological_sort(g: Mapping[str, List[str]]) -> Dict[str, int]:
    """
    Return a linear ordering of a directed graph's vertices such that for every directed edge uv from vertex u to vertex v, u comes before v in the ordering (definition from wikipedia).

    :param dict g: graph represented as an adjacency list.
    :returns: dict of nodes as keys with values representing their topological ordering.
    """
    global explored
    global topological_order
    global current_label

    nodes = g.keys()
    explored = set([])
    current_label = len(g) # count backwards
    topological_order = {}

    for node in nodes:
        if node not in explored:
            dfs(g, node)
        explored.add(node)
    return topological_order

def dfs(g: Mapping[str, List[str]], node: str):
    """
    Depth first search of a graph given a starting node that adds leaf or 'sink' nodes to topological_order as they are popped off the recursive stack.

    :param dict g: graph represented as an adjacency list.
    :param str node: string representing node in graph.
    """
    global explored
    global topological_order
    global current_label
    neighboring_nodes = g.get(node, None)
    if neighboring_nodes == None:
        return
    for neighbor in neighboring_nodes:
        if neighbor not in explored:
            explored.add(neighbor)
            dfs(g, neighbor)
    topological_order[node] = current_label
    current_label -= 1
<commit_msg>update formatting for linting<commit_after>from typing import Dict, Mapping, List

explored = None
topological_order = None
current_label = None

def topological_sort(g: Mapping[str, List[str]]) -> Dict[str, int]:
    """
    Return a linear ordering of a directed graph's vertices such that for every directed edge uv from vertex u to
    vertex v, u comes before v in the ordering (definition from wikipedia).

    :param dict g: graph represented as an adjacency list.
    :returns: dict of nodes as keys with values representing their topological ordering.
    """
    global explored
    global topological_order
    global current_label

    nodes = g.keys()
    explored = set([])
    current_label = len(g)  # count backwards
    topological_order = {}

    for node in nodes:
        if node not in explored:
            dfs(g, node)
        explored.add(node)
    return topological_order

def dfs(g: Mapping[str, List[str]], node: str):
    """
    Depth first search of a graph given a starting node that adds leaf or 'sink' nodes to topological_order as they
    are popped off the recursive stack.

    :param dict g: graph represented as an adjacency list.
    :param str node: string representing node in graph.
    """
    global explored
    global topological_order
    global current_label
    neighboring_nodes = g.get(node, None)
    if neighboring_nodes is None:
        return
    for neighbor in neighboring_nodes:
        if neighbor not in explored:
            explored.add(neighbor)
            dfs(g, neighbor)
    topological_order[node] = current_label
    current_label -= 1
<|endoftext|>


<commit_before>
def is_balanced(root):
    """
    O(N) solution
    """
    return -1 != get_depth(root)

def get_depth(root):
    """
    return 0 if unbalanced else depth + 1
    """
    if not root:
        return 0
    left  = get_depth(root.left)
    right = get_depth(root.right)
    if abs(left-right) > 1:
        return -1
    return 1 + max(left, right)

################################

def is_balanced(root):
    """
    O(N^2) solution
    """
    left = max_height(root.left)
    right = max_height(root.right)
    return abs(left-right) <= 1 and is_balanced(root.left) and is_balanced(root.right)

def max_height(root):
    if not root:
        return 0
    return max(max_height(root.left), max_height(root.right)) + 1
<commit_msg>Update is_balanced.py (#398)<commit_after>
def is_balanced(root):
    """
    O(N) solution
    """
    return -1 != get_depth(root)

def get_depth(root):
    """
    return 0 if unbalanced else depth + 1
    """
    if not root:
        return 0
    left  = get_depth(root.left)
    right = get_depth(root.right)
    if abs(left-right) > 1 or left == -1 or right == -1:
        return -1
    return 1 + max(left, right)

################################

def is_balanced(root):
    """
    O(N^2) solution
    """
    left = max_height(root.left)
    right = max_height(root.right)
    return abs(left-right) <= 1 and is_balanced(root.left) and is_balanced(root.right)

def max_height(root):
    if not root:
        return 0
    return max(max_height(root.left), max_height(root.right)) + 1
<|endoftext|>


<commit_before>import asyncio
import asyncio.events as events
import os
import sys
import threading
from contextlib import contextmanager, suppress
from heapq import heappop

def apply(loop=None):
    """Patch asyncio to make its event loop reentrant."""
    loop = loop or events._get_running_loop()
    if not isinstance(loop, asyncio.BaseEventLoop):
        raise ValueError('Can\'t patch loop of type %s' % type(loop))
    if getattr(loop, '_nest_patched', None):
        # already patched
        return
    _patch_asyncio()
    _patch_loop(loop)
    _patch_task()
    _patch_tornado()

def _patch_asyncio():
    """
    Patch asyncio module to use pure Python tasks and futures,
    use module level _current_tasks, all_tasks and patch run method.
    """
    def run(main, *, debug=False):
        loop = events._get_running_loop()
        if not loop:
            loop = events.new_event_loop()
            events.set_event_loop(loop)
            _patch_loop(loop)
        loop.set_debug(debug)
        task = asyncio.ensure_future(main)
        try:
            return loop.run_until_complete(task)
        finally:
            if not task.done():
                task.cancel()
                with suppress(asyncio.CancelledError):
                    loop.run_until_complete(task)

    if sys.version_info >= (3, 6, 0):
        asyncio.Task = asyncio.tasks._CTask = asyncio.tasks.Task = \
            asyncio.tasks._PyTask
        asyncio.Future = asyncio.futures._CFuture = asyncio.futures.Future = \
            asyncio.futures._PyFuture
    if sys.version_info < (3, 7, 0):
        asyncio.tasks._current_tasks = asyncio.tasks.Task._current_tasks
        asyncio.all_tasks = asyncio.tasks.Task.all_tasks
    if not hasattr(asyncio, '_run_orig'):
        asyncio._run_orig = getattr(asyncio, 'run', None)
        asyncio.run = run

def _patch_loop(loop):
    """Patch loop to make it reentrant."""

    def run_forever(self):
        with manage_run(self), manage_asyncgens(self):
            while True:
                self._run_once()
                if self._stopping:
                    break
        self._stopping = False

    def run_until_complete(self, future):
        with manage_run(self):
            f = asyncio.ensure_future(future, loop=self)
            if f is not future:
                f._log_destroy_pending = False
            while not f.done():
                self._run_once()
                if self._stopping:
                    break
            if not f.done():
                raise RuntimeError(
                    'Event loop stopped before Future completed.')
            return f.result()

    def _run_once(self):
        """
        Simplified re-implementation of asyncio's _run_once that
        runs handles as they become ready.
        """
        ready = self._ready
        scheduled = self._scheduled
        while scheduled and scheduled[0]._cancelled:
            heappop(scheduled)

        timeout = (
            0 if ready or self._stopping
            else min(max(
                scheduled[0]._when - self.time(), 0), 86400) if scheduled
            else None)
        event_list = self._selector.select(timeout)
        self._process_events(event_list)

        end_time = self.time() + self._clock_resolution
        while scheduled and scheduled[0]._when < end_time:
            handle = heappop(scheduled)
            ready.append(handle)

        for _ in range(len(ready)):
            if not ready:
                break
            handle = ready.popleft()
            if not handle._cancelled:
                handle._run()
        handle = None

    @contextmanager
    def manage_run(self):
        """Set up the loop for running."""
        self._check_closed()
        old_thread_id = self._thread_id
        old_running_loop = events._get_running_loop()
        try:
            self._thread_id = threading.get_ident()
            events._set_running_loop(self)
            self._num_runs_pending += 1
            if self._is_proactorloop:
                if self._self_reading_future is None:
                    self.call_soon(self._loop_self_reading)
            yield
        finally:
            self._thread_id = old_thread_id
            events._set_running_loop(old_running_loop)
            self._num_runs_pending -= 1
            if self._is_proactorloop:
                if (self._num_runs_pending == 0
                        and self._self_reading_future is not None):
                    ov = self._self_reading_future._ov
                    self._self_reading_future.cancel()
                    if ov is not None:
                        self._proactor._unregister(ov)
                    self._self_reading_future = None

    @contextmanager
    def manage_asyncgens(self):
        if not hasattr(sys, 'get_asyncgen_hooks'):
            # Python version is too old.
            return
        old_agen_hooks = sys.get_asyncgen_hooks()
        try:
            self._set_coroutine_origin_tracking(self._debug)
            if self._asyncgens is not None:
                sys.set_asyncgen_hooks(
                    firstiter=self._asyncgen_firstiter_hook,
                    finalizer=self._asyncgen_finalizer_hook)
            yield
        finally:
            self._set_coroutine_origin_tracking(False)
            if self._asyncgens is not None:
                sys.set_asyncgen_hooks(*old_agen_hooks)

    def _check_running(self):
        """Do not throw exception if loop is already running."""
        pass

    cls = loop.__class__
    cls.run_forever = run_forever
    cls.run_until_complete = run_until_complete
    cls._run_once = _run_once
    cls._check_running = _check_running
    cls._check_runnung = _check_running  # typo in Python 3.7 source
    cls._nest_patched = True
    cls._num_runs_pending = 0
    cls._is_proactorloop = (
            os.name == 'nt' and issubclass(cls, asyncio.ProactorEventLoop))
    if sys.version_info < (3, 7, 0):
        cls._set_coroutine_origin_tracking = cls._set_coroutine_wrapper

def _patch_task():
    """Patch the Task's step and enter/leave methods to make it reentrant."""
    Task = asyncio.Task
    if sys.version_info >= (3, 7, 0):

        def enter_task(loop, task):
            curr_tasks[loop] = task

        def leave_task(loop, task):
            curr_tasks.pop(loop, None)

        asyncio.tasks._enter_task = enter_task
        asyncio.tasks._leave_task = leave_task
        curr_tasks = asyncio.tasks._current_tasks
    else:

        def step(task, exc=None):
            curr_task = curr_tasks.get(task._loop)
            try:
                step_orig(task, exc)
            finally:
                if curr_task is None:
                    curr_tasks.pop(task._loop, None)
                else:
                    curr_tasks[task._loop] = curr_task

        curr_tasks = Task._current_tasks
        step_orig = Task._step
        Task._step = step

def _patch_tornado():
        """
        If tornado is imported before nest_asyncio, make tornado aware of
        the pure-Python asyncio Future.
        """
        if 'tornado' in sys.modules:
            import tornado.concurrent as tc
            tc.Future = asyncio.Future
            if asyncio.Future not in tc.FUTURES:
                tc.FUTURES += (asyncio.Future,)
<commit_msg>Revert to using asyncio.get_event_loop again, fixes #62<commit_after>import asyncio
import asyncio.events as events
import os
import sys
import threading
from contextlib import contextmanager, suppress
from heapq import heappop

def apply(loop=None):
    """Patch asyncio to make its event loop reentrant."""
    loop = loop or asyncio.get_event_loop()
    if not isinstance(loop, asyncio.BaseEventLoop):
        raise ValueError('Can\'t patch loop of type %s' % type(loop))
    if getattr(loop, '_nest_patched', None):
        # already patched
        return
    _patch_asyncio()
    _patch_loop(loop)
    _patch_task()
    _patch_tornado()

def _patch_asyncio():
    """
    Patch asyncio module to use pure Python tasks and futures,
    use module level _current_tasks, all_tasks and patch run method.
    """
    def run(main, *, debug=False):
        loop = events._get_running_loop()
        if not loop:
            loop = events.new_event_loop()
            events.set_event_loop(loop)
            _patch_loop(loop)
        loop.set_debug(debug)
        task = asyncio.ensure_future(main)
        try:
            return loop.run_until_complete(task)
        finally:
            if not task.done():
                task.cancel()
                with suppress(asyncio.CancelledError):
                    loop.run_until_complete(task)

    if sys.version_info >= (3, 6, 0):
        asyncio.Task = asyncio.tasks._CTask = asyncio.tasks.Task = \
            asyncio.tasks._PyTask
        asyncio.Future = asyncio.futures._CFuture = asyncio.futures.Future = \
            asyncio.futures._PyFuture
    if sys.version_info < (3, 7, 0):
        asyncio.tasks._current_tasks = asyncio.tasks.Task._current_tasks
        asyncio.all_tasks = asyncio.tasks.Task.all_tasks
    if not hasattr(asyncio, '_run_orig'):
        asyncio._run_orig = getattr(asyncio, 'run', None)
        asyncio.run = run

def _patch_loop(loop):
    """Patch loop to make it reentrant."""

    def run_forever(self):
        with manage_run(self), manage_asyncgens(self):
            while True:
                self._run_once()
                if self._stopping:
                    break
        self._stopping = False

    def run_until_complete(self, future):
        with manage_run(self):
            f = asyncio.ensure_future(future, loop=self)
            if f is not future:
                f._log_destroy_pending = False
            while not f.done():
                self._run_once()
                if self._stopping:
                    break
            if not f.done():
                raise RuntimeError(
                    'Event loop stopped before Future completed.')
            return f.result()

    def _run_once(self):
        """
        Simplified re-implementation of asyncio's _run_once that
        runs handles as they become ready.
        """
        ready = self._ready
        scheduled = self._scheduled
        while scheduled and scheduled[0]._cancelled:
            heappop(scheduled)

        timeout = (
            0 if ready or self._stopping
            else min(max(
                scheduled[0]._when - self.time(), 0), 86400) if scheduled
            else None)
        event_list = self._selector.select(timeout)
        self._process_events(event_list)

        end_time = self.time() + self._clock_resolution
        while scheduled and scheduled[0]._when < end_time:
            handle = heappop(scheduled)
            ready.append(handle)

        for _ in range(len(ready)):
            if not ready:
                break
            handle = ready.popleft()
            if not handle._cancelled:
                handle._run()
        handle = None

    @contextmanager
    def manage_run(self):
        """Set up the loop for running."""
        self._check_closed()
        old_thread_id = self._thread_id
        old_running_loop = events._get_running_loop()
        try:
            self._thread_id = threading.get_ident()
            events._set_running_loop(self)
            self._num_runs_pending += 1
            if self._is_proactorloop:
                if self._self_reading_future is None:
                    self.call_soon(self._loop_self_reading)
            yield
        finally:
            self._thread_id = old_thread_id
            events._set_running_loop(old_running_loop)
            self._num_runs_pending -= 1
            if self._is_proactorloop:
                if (self._num_runs_pending == 0
                        and self._self_reading_future is not None):
                    ov = self._self_reading_future._ov
                    self._self_reading_future.cancel()
                    if ov is not None:
                        self._proactor._unregister(ov)
                    self._self_reading_future = None

    @contextmanager
    def manage_asyncgens(self):
        if not hasattr(sys, 'get_asyncgen_hooks'):
            # Python version is too old.
            return
        old_agen_hooks = sys.get_asyncgen_hooks()
        try:
            self._set_coroutine_origin_tracking(self._debug)
            if self._asyncgens is not None:
                sys.set_asyncgen_hooks(
                    firstiter=self._asyncgen_firstiter_hook,
                    finalizer=self._asyncgen_finalizer_hook)
            yield
        finally:
            self._set_coroutine_origin_tracking(False)
            if self._asyncgens is not None:
                sys.set_asyncgen_hooks(*old_agen_hooks)

    def _check_running(self):
        """Do not throw exception if loop is already running."""
        pass

    cls = loop.__class__
    cls.run_forever = run_forever
    cls.run_until_complete = run_until_complete
    cls._run_once = _run_once
    cls._check_running = _check_running
    cls._check_runnung = _check_running  # typo in Python 3.7 source
    cls._nest_patched = True
    cls._num_runs_pending = 0
    cls._is_proactorloop = (
            os.name == 'nt' and issubclass(cls, asyncio.ProactorEventLoop))
    if sys.version_info < (3, 7, 0):
        cls._set_coroutine_origin_tracking = cls._set_coroutine_wrapper

def _patch_task():
    """Patch the Task's step and enter/leave methods to make it reentrant."""
    Task = asyncio.Task
    if sys.version_info >= (3, 7, 0):

        def enter_task(loop, task):
            curr_tasks[loop] = task

        def leave_task(loop, task):
            curr_tasks.pop(loop, None)

        asyncio.tasks._enter_task = enter_task
        asyncio.tasks._leave_task = leave_task
        curr_tasks = asyncio.tasks._current_tasks
    else:

        def step(task, exc=None):
            curr_task = curr_tasks.get(task._loop)
            try:
                step_orig(task, exc)
            finally:
                if curr_task is None:
                    curr_tasks.pop(task._loop, None)
                else:
                    curr_tasks[task._loop] = curr_task

        curr_tasks = Task._current_tasks
        step_orig = Task._step
        Task._step = step

def _patch_tornado():
        """
        If tornado is imported before nest_asyncio, make tornado aware of
        the pure-Python asyncio Future.
        """
        if 'tornado' in sys.modules:
            import tornado.concurrent as tc
            tc.Future = asyncio.Future
            if asyncio.Future not in tc.FUTURES:
                tc.FUTURES += (asyncio.Future,)
<|endoftext|>


<commit_before>from unittest.mock import Mock

from django.template.loader import render_to_string
from django.test import TestCase

from osmaxx.excerptexport.forms.order_options_mixin import available_format_choices

class MailtoUriTestCase(TestCase):
    def setUp(self):
        order = Mock(
            excerpt_name="Neverland",
        )
        files = [
            Mock(
                deleted_on_filesystem=False,
                content_type=format_choice[0],
                public_identifier='some-long-id',
            ) for format_choice in available_format_choices
        ]
        order.configure_mock(
            **{
                'output_files.all': files,
            }
        )
        self.context = dict(
            extraction_order=order,
            host_domain='example.com',
            protocol='http',
        )

    def test_send_all_links_link_has_expected_URI(self):
        actual_result_string = render_to_string('excerptexport/partials/send_all_links.html', context=self.context)
        expected_send_all_links_link = """
        <a href="mailto:?subject=Download%20map%20data%20of%20Neverland&body=ESRI%20File%20Geodatabase%20%28fgdb%29%3A%20http%3A//example.com/downloads/some-long-id/%0D%0AESRI%20Shapefile%20%28shp%29%3A%20http%3A//example.com/downloads/some-long-id/%0D%0AGeoPackage%20%28gpkg%29%3A%20http%3A//example.com/downloads/some-long-id/%0D%0ASQLite%20based%20SpatiaLite%20%28spatialite%29%3A%20http%3A//example.com/downloads/some-long-id/%0D%0AGarmin%20navigation%20%26%20map%20data%20%28garmin%29%3A%20http%3A//example.com/downloads/some-long-id/%0D%0A">
            <button>✉ Send all links</button>
        """  # noqa
        self.assertHTMLEqual(expected_send_all_links_link, actual_result_string)

    def test_send_link_links_in_download_list_have_expected_URIs(self):
        actual_result_string = render_to_string('excerptexport/partials/download_list.html', context=self.context)
        self.assertInHTML(
            """

               <button>✉ Send link</button>

            """,  # noqa
            actual_result_string
        )
        self.assertInHTML(
            """

                <button>✉ Send link</button>

            """,  # noqa
            actual_result_string
        )
        self.assertInHTML(
            """

                <button>✉ Send link</button>

            """,  # noqa
            actual_result_string
        )
        self.assertInHTML(
            """

                <button>✉ Send link</button>

            """,  # noqa
            actual_result_string
        )
        self.assertInHTML(
            """

                <button>✉ Send link</button>

            """,  # noqa
            actual_result_string
        )
<commit_msg>simplify mailto partials test setUp #381<commit_after>from unittest.mock import Mock

from django.template.loader import render_to_string
from django.test import TestCase

from osmaxx.excerptexport.forms.order_options_mixin import available_format_choices

class MailtoUriTestCase(TestCase):
    def setUp(self):
        files = [
            Mock(
                deleted_on_filesystem=False,
                content_type=format_choice[0],
                public_identifier='some-long-id',
            ) for format_choice in available_format_choices
        ]
        order = Mock(
            excerpt_name="Neverland",
            **{
                'output_files.all': files,
            }
        )
        self.context = dict(
            extraction_order=order,
            host_domain='example.com',
            protocol='http',
        )

    def test_send_all_links_link_has_expected_URI(self):
        actual_result_string = render_to_string('excerptexport/partials/send_all_links.html', context=self.context)
        expected_send_all_links_link = """
        <a href="mailto:?subject=Download%20map%20data%20of%20Neverland&body=ESRI%20File%20Geodatabase%20%28fgdb%29%3A%20http%3A//example.com/downloads/some-long-id/%0D%0AESRI%20Shapefile%20%28shp%29%3A%20http%3A//example.com/downloads/some-long-id/%0D%0AGeoPackage%20%28gpkg%29%3A%20http%3A//example.com/downloads/some-long-id/%0D%0ASQLite%20based%20SpatiaLite%20%28spatialite%29%3A%20http%3A//example.com/downloads/some-long-id/%0D%0AGarmin%20navigation%20%26%20map%20data%20%28garmin%29%3A%20http%3A//example.com/downloads/some-long-id/%0D%0A">
            <button>✉ Send all links</button>
        """  # noqa
        self.assertHTMLEqual(expected_send_all_links_link, actual_result_string)

    def test_send_link_links_in_download_list_have_expected_URIs(self):
        actual_result_string = render_to_string('excerptexport/partials/download_list.html', context=self.context)
        self.assertInHTML(
            """

               <button>✉ Send link</button>

            """,  # noqa
            actual_result_string
        )
        self.assertInHTML(
            """

                <button>✉ Send link</button>

            """,  # noqa
            actual_result_string
        )
        self.assertInHTML(
            """

                <button>✉ Send link</button>

            """,  # noqa
            actual_result_string
        )
        self.assertInHTML(
            """

                <button>✉ Send link</button>

            """,  # noqa
            actual_result_string
        )
        self.assertInHTML(
            """

                <button>✉ Send link</button>

            """,  # noqa
            actual_result_string
        )
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding: utf-8
"""
    em
    ~~

    em program is a terminal tool that prints FILE(s), or standard input
    to standard output and highlights the expressions that are matched
    the PATTERN.

    The expression will be highlighted iff the terminal is ANSI-compatible.

    :copyright: (c) 2014, Igor Kalnitsky
    :license: BSD, see LICENSE for details
"""
from __future__ import print_function, unicode_literals

import re
import os
import sys
import gettext
import argparse

__version__ = '0.3-dev'

#: True if Python 2.x interpreter was detected
PY2 = sys.version_info[0] == 2

def get_ansi_color(color):
    """
    Returns an ANSI-escape code for a given color.
    """
    colors = {
        # attributes
        'reset':        '\033[0m',
        'bold':         '\033[1m',
        'underline':    '\033[4m',

        # colors
        'grey':         '\033[30m',
        'red':          '\033[31m',
        'green':        '\033[32m',
        'yellow':       '\033[33m',
        'blue':         '\033[34m',
        'magenta':      '\033[35m',
        'cyan':         '\033[36m',
        'white':        '\033[37m',
    }
    return colors.get(color.lower())

def error(statuscode, cause, message):
    """
    Print a given `message` to standard error and terminate the program
    with `statuscode` value. It's a good practice to use Unix convention
    for status code: `2` for command line syntax error and `1` for all
    other kind of errors.
    """
    print('{0}: {1}: {2}'.format(__name__, cause, message), file=sys.stderr)
    sys.exit(statuscode)

def iterate_over_stream(stream):
    """
    Iterate over lines from a given ``stream``. Unlike stream's iterative
    interface this function doesn't block code execution in Python 2.x.
    """
    BUF_SIZE = 32768

    for buf in iter(lambda: os.read(stream.fileno(), BUF_SIZE), b''):
        for line in buf.decode(sys.getfilesystemencoding()).splitlines():
            yield line

def emphasize(stream, patterns):
    """
    Emphasize a given ``patterns`` in a given ``stream`` and print the
    result to ``stdout``. The ``patterns`` argument must be represent as
    a dictionay where the key is a pattern, and the value is a format
    lexem (e.g. RED, BLUE, UNDERLINE).
    """
    # compile patterns for quick execution
    def re_compile(k, v):
        flags = re.UNICODE
        flags |= re.IGNORECASE if v['ignore_case'] else 0
        return re.compile('(%s)' % k, flags)
    patterns = {re_compile(k, v): v for k, v in patterns.items()}

    # don't use `stream.read()` (or its iterative interface) here as it can
    # possibly block until the block of the requested size is read fully
    # (actual for python 2.x)
    for line in iterate_over_stream(stream):
        # colorize matched patterns with ANSI-escapes
        for pattern, settings in patterns.items():
            if settings['line_mode'] and pattern.search(line):
                line = '{style}{line}{reset}'.format(
                    style=get_ansi_color(settings['format']),
                    line=line,
                    reset=get_ansi_color('reset')
                )
            else:
                line = pattern.sub(r'{style}\1{reset}'.format(
                    style=get_ansi_color(settings['format']),
                    reset=get_ansi_color('reset')
                ), line)
        print(line)

def get_arguments():
    """
    Create and parse command line interface.
    """
    def _str_to_unicode(string):
        return string.decode(sys.getfilesystemencoding()) if PY2 else string

    parser = argparse.ArgumentParser(
        description=_(
            '%(prog)s program is a terminal tool that prints FILE(s), '
            'or standard input to standard output and highlights the '
            'expressions that are matched the PATTERN.'),
        epilog=_(
            'With no FILE, or when FILE is -, read standard input.'
            '  '
            'The FORMAT option must be one of: BOLD, UNDERLINE, GREY, RED, '
            'GREEN, YELLOW, BLUE, MAGENTA, CYAN or WHITE.')
    )

    arg = parser.add_argument('pattern', metavar='PATTERN')
    arg.help = _('a pattern to highlight')
    arg.type = _str_to_unicode

    arg = parser.add_argument('format', metavar='FORMAT')
    arg.help = _('a color to highlight matched expressions')

    arg = parser.add_argument('files', metavar='FILE', nargs='*', default=['-'])
    arg.help = _('search for pattern in these file(s)')
    arg.type = _str_to_unicode

    arg = parser.add_argument('-i', '--ignore-case', action='store_true')
    arg.help = _('ignore case distinctions')

    arg = parser.add_argument('-l', '--line-mode', action='store_true')
    arg.help = _('highlight entire line')

    arg = parser.add_argument('-s', '--safe-mode', action='store_true')
    arg.help = _('highlight only when stdout refers to tty')

    arg = parser.add_argument('--version', action='version')
    arg.version = '%(prog)s ' + __version__
    arg.help = _("show program's version number and exit")

    # TODO: add option to load pattern/format settings from the file

    # localization trick (originally stores in argparse module)
    _('show this help message and exit')
    _("show program's version number and exit")

    return parser.parse_args()

def validate_arguments(arguments):
    """
    Validate arguments from the command line.
    """
    if not get_ansi_color(arguments.format):
        error(2, arguments.format, _('Unknown format value'))

def main():
    # initialize localization subsystem
    localedir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'locale')
    kwargs = {'unicode': True, } if PY2 else {}
    gettext.install('em', localedir, **kwargs)

    # parse command line arguments and validate it.
    # terminate the program if error was occured.
    arguments = get_arguments()
    validate_arguments(arguments)
    patterns = {
        arguments.pattern: {
            'format': arguments.format,
            'ignore_case': arguments.ignore_case,
            'line_mode': arguments.line_mode,
        }
    }

    # iterate over the files and highlight the given patterns
    for filename in arguments.files:
        try:
            stream = sys.stdin if filename == '-' else open(filename, 'r')

            if arguments.safe_mode and not sys.stdout.isatty():
                sys.stdout.write(stream.read())
            else:
                emphasize(stream, patterns)

        # since Python 3.3 `IOEror` has been merge into `OSError`, but
        # still available as alias to the last one
        except IOError as e:
            error(1, '{0}: {1}: {2}'.format(
                __name__, e.filename, e.strerror
            ))
        finally:
            if filename != '-':
                stream.close()

if __name__ == '__main__':
    main()
<commit_msg>Refactor some parts of code.<commit_after>#!/usr/bin/env python
# coding: utf-8
"""
    em
    ~~

    em program is a terminal tool that prints FILE(s), or standard input
    to standard output and highlights the expressions that are matched
    the PATTERN.

    The expression will be highlighted iff the terminal is ANSI-compatible.

    :copyright: (c) 2014, Igor Kalnitsky
    :license: BSD, see LICENSE for details
"""
from __future__ import print_function, unicode_literals

import re
import os
import sys
import gettext
import argparse

__version__ = '0.3-dev'

#: True if Python 2.x interpreter was detected
PY2 = sys.version_info[0] == 2

def get_ansi_color(color):
    """
    Returns an ANSI-escape code for a given color.
    """
    colors = {
        # attributes
        'reset':        '\033[0m',
        'bold':         '\033[1m',
        'underline':    '\033[4m',

        # colors
        'grey':         '\033[30m',
        'red':          '\033[31m',
        'green':        '\033[32m',
        'yellow':       '\033[33m',
        'blue':         '\033[34m',
        'magenta':      '\033[35m',
        'cyan':         '\033[36m',
        'white':        '\033[37m',
    }
    return colors.get(color.lower())

def error(statuscode, cause, message):
    """
    Print a given `message` to standard error and terminate the program
    with `statuscode` value. It's a good practice to use Unix convention
    for status code: `2` for command line syntax error and `1` for all
    other kind of errors.
    """
    print('{0}: {1}: {2}'.format(__name__, cause, message), file=sys.stderr)
    sys.exit(statuscode)

def iterate_over_stream(stream):
    """
    Iterate over lines from a given `stream`. Unlike stream's iterative
    interface this function doesn't block code execution in Python 2.x.
    """
    BUF_SIZE = 32768

    for buf in iter(lambda: os.read(stream.fileno(), BUF_SIZE), b''):
        for line in buf.decode(sys.getfilesystemencoding()).splitlines():
            yield line

def emphasize_line(line, pattern, color):
    """
    Highlight a given `line` with a given `color` if the `pattern` has
    been found in the line.
    """
    if pattern.search(line):
        color = get_ansi_color(color)
        reset = get_ansi_color('reset')
        return '{0}{1}{2}'.format(color, line, reset)
    return line

def emphasize_pattern(line, pattern, color):
    """
    Search for `pattern` in the `line` and highlight it if it has been
    found.
    """
    color = get_ansi_color(color)
    reset = get_ansi_color('reset')
    return pattern.sub(r'{0}\1{1}'.format(color, reset), line)

def emphasize(stream, patterns):
    """
    Emphasize a given `patterns` in a given `stream` and print the result
    to `stdout`. The `patterns` argument must be represent as a dictionry
    where the key is a pattern, and the value is a format lexem
    (e.g. RED, BLUE, UNDERLINE).
    """
    # compile patterns for quick execution
    def re_compile(k, v):
        flags = re.UNICODE
        flags |= re.IGNORECASE if v['ignore_case'] else 0
        return re.compile('(%s)' % k, flags)
    patterns = {re_compile(k, v): v for k, v in patterns.items()}

    # don't use `stream.read()` (or its iterative interface) here as it can
    # possibly block until the block of the requested size is read fully
    # (actual for python 2.x)
    for line in iterate_over_stream(stream):
        for pattern, settings in patterns.items():
            color = settings['format']
            if settings['line_mode']:
                line = emphasize_line(line, pattern, color)
            else:
                line = emphasize_pattern(line, pattern, color)
        print(line)

def get_arguments():
    """
    Create and parse command line interface.
    """
    def _str_to_unicode(string):
        return string.decode(sys.getfilesystemencoding()) if PY2 else string

    parser = argparse.ArgumentParser(
        description=_(
            '%(prog)s program is a terminal tool that prints FILE(s), '
            'or standard input to standard output and highlights the '
            'expressions that are matched the PATTERN.'),
        epilog=_(
            'With no FILE, or when FILE is -, read standard input.'
            '  '
            'The FORMAT option must be one of: BOLD, UNDERLINE, GREY, RED, '
            'GREEN, YELLOW, BLUE, MAGENTA, CYAN or WHITE.')
    )

    arg = parser.add_argument('pattern', metavar='PATTERN')
    arg.help = _('a pattern to highlight')
    arg.type = _str_to_unicode

    arg = parser.add_argument('format', metavar='FORMAT')
    arg.help = _('a color to highlight matched expressions')

    arg = parser.add_argument('files', metavar='FILE', nargs='*', default=['-'])
    arg.help = _('search for pattern in these file(s)')
    arg.type = _str_to_unicode

    arg = parser.add_argument('-i', '--ignore-case', action='store_true')
    arg.help = _('ignore case distinctions')

    arg = parser.add_argument('-l', '--line-mode', action='store_true')
    arg.help = _('highlight entire line')

    arg = parser.add_argument('-s', '--safe-mode', action='store_true')
    arg.help = _('highlight only when stdout refers to tty')

    arg = parser.add_argument('--version', action='version')
    arg.version = '%(prog)s ' + __version__
    arg.help = _("show program's version number and exit")

    # TODO: add option to load pattern/format settings from the file

    # localization trick (originally stores in argparse module)
    _('show this help message and exit')
    _("show program's version number and exit")

    return parser.parse_args()

def validate_arguments(arguments):
    """
    Validate arguments from the command line.
    """
    if not get_ansi_color(arguments.format):
        error(2, arguments.format, _('Unknown format value'))

def main():
    # initialize localization subsystem
    localedir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'locale')
    kwargs = {'unicode': True, } if PY2 else {}
    gettext.install('em', localedir, **kwargs)

    # parse command line arguments and validate it.
    # terminate the program if error was occured.
    arguments = get_arguments()
    validate_arguments(arguments)
    patterns = {
        arguments.pattern: {
            'format': arguments.format,
            'ignore_case': arguments.ignore_case,
            'line_mode': arguments.line_mode,
        }
    }

    # iterate over the files and highlight the given patterns
    for filename in arguments.files:
        try:
            # bind variable to stdin or file
            stream = sys.stdin if filename == '-' else open(filename, 'r')

            # skip emphasizing process in safe mode iff stdout is atty
            if arguments.safe_mode and not sys.stdout.isatty():
                sys.stdout.write(stream.read())
            else:
                emphasize(stream, patterns)

            # don't clode stdin
            if filename != '-':
                stream.close()

        # since Python 3.3 `IOEror` has been merge into `OSError`, but
        # still available as alias to the last one
        except IOError as e:
            error(1, e.filename, e.strerror)

if __name__ == '__main__':
    main()
<|endoftext|>