<commit_before># -*- coding: utf-8 -*-

'''This module defines a base class for optimization techniques.'''

import climate
import collections
import numpy as np
import theano
import theano.tensor as TT
import warnings

from . import util

logging = climate.get_logger(__name__)

def build(algo, loss, params=None, inputs=None, updates=(), monitors=(),
          monitor_gradients=False):
    '''Construct an optimizer by name.

    Parameters
    ----------
    algo : str
        The name of the optimization algorithm to build.
    loss : Theano expression
        Loss function to minimize. This must be a scalar-valued expression.
    params : list of Theano variables, optional
        Symbolic variables to adjust to minimize the loss. If not given, these
        will be computed automatically by walking the computation graph.
    inputs : list of Theano variables, optional
        Symbolic variables required to compute the loss. If not given, these
        will be computed automatically by walking the computation graph.
    updates : list of update pairs, optional
        A list of pairs providing updates for the internal of the loss
        computation. Normally this is empty, but it can be provided if the loss,
        for example, requires an update to an internal random number generator.
    monitors : dict or sequence of (str, Theano expression) tuples, optional
        Additional values to monitor during optimization. These must be provided
        as either a sequence of (name, expression) tuples, or as a dictionary
        mapping string names to Theano expressions.
    monitor_gradients : bool, optional
        If True, add monitors to log the norms of the parameter gradients during
        optimization. Defaults to False.

    Returns
    -------
    optimizer : :class:`Optimizer`
        An optimizer instance.
    '''
    return Optimizer.build(algo, loss, params, inputs,
                           updates=updates, monitors=monitors,
                           monitor_gradients=monitor_gradients)

class Optimizer(util.Registrar(str('Base'), (), {})):
    '''An optimizer computes gradient updates to iteratively optimize a loss.

    Parameters
    ----------
    loss : Theano expression
        Loss function to minimize. This must be a scalar-valued expression.
    params : list of Theano variables, optional
        Symbolic variables to adjust to minimize the loss. If not given, these
        will be computed automatically by walking the computation graph.
    inputs : list of Theano variables, optional
        Symbolic variables required to compute the loss. If not given, these
        will be computed automatically by walking the computation graph.
    updates : list of update pairs, optional
        A list of pairs providing updates for the internals of the loss
        computation. Normally this is empty, but it can be provided if the loss,
        for example, requires an update to an internal random number generator.
    monitors : sequence of (str, Theano expression) tuples, optional
        Additional values to monitor during optimization. These must be provided
        as a sequence of (name, expression) tuples.
    monitor_gradients : bool, optional
        If True, add monitors to log the norms of the parameter gradients during
        optimization. Defaults to False.
    '''

    def __init__(self, loss, params=None, inputs=None, updates=(), monitors=(),
                 monitor_gradients=False):
        inputs_, params_ = util.find_inputs_and_params(loss)

        self._loss = loss
        self._params = params or params_
        self._inputs = inputs or inputs_
        self._updates = updates

        self._shapes = [p.get_value(borrow=True).shape for p in self._params]
        self._counts = [np.prod(s) for s in self._shapes]
        self._starts = np.cumsum([0] + self._counts)[:-1]
        self._dtype = self._params[0].get_value().dtype

        self._curr_iter = 0
        self._best_iter = 0
        self._best_loss = 1e100
        self._best_params = [p.get_value().copy() for p in self._params]

        self._monitor_exprs = [self._loss]
        self._monitor_names = ['loss']
        for name, monitor in monitors:
            self._monitor_names.append(name)
            self._monitor_exprs.append(monitor)
        if monitor_gradients:
            for p, g in zip(self._params, TT.grad(self._loss, self._params)):
                self._monitor_names.append('∂{}'.format(p.name))
                self._monitor_exprs.append((g * g).sum())

    def _compile(self):
        '''Compile the Theano functions for evaluating and updating our model.
        '''
        label = self.__class__.__name__.lower()
        name = label + '-eval'
        logging.info('compiling %s function', name)
        self.f_eval = theano.function(self._inputs,
                                      self._monitor_exprs,
                                      updates=self._updates,
                                      name=name)
        name = label + '-step'
        logging.info('compiling %s function', name)
        updates = list(self._updates) + list(self._get_updates())
        self.f_step = theano.function(self._inputs,
                                      self._monitor_exprs,
                                      updates=updates,
                                      name=name)

    def _get_updates(self):
        '''Get parameter update expressions for performing optimization.

        Yields
        ------
        updates : (parameter, expression) tuples
            A sequence of parameter updates to be applied during optimization.
        '''
        for param, grad in self._differentiate():
            for var, expr in self._get_updates_for(param, grad):
                if self.momentum == 0 or var != param:
                    yield var, expr
                    continue
                delta = expr - param
                vel_tm1 = util.shared_like(param, 'vel')
                vel_t = util.as_float(self.momentum) * vel_tm1 + delta
                yield vel_tm1, vel_t
                if self.nesterov:
                    # see http://arxiv.org/pdf/1212.0901v2.pdf (eq 7) and
                    # https://github.com/lisa-lab/pylearn2/pull/136#issuecomment-10381617
                    mom_sqr = util.as_float(self.momentum ** 2)
                    mom_inc = util.as_float(1 + self.momentum)
                    yield param, param + mom_sqr * vel_tm1 + mom_inc * delta
                else:
                    yield param, param + vel_t

    def _get_updates_for(self, param, grad):
        '''Generate some update pairs for the given model parameter.

        Returns
        -------
        updates : sequence of (parameter, expression) tuples
            A sequence of parameter updates to be applied during optimization.
        '''
        raise NotImplementedError

    def _differentiate(self, params=None):
        '''Return a sequence of gradients for our parameters.

        If this optimizer has been configured with a gradient norm limit, or
        with elementwise gradient clipping, this method applies the appropriate
        rescaling and clipping operations before returning the gradient.

        Parameters
        ----------
        params : list of Theano variables, optional
            Return the gradient with respect to these parameters. Defaults to
            all parameters that the optimizer knows about.

        Yields
        ------
        pairs : (param, grad) tuples
            Generates a sequence of tuples representing each of the parameters
            requested and the corresponding Theano gradient expressions.
        '''
        if params is None:
            params = self._params
        for param, grad in zip(params, TT.grad(self._loss, params)):
            if self.max_gradient_elem > 0:
                limit = util.as_float(self.max_gradient_elem)
                yield param, TT.clip(grad, -limit, limit)
            elif self.max_gradient_norm > 0:
                norm = TT.sqrt((grad * grad).sum())
                limit = util.as_float(self.max_gradient_norm)
                yield param, grad * TT.minimum(1, limit / norm)
            else:
                yield param, grad

    def set_params(self, targets=None):
        '''Set the values of the parameters to the given target values.

        Parameters
        ----------
        targets : sequence of ndarray, optional
            Arrays for setting the parameters of our model. If this is not
            provided, the current best parameters for this optimizer will be
            used.
        '''
        if not isinstance(targets, (list, tuple)):
            targets = self._best_params
        for param, target in zip(self._params, targets):
            param.set_value(target)

    def _log(self, monitors, iteration, label='', suffix=''):
        '''Log the state of the optimizer through the logging system.

        Parameters
        ----------
        monitors : OrderedDict
            A dictionary of monitor names mapped to values. These names and
            values are what is being logged.
        iteration : int
            Optimization iteration that we are logging.
        label : str, optional
            A label for the name of the optimizer creating the log line.
            Defaults to the name of the current class.
        suffix : str, optional
            A suffix to add to the end of the log line, if any.
        '''
        label = label or self.__class__.__name__
        fields = (('{}={:.6f}').format(k, v) for k, v in monitors.items())
        logging.info('%s %i %s%s', label, iteration, ' '.join(fields), suffix)

    def evaluate(self, dataset):
        '''Evaluate the current model parameters on a dataset.

        Parameters
        ----------
        dataset : :class:`Dataset <downhill.dataset.Dataset>`
            A set of data to use for evaluating the model.

        Returns
        -------
        monitors : OrderedDict
            A dictionary mapping monitor names to values. Monitors are
            quantities of interest during optimization---for example, loss
            function, accuracy, or whatever the optimization task requires.
        '''
        values = [self.f_eval(*x) for x in dataset]
        monitors = zip(self._monitor_names, np.mean(values, axis=0))
        return collections.OrderedDict(monitors)

    def _test_patience(self, monitors):
        '''Test whether our patience with optimization has elapsed.

        Parameters
        ----------
        monitors : dict
            A dictionary mapping monitor names to values. The 'loss' key from
            this dictionary will be used to evaluate optimization progress.

        Returns
        -------
        elapsed : bool
            True iff our patience has elapsed and the model is no longer
            improving.
        '''
        self._curr_iter += 1
        marker = ''
        loss = monitors['loss']
        if self._best_loss - loss > self._best_loss * self.min_improvement:
            self._best_loss = loss
            self._best_iter = self._curr_iter
            self._best_params = [p.get_value().copy() for p in self._params]
            marker = ' *'
        self._log(monitors, self._curr_iter - 1, 'validation', marker)
        return self._curr_iter - self._best_iter > self.patience

    def _prepare(self, **kwargs):
        '''Set up properties for optimization.

        This method can be overridden by base classes to provide parameters that
        are specific to a particular optimization technique (e.g., setting up a
        learning rate value).
        '''
        pass

    def iterate(self,
                train,
                valid=None,
                patience=5,
                validate_every=10,
                min_improvement=0,
                max_gradient_norm=0,
                max_gradient_elem=0,
                learning_rate=1e-4,
                momentum=0,
                nesterov=False,
                **kwargs):
        r'''Optimize a loss iteratively using a training and validation dataset.

        This method yields a series of monitor values to the caller. After every
        optimization epoch, a pair of monitor dictionaries is generated: one
        evaluated on the training dataset during the epoch, and another
        evaluated on the validation dataset at the most recent validation epoch.

        The validation monitors might not be updated during every optimization
        iteration; in this case, the most recent validation monitors will be
        yielded along with the training monitors.

        Parameters
        ----------
        train : sequence or :class:`Dataset <downhill.dataset.Dataset>`
            A set of training data for computing updates to model parameters.
        valid : sequence or :class:`Dataset <downhill.dataset.Dataset>`, optional
            A set of validation data for computing monitor values and
            determining when the loss has stopped improving. Defaults to the
            training data.
        patience : int, optional
            Number of validation "failures" that we are willing to tolerate
            before stopping the optimization process. A validation failure
            happens whenever the loss on the validation dataset decreases by
            less than ``min_improvement`` (relative) over the previous best
            validation loss. Defaults to 5.
        validate_every : int, optional
            Evaluate the loss on the validation dataset after making this many
            passes over the training data. Defaults to 10.
        min_improvement : float, optional
            Insist that the validation loss must improve by this relative amount
            before considering that the optimization has made progress. The
            optimization process halts when ``patience`` validations have failed
            to make this relative improvement. Defaults to 0; set to a larger
            value (e.g., 0.01 for 1% improvement) to halt the optimization
            process sooner.
        max_gradient_norm : float, optional
            Rescale each parameter's gradient so that it has at most this L2
            norm. Set to 0 (the default) to disable norm rescaling. If
            ``max_gradient_elem`` is also specified, then this has no effect.
        max_gradient_elem : float, optional
            Perform elementwise clipping on the magnitude of gradient values.
            Set to 0 (the default) to disable. If elementwise clipping is
            enabled, norm rescaling (via ``max_gradient_norm``) will have no
            effect. Deprecated synonyms of this parameter are
            "max_gradient_clip" and "gradient_clip".
        learning_rate : float, optional
            Many SGD-based optimization algorithms require a learning rate
            hyperparameter that scales the gradient step. Defaults to 1e-4.
        momentum : float, optional
            Apply momentum to the parameter updates for this optimizer, with the
            given strength. Typically this value ranges from 0 (no momentum) to
            :math:`1 - \epsilon` (large momentum). Defaults to 0.
        nesterov : bool, optional
            If True, and ``momentum`` is nonzero, apply Nesterov-style momentum
            to parameter updates for this optimizer. If False, and ``momentum``
            is nonzero, "regular" momentum is applied. Has no effect if
            ``momentum`` is zero. See :class:`NAG <downhill.NAG>` for a
            description of Nesterov momentum.

        Yields
        ------
        train_monitors : dict
            A dictionary mapping monitor names to values, evaluated on the
            training dataset.
        valid_monitors : dict
            A dictionary containing monitor values evaluated on the validation
            dataset.
        '''
        self.patience = patience
        self.validate_every = validate_every
        self.min_improvement = min_improvement
        self.max_gradient_norm = max_gradient_norm
        self.max_gradient_elem = max_gradient_elem
        if 'max_gradient_clip' in kwargs:
            warnings.warn('Use "max_gradient_elem" instead of "max_gradient_clip"',
                          DeprecationWarning)
            self.max_gradient_elem = kwargs.pop('max_gradient_clip')
        if 'gradient_clip' in kwargs:
            warnings.warn('Use "max_gradient_elem" instead of "gradient_clip"',
                          DeprecationWarning)
            self.max_gradient_elem = kwargs.pop('gradient_clip')
        self.learning_rate = util.as_float(learning_rate)
        self.momentum = momentum
        self.nesterov = nesterov
        logging.info('-- patience = %s', patience)
        logging.info('-- validate_every = %s', validate_every)
        logging.info('-- min_improvement = %s', min_improvement)
        logging.info('-- max_gradient_norm = %s', max_gradient_norm)
        logging.info('-- max_gradient_elem = %s', max_gradient_elem)
        logging.info('-- learning_rate = %s', learning_rate)
        logging.info('-- momentum = %s', momentum)
        logging.info('-- nesterov = %s', nesterov)

        self._prepare(**kwargs)
        self._compile()

        if valid is None:
            valid = train
        iteration = 0
        training = validation = None
        while True:
            if not iteration % self.validate_every:
                try:
                    validation = self.evaluate(valid)
                except KeyboardInterrupt:
                    logging.info('interrupted!')
                    break
                if self._test_patience(validation):
                    logging.info('patience elapsed!')
                    break
            try:
                training = self._step(train)
            except KeyboardInterrupt:
                logging.info('interrupted!')
                break
            iteration += 1
            self._log(training, iteration)
            yield training, validation
        self.set_params('best')

    def minimize(self, *args, **kwargs):
        '''Optimize our loss exhaustively.

        This method is a thin wrapper over the :func:`iterate` method. It simply
        exhausts the iterative optimization process and returns the final
        monitor values.

        Returns
        -------
        train_monitors : dict
            A dictionary mapping monitor names to values, evaluated on the
            training dataset.
        valid_monitors : dict
            A dictionary containing monitor values evaluated on the validation
            dataset.
        '''
        monitors = None
        for monitors in self.iterate(*args, **kwargs):
            pass
        return monitors

    def _step(self, dataset):
        '''Advance the state of the optimizer by one step.

        Parameters
        ----------
        dataset : :class:`Dataset <downhill.dataset.Dataset>`
            A dataset for optimizing the model.

        Returns
        -------
        train_monitors : dict
            A dictionary mapping monitor names to values.
        '''
        values = [self.f_step(*x) for x in dataset]
        return collections.OrderedDict(
            zip(self._monitor_names, np.mean(values, axis=0)))
<commit_msg>Use "evaluation" and (class) for function names.<commit_after># -*- coding: utf-8 -*-

'''This module defines a base class for optimization techniques.'''

import climate
import collections
import numpy as np
import theano
import theano.tensor as TT
import warnings

from . import util

logging = climate.get_logger(__name__)

def build(algo, loss, params=None, inputs=None, updates=(), monitors=(),
          monitor_gradients=False):
    '''Construct an optimizer by name.

    Parameters
    ----------
    algo : str
        The name of the optimization algorithm to build.
    loss : Theano expression
        Loss function to minimize. This must be a scalar-valued expression.
    params : list of Theano variables, optional
        Symbolic variables to adjust to minimize the loss. If not given, these
        will be computed automatically by walking the computation graph.
    inputs : list of Theano variables, optional
        Symbolic variables required to compute the loss. If not given, these
        will be computed automatically by walking the computation graph.
    updates : list of update pairs, optional
        A list of pairs providing updates for the internal of the loss
        computation. Normally this is empty, but it can be provided if the loss,
        for example, requires an update to an internal random number generator.
    monitors : dict or sequence of (str, Theano expression) tuples, optional
        Additional values to monitor during optimization. These must be provided
        as either a sequence of (name, expression) tuples, or as a dictionary
        mapping string names to Theano expressions.
    monitor_gradients : bool, optional
        If True, add monitors to log the norms of the parameter gradients during
        optimization. Defaults to False.

    Returns
    -------
    optimizer : :class:`Optimizer`
        An optimizer instance.
    '''
    return Optimizer.build(algo, loss, params, inputs,
                           updates=updates, monitors=monitors,
                           monitor_gradients=monitor_gradients)

class Optimizer(util.Registrar(str('Base'), (), {})):
    '''An optimizer computes gradient updates to iteratively optimize a loss.

    Parameters
    ----------
    loss : Theano expression
        Loss function to minimize. This must be a scalar-valued expression.
    params : list of Theano variables, optional
        Symbolic variables to adjust to minimize the loss. If not given, these
        will be computed automatically by walking the computation graph.
    inputs : list of Theano variables, optional
        Symbolic variables required to compute the loss. If not given, these
        will be computed automatically by walking the computation graph.
    updates : list of update pairs, optional
        A list of pairs providing updates for the internals of the loss
        computation. Normally this is empty, but it can be provided if the loss,
        for example, requires an update to an internal random number generator.
    monitors : sequence of (str, Theano expression) tuples, optional
        Additional values to monitor during optimization. These must be provided
        as a sequence of (name, expression) tuples.
    monitor_gradients : bool, optional
        If True, add monitors to log the norms of the parameter gradients during
        optimization. Defaults to False.
    '''

    def __init__(self, loss, params=None, inputs=None, updates=(), monitors=(),
                 monitor_gradients=False):
        inputs_, params_ = util.find_inputs_and_params(loss)

        self._loss = loss
        self._params = params or params_
        self._inputs = inputs or inputs_
        self._updates = updates

        self._shapes = [p.get_value(borrow=True).shape for p in self._params]
        self._counts = [np.prod(s) for s in self._shapes]
        self._starts = np.cumsum([0] + self._counts)[:-1]
        self._dtype = self._params[0].get_value().dtype

        self._curr_iter = 0
        self._best_iter = 0
        self._best_loss = 1e100
        self._best_params = [p.get_value().copy() for p in self._params]

        self._monitor_exprs = [self._loss]
        self._monitor_names = ['loss']
        for name, monitor in monitors:
            self._monitor_names.append(name)
            self._monitor_exprs.append(monitor)
        if monitor_gradients:
            for p, g in zip(self._params, TT.grad(self._loss, self._params)):
                self._monitor_names.append('∂{}'.format(p.name))
                self._monitor_exprs.append((g * g).sum())

    def _compile(self):
        '''Compile the Theano functions for evaluating and updating our model.
        '''
        logging.info('compiling evaluation function')
        self.f_eval = theano.function(self._inputs,
                                      self._monitor_exprs,
                                      updates=self._updates,
                                      name='evaluation')
        label = self.__class__.__name__
        logging.info('compiling %s function', label)
        updates = list(self._updates) + list(self._get_updates())
        self.f_step = theano.function(self._inputs,
                                      self._monitor_exprs,
                                      updates=updates,
                                      name=label)

    def _get_updates(self):
        '''Get parameter update expressions for performing optimization.

        Yields
        ------
        updates : (parameter, expression) tuples
            A sequence of parameter updates to be applied during optimization.
        '''
        for param, grad in self._differentiate():
            for var, expr in self._get_updates_for(param, grad):
                if self.momentum == 0 or var != param:
                    yield var, expr
                    continue
                delta = expr - param
                vel_tm1 = util.shared_like(param, 'vel')
                vel_t = util.as_float(self.momentum) * vel_tm1 + delta
                yield vel_tm1, vel_t
                if self.nesterov:
                    # see http://arxiv.org/pdf/1212.0901v2.pdf (eq 7) and
                    # https://github.com/lisa-lab/pylearn2/pull/136#issuecomment-10381617
                    mom_sqr = util.as_float(self.momentum ** 2)
                    mom_inc = util.as_float(1 + self.momentum)
                    yield param, param + mom_sqr * vel_tm1 + mom_inc * delta
                else:
                    yield param, param + vel_t

    def _get_updates_for(self, param, grad):
        '''Generate some update pairs for the given model parameter.

        Returns
        -------
        updates : sequence of (parameter, expression) tuples
            A sequence of parameter updates to be applied during optimization.
        '''
        raise NotImplementedError

    def _differentiate(self, params=None):
        '''Return a sequence of gradients for our parameters.

        If this optimizer has been configured with a gradient norm limit, or
        with elementwise gradient clipping, this method applies the appropriate
        rescaling and clipping operations before returning the gradient.

        Parameters
        ----------
        params : list of Theano variables, optional
            Return the gradient with respect to these parameters. Defaults to
            all parameters that the optimizer knows about.

        Yields
        ------
        pairs : (param, grad) tuples
            Generates a sequence of tuples representing each of the parameters
            requested and the corresponding Theano gradient expressions.
        '''
        if params is None:
            params = self._params
        for param, grad in zip(params, TT.grad(self._loss, params)):
            if self.max_gradient_elem > 0:
                limit = util.as_float(self.max_gradient_elem)
                yield param, TT.clip(grad, -limit, limit)
            elif self.max_gradient_norm > 0:
                norm = TT.sqrt((grad * grad).sum())
                limit = util.as_float(self.max_gradient_norm)
                yield param, grad * TT.minimum(1, limit / norm)
            else:
                yield param, grad

    def set_params(self, targets=None):
        '''Set the values of the parameters to the given target values.

        Parameters
        ----------
        targets : sequence of ndarray, optional
            Arrays for setting the parameters of our model. If this is not
            provided, the current best parameters for this optimizer will be
            used.
        '''
        if not isinstance(targets, (list, tuple)):
            targets = self._best_params
        for param, target in zip(self._params, targets):
            param.set_value(target)

    def _log(self, monitors, iteration, label='', suffix=''):
        '''Log the state of the optimizer through the logging system.

        Parameters
        ----------
        monitors : OrderedDict
            A dictionary of monitor names mapped to values. These names and
            values are what is being logged.
        iteration : int
            Optimization iteration that we are logging.
        label : str, optional
            A label for the name of the optimizer creating the log line.
            Defaults to the name of the current class.
        suffix : str, optional
            A suffix to add to the end of the log line, if any.
        '''
        label = label or self.__class__.__name__
        fields = (('{}={:.6f}').format(k, v) for k, v in monitors.items())
        logging.info('%s %i %s%s', label, iteration, ' '.join(fields), suffix)

    def evaluate(self, dataset):
        '''Evaluate the current model parameters on a dataset.

        Parameters
        ----------
        dataset : :class:`Dataset <downhill.dataset.Dataset>`
            A set of data to use for evaluating the model.

        Returns
        -------
        monitors : OrderedDict
            A dictionary mapping monitor names to values. Monitors are
            quantities of interest during optimization---for example, loss
            function, accuracy, or whatever the optimization task requires.
        '''
        values = [self.f_eval(*x) for x in dataset]
        monitors = zip(self._monitor_names, np.mean(values, axis=0))
        return collections.OrderedDict(monitors)

    def _test_patience(self, monitors):
        '''Test whether our patience with optimization has elapsed.

        Parameters
        ----------
        monitors : dict
            A dictionary mapping monitor names to values. The 'loss' key from
            this dictionary will be used to evaluate optimization progress.

        Returns
        -------
        elapsed : bool
            True iff our patience has elapsed and the model is no longer
            improving.
        '''
        self._curr_iter += 1
        marker = ''
        loss = monitors['loss']
        if self._best_loss - loss > self._best_loss * self.min_improvement:
            self._best_loss = loss
            self._best_iter = self._curr_iter
            self._best_params = [p.get_value().copy() for p in self._params]
            marker = ' *'
        self._log(monitors, self._curr_iter - 1, 'validation', marker)
        return self._curr_iter - self._best_iter > self.patience

    def _prepare(self, **kwargs):
        '''Set up properties for optimization.

        This method can be overridden by base classes to provide parameters that
        are specific to a particular optimization technique (e.g., setting up a
        learning rate value).
        '''
        pass

    def iterate(self,
                train,
                valid=None,
                patience=5,
                validate_every=10,
                min_improvement=0,
                max_gradient_norm=0,
                max_gradient_elem=0,
                learning_rate=1e-4,
                momentum=0,
                nesterov=False,
                **kwargs):
        r'''Optimize a loss iteratively using a training and validation dataset.

        This method yields a series of monitor values to the caller. After every
        optimization epoch, a pair of monitor dictionaries is generated: one
        evaluated on the training dataset during the epoch, and another
        evaluated on the validation dataset at the most recent validation epoch.

        The validation monitors might not be updated during every optimization
        iteration; in this case, the most recent validation monitors will be
        yielded along with the training monitors.

        Parameters
        ----------
        train : sequence or :class:`Dataset <downhill.dataset.Dataset>`
            A set of training data for computing updates to model parameters.
        valid : sequence or :class:`Dataset <downhill.dataset.Dataset>`, optional
            A set of validation data for computing monitor values and
            determining when the loss has stopped improving. Defaults to the
            training data.
        patience : int, optional
            Number of validation "failures" that we are willing to tolerate
            before stopping the optimization process. A validation failure
            happens whenever the loss on the validation dataset decreases by
            less than ``min_improvement`` (relative) over the previous best
            validation loss. Defaults to 5.
        validate_every : int, optional
            Evaluate the loss on the validation dataset after making this many
            passes over the training data. Defaults to 10.
        min_improvement : float, optional
            Insist that the validation loss must improve by this relative amount
            before considering that the optimization has made progress. The
            optimization process halts when ``patience`` validations have failed
            to make this relative improvement. Defaults to 0; set to a larger
            value (e.g., 0.01 for 1% improvement) to halt the optimization
            process sooner.
        max_gradient_norm : float, optional
            Rescale each parameter's gradient so that it has at most this L2
            norm. Set to 0 (the default) to disable norm rescaling. If
            ``max_gradient_elem`` is also specified, then this has no effect.
        max_gradient_elem : float, optional
            Perform elementwise clipping on the magnitude of gradient values.
            Set to 0 (the default) to disable. If elementwise clipping is
            enabled, norm rescaling (via ``max_gradient_norm``) will have no
            effect. Deprecated synonyms of this parameter are
            "max_gradient_clip" and "gradient_clip".
        learning_rate : float, optional
            Many SGD-based optimization algorithms require a learning rate
            hyperparameter that scales the gradient step. Defaults to 1e-4.
        momentum : float, optional
            Apply momentum to the parameter updates for this optimizer, with the
            given strength. Typically this value ranges from 0 (no momentum) to
            :math:`1 - \epsilon` (large momentum). Defaults to 0.
        nesterov : bool, optional
            If True, and ``momentum`` is nonzero, apply Nesterov-style momentum
            to parameter updates for this optimizer. If False, and ``momentum``
            is nonzero, "regular" momentum is applied. Has no effect if
            ``momentum`` is zero. See :class:`NAG <downhill.NAG>` for a
            description of Nesterov momentum.

        Yields
        ------
        train_monitors : dict
            A dictionary mapping monitor names to values, evaluated on the
            training dataset.
        valid_monitors : dict
            A dictionary containing monitor values evaluated on the validation
            dataset.
        '''
        self.patience = patience
        self.validate_every = validate_every
        self.min_improvement = min_improvement
        self.max_gradient_norm = max_gradient_norm
        self.max_gradient_elem = max_gradient_elem
        if 'max_gradient_clip' in kwargs:
            warnings.warn('Use "max_gradient_elem" instead of "max_gradient_clip"',
                          DeprecationWarning)
            self.max_gradient_elem = kwargs.pop('max_gradient_clip')
        if 'gradient_clip' in kwargs:
            warnings.warn('Use "max_gradient_elem" instead of "gradient_clip"',
                          DeprecationWarning)
            self.max_gradient_elem = kwargs.pop('gradient_clip')
        self.learning_rate = util.as_float(learning_rate)
        self.momentum = momentum
        self.nesterov = nesterov
        logging.info('-- patience = %s', patience)
        logging.info('-- validate_every = %s', validate_every)
        logging.info('-- min_improvement = %s', min_improvement)
        logging.info('-- max_gradient_norm = %s', max_gradient_norm)
        logging.info('-- max_gradient_elem = %s', max_gradient_elem)
        logging.info('-- learning_rate = %s', learning_rate)
        logging.info('-- momentum = %s', momentum)
        logging.info('-- nesterov = %s', nesterov)

        self._prepare(**kwargs)
        self._compile()

        if valid is None:
            valid = train
        iteration = 0
        training = validation = None
        while True:
            if not iteration % self.validate_every:
                try:
                    validation = self.evaluate(valid)
                except KeyboardInterrupt:
                    logging.info('interrupted!')
                    break
                if self._test_patience(validation):
                    logging.info('patience elapsed!')
                    break
            try:
                training = self._step(train)
            except KeyboardInterrupt:
                logging.info('interrupted!')
                break
            iteration += 1
            self._log(training, iteration)
            yield training, validation
        self.set_params('best')

    def minimize(self, *args, **kwargs):
        '''Optimize our loss exhaustively.

        This method is a thin wrapper over the :func:`iterate` method. It simply
        exhausts the iterative optimization process and returns the final
        monitor values.

        Returns
        -------
        train_monitors : dict
            A dictionary mapping monitor names to values, evaluated on the
            training dataset.
        valid_monitors : dict
            A dictionary containing monitor values evaluated on the validation
            dataset.
        '''
        monitors = None
        for monitors in self.iterate(*args, **kwargs):
            pass
        return monitors

    def _step(self, dataset):
        '''Advance the state of the optimizer by one step.

        Parameters
        ----------
        dataset : :class:`Dataset <downhill.dataset.Dataset>`
            A dataset for optimizing the model.

        Returns
        -------
        train_monitors : dict
            A dictionary mapping monitor names to values.
        '''
        values = [self.f_step(*x) for x in dataset]
        return collections.OrderedDict(
            zip(self._monitor_names, np.mean(values, axis=0)))
<|endoftext|>


<commit_before>"""Download data relevant to train the KittiSeg model."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import logging
import sys
import os
import subprocess

import zipfile

from six.moves import urllib
from shutil import copy2

import argparse

logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
                    level=logging.INFO,
                    stream=sys.stdout)

sys.path.insert(1, 'incl')

# Please set kitti_data_url to the download link for the Kitti DATA.
#
# You can obtain by going to this website:
# http://www.cvlibs.net/download.php?file=data_road.zip
#
# Replace 'http://kitti.is.tue.mpg.de/kitti/?????????.???' by the
# correct URL.

vgg_url = 'https://dl.dropboxusercontent.com/u/50333326/vgg16.npy'

def get_pathes():
    """
    Get location of `data_dir` and `run_dir'.

    Defaut is ./DATA and ./RUNS.
    Alternativly they can be set by the environoment variabels
    'TV_DIR_DATA' and 'TV_DIR_RUNS'.
    """

    if 'TV_DIR_DATA' in os.environ:
        data_dir = os.path.join(['hypes'], os.environ['TV_DIR_DATA'])
    else:
        data_dir = "DATA"

    if 'TV_DIR_RUNS' in os.environ:
        run_dir = os.path.join(['hypes'], os.environ['TV_DIR_DATA'])
    else:
        run_dir = "RUNS"

    return data_dir, run_dir

def download(url, dest_directory):
    filename = url.split('/')[-1]
    filepath = os.path.join(dest_directory, filename)

    logging.info("Download URL: {}".format(url))
    logging.info("Download DIR: {}".format(dest_directory))

    def _progress(count, block_size, total_size):
                prog = float(count * block_size) / float(total_size) * 100.0
                sys.stdout.write('\r>> Downloading %s %.1f%%' %
                                 (filename, prog))
                sys.stdout.flush()

    filepath, _ = urllib.request.urlretrieve(url, filepath,
                                             reporthook=_progress)
    print()
    return filepath

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--kitti_url', default='', type=str)
    args = parser.parse_args()

    kitti_data_url = args.kitti_url

    data_dir, run_dir = get_pathes()

    vgg_weights = os.path.join(data_dir, 'vgg16.npy')

    if not os.path.exists(data_dir):
        os.makedirs(data_dir)

    kitti_dec_dir = os.path.join(data_dir, 'KittiBox')

    if not os.path.exists(kitti_dec_dir):
        os.makedirs(kitti_dec_dir)

    # Download VGG DATA
    if not os.path.exists(vgg_weights):
        download_command = "wget {} -P {}".format(vgg_url, data_dir)
        logging.info("Downloading VGG weights.")
        download(vgg_url, data_dir)
    else:
        logging.warning("File: {} exists.".format(vgg_weights))
        logging.warning("Please delete to redownload VGG weights.")

    kitti_dec_zip = os.path.join(kitti_dec_dir, 'data_object_image_2.zip')
    kitti_label_zip = os.path.join(kitti_dec_dir, "data_object_label_2.zip")

    # Download KITTI DATA
    if not os.path.exists(kitti_dec_zip):
        if kitti_data_url == '':
            logging.error("Data URL for Kitti Data not provided.")
            url = "http://www.cvlibs.net/download.php?file=data_object_image_2.zip"
            logging.error("Please visit: {}".format(url))
            logging.error("and request Kitti Download link.")
            logging.error("Rerun scipt using"
                          "'python download_data.py --kitti_url [url]'")
            exit(1)
        if not kitti_data_url[-29:] == 'kitti/data_object_image_2.zip':
            logging.error("Wrong url.")
            url = "http://www.cvlibs.net/download.php?file=data_object_image_2.zip"
            logging.error("Please visit: {}".format(url))
            logging.error("and request Kitti Download link.")
            logging.error("Rerun scipt using"
                          "'python download_data.py --kitti_url [url]'")
            exit(1)
        else:
            logging.info("Downloading Kitti Road Data.")
            download(kitti_data_url, kitti_dec_dir)

    if not os.path.exists(kitti_label_zip):
        logging.info("Downloading Kitti Label Data.")
        kitti_main = os.path.dirname(kitti_data_url)
        kitti_label_url = os.path.join(kitti_main, kitti_label_zip)
        kitti_label_url = os.path.join(kitti_main,
                                       os.path.basename(kitti_label_zip))
        download(kitti_label_url, kitti_dec_dir)

    # Extract and prepare KITTI DATA
    logging.info("Extracting kitti_road data.")
    zipfile.ZipFile(kitti_dec_zip, 'r').extractall(kitti_dec_dir)
    zipfile.ZipFile(kitti_label_zip, 'r').extractall(kitti_dec_dir)

    logging.info("Preparing kitti_road data.")

    copyfiles = ["train_2.idl", "train_3.idl", "train_4.idl",
                 "val_2.idl", "val_3.idl", "val_4.idl", "train.txt", "val.txt"]

    for file in copyfiles:
        filename = os.path.join('data', file)
        copy2(filename, kitti_dec_dir)

    logging.info("All data have been downloaded successful.")

if __name__ == '__main__':
    main()
<commit_msg>Change VGG Url to Cambridge FTP server<commit_after>"""Download data relevant to train the KittiSeg model."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import logging
import sys
import os
import subprocess

import zipfile

from six.moves import urllib
from shutil import copy2

import argparse

logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
                    level=logging.INFO,
                    stream=sys.stdout)

sys.path.insert(1, 'incl')

# Please set kitti_data_url to the download link for the Kitti DATA.
#
# You can obtain by going to this website:
# http://www.cvlibs.net/download.php?file=data_road.zip
#
# Replace 'http://kitti.is.tue.mpg.de/kitti/?????????.???' by the
# correct URL.

vgg_url = 'ftp://mi.eng.cam.ac.uk/pub/mttt2/models/vgg16.npy'

def get_pathes():
    """
    Get location of `data_dir` and `run_dir'.

    Defaut is ./DATA and ./RUNS.
    Alternativly they can be set by the environoment variabels
    'TV_DIR_DATA' and 'TV_DIR_RUNS'.
    """

    if 'TV_DIR_DATA' in os.environ:
        data_dir = os.path.join(['hypes'], os.environ['TV_DIR_DATA'])
    else:
        data_dir = "DATA"

    if 'TV_DIR_RUNS' in os.environ:
        run_dir = os.path.join(['hypes'], os.environ['TV_DIR_DATA'])
    else:
        run_dir = "RUNS"

    return data_dir, run_dir

def download(url, dest_directory):
    filename = url.split('/')[-1]
    filepath = os.path.join(dest_directory, filename)

    logging.info("Download URL: {}".format(url))
    logging.info("Download DIR: {}".format(dest_directory))

    def _progress(count, block_size, total_size):
                prog = float(count * block_size) / float(total_size) * 100.0
                sys.stdout.write('\r>> Downloading %s %.1f%%' %
                                 (filename, prog))
                sys.stdout.flush()

    filepath, _ = urllib.request.urlretrieve(url, filepath,
                                             reporthook=_progress)
    print()
    return filepath

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--kitti_url', default='', type=str)
    args = parser.parse_args()

    kitti_data_url = args.kitti_url

    data_dir, run_dir = get_pathes()

    vgg_weights = os.path.join(data_dir, 'vgg16.npy')

    if not os.path.exists(data_dir):
        os.makedirs(data_dir)

    kitti_dec_dir = os.path.join(data_dir, 'KittiBox')

    if not os.path.exists(kitti_dec_dir):
        os.makedirs(kitti_dec_dir)

    # Download VGG DATA
    if not os.path.exists(vgg_weights):
        download_command = "wget {} -P {}".format(vgg_url, data_dir)
        logging.info("Downloading VGG weights.")
        download(vgg_url, data_dir)
    else:
        logging.warning("File: {} exists.".format(vgg_weights))
        logging.warning("Please delete to redownload VGG weights.")

    kitti_dec_zip = os.path.join(kitti_dec_dir, 'data_object_image_2.zip')
    kitti_label_zip = os.path.join(kitti_dec_dir, "data_object_label_2.zip")

    # Download KITTI DATA
    if not os.path.exists(kitti_dec_zip):
        if kitti_data_url == '':
            logging.error("Data URL for Kitti Data not provided.")
            url = "http://www.cvlibs.net/download.php?file=data_object_image_2.zip"
            logging.error("Please visit: {}".format(url))
            logging.error("and request Kitti Download link.")
            logging.error("Rerun scipt using"
                          "'python download_data.py --kitti_url [url]'")
            exit(1)
        if not kitti_data_url[-29:] == 'kitti/data_object_image_2.zip':
            logging.error("Wrong url.")
            url = "http://www.cvlibs.net/download.php?file=data_object_image_2.zip"
            logging.error("Please visit: {}".format(url))
            logging.error("and request Kitti Download link.")
            logging.error("Rerun scipt using"
                          "'python download_data.py --kitti_url [url]'")
            exit(1)
        else:
            logging.info("Downloading Kitti Road Data.")
            download(kitti_data_url, kitti_dec_dir)

    if not os.path.exists(kitti_label_zip):
        logging.info("Downloading Kitti Label Data.")
        kitti_main = os.path.dirname(kitti_data_url)
        kitti_label_url = os.path.join(kitti_main, kitti_label_zip)
        kitti_label_url = os.path.join(kitti_main,
                                       os.path.basename(kitti_label_zip))
        download(kitti_label_url, kitti_dec_dir)

    # Extract and prepare KITTI DATA
    logging.info("Extracting kitti_road data.")
    zipfile.ZipFile(kitti_dec_zip, 'r').extractall(kitti_dec_dir)
    zipfile.ZipFile(kitti_label_zip, 'r').extractall(kitti_dec_dir)

    logging.info("Preparing kitti_road data.")

    copyfiles = ["train_2.idl", "train_3.idl", "train_4.idl",
                 "val_2.idl", "val_3.idl", "val_4.idl", "train.txt", "val.txt"]

    for file in copyfiles:
        filename = os.path.join('data', file)
        copy2(filename, kitti_dec_dir)

    logging.info("All data have been downloaded successful.")

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright (C) 2015 Dmitry Rodionov
# This file is part of my GSoC'15 project for Cuckoo Sandbox:
#	http://www.cuckoosandbox.org
# This software may be modified and distributed under the terms
# of the MIT license. See the LICENSE file for details.

from csv import reader
from sys import argv
from collections import namedtuple
from subprocess import check_output, STDOUT

syscall = namedtuple("syscall", "name args result errno")

def dtruss(target, syscall=None):
	"""Returns a list of syscalls made by a target.

	Note: dtruss is unable to deal with spaces in paths (even when escaped), so
	the |target| path must not contain spaces.

	Every syscall is a named tuple with the following properties:
	name (string), args (list of strings), result (int), errno (int).
	"""
	if syscall is None:
		cmd = ["sudo", "/usr/bin/dtruss", target]
	else:
		cmd = ["sudo", "/usr/bin/dtruss", "-t", syscall, target]

	output = check_output(cmd, stderr=STDOUT).splitlines()

	# We're only interested in dtruss' output, not the target's: remove anything
	# before the dtruss header
	dtruss_header_idx = output.index("SYSCALL(args) \t\t = return")
	del output[:dtruss_header_idx+1]

	return _parse_dtruss_output(output)

#
# Parsing implementation details
#
# dtruss' output format:
# SYSCALL(arg0, arg1, arg2, ..) 		 = result, errno
#

def _parse_dtruss_output(lines):
	"""Parses dtruss' output into separate syscalls tuples
	"""
	# Remove empty lines first
	lines = filter(None, lines)
	results = map(_parse_syscall, lines)
	return results

def _parse_syscall(string):
	name   = _syscall_name_from_dtruss_output(string)
	args   = _syscall_args_from_dtruss_output(string)
	# Result and errno are either decimal or hex numbers
	result = int(_syscall_result_from_dtruss_output(string), 0)
	errno  = int(_syscall_errno_from_dtruss_output(string), 0)

	return syscall(name=name, args=args, result=result, errno=errno)

def _syscall_name_from_dtruss_output(output_line):
	length = output_line.index('(')
	return output_line[:length]

def _syscall_args_from_dtruss_output(output_line):
	args_string = output_line[output_line.index('(')+1 : output_line.rfind(")")]
	args_string = args_string.replace('\0', '')
	# --HACK--
	# I noticed that a syscall arguments look like a CSV string,
	# so let's just use the built-in csv module for parsing them.
	# Why not just split(',') it? Because some arguments (strings)
	# may contain commas as well -- and I don't want to deal with this.
	#
	# But! csv won't handle fields with commas inside them without
	# skipinitialspace set to True
	parsed_rows = list(reader([args_string], skipinitialspace=True))
	# We have only one row here
	args = parsed_rows[0]
	# Remove trailing zeros from strings
	return list(x.replace('\0', '') for x in args)

def _syscall_result_from_dtruss_output(output_line):
	result_errno_tag = "\t\t = "
	from_idx = output_line.rfind(result_errno_tag) + len(result_errno_tag)
	tail = output_line[from_idx :].strip()
	return tail.split(' ')[0]

def _syscall_errno_from_dtruss_output(output_line):
	result_errno_tag = "\t\t = "
	from_idx = output_line.rfind(result_errno_tag) + len(result_errno_tag)
	tail = output_line[from_idx :].strip()
	errno = tail.split(' ')[1]
	# Also remove 'Err#' prefix from errno if any
	if errno.startswith('Err#'):
		errno = errno[len('Err#'):]
	return errno

if __name__ == "__main__":
	if len(argv) < 2:
		print "Usage: %s <target> [syscall]" % argv[0]
		exit(0)

	target = argv[1]
	optional_probe = argv[2] if len(argv) > 2 else None

	for syscall in dtruss(target, optional_probe):
		print "%s(%s) -> %#x %s" % (
			syscall.name,
			", ".join(syscall.args) if len(syscall.args) > 0 else "",
			syscall.result,
			"(errno = %s)" % syscall.errno if syscall.errno != 0 else ""
		)
<commit_msg>dtruss.py: everything's better with function composition<commit_after>#!/usr/bin/env python
# Copyright (C) 2015 Dmitry Rodionov
# This file is part of my GSoC'15 project for Cuckoo Sandbox:
#	http://www.cuckoosandbox.org
# This software may be modified and distributed under the terms
# of the MIT license. See the LICENSE file for details.

from csv import reader
from sys import argv
from collections import namedtuple
from subprocess import check_output, STDOUT

syscall = namedtuple("syscall", "name args result errno")

def dtruss(target, syscall=None):
	"""Returns a list of syscalls made by a target.

	Note: dtruss is unable to deal with spaces in paths (even when escaped), so
	the |target| path must not contain spaces.

	Every syscall is a named tuple with the following properties:
	name (string), args (list of strings), result (int), errno (int).
	"""
	if syscall is None:
		cmd = ["sudo", "/usr/bin/dtruss", target]
	else:
		cmd = ["sudo", "/usr/bin/dtruss", "-t", syscall, target]

	output = check_output(cmd, stderr=STDOUT).splitlines()

	# We're only interested in dtruss' output, not the target's: remove anything
	# before the dtruss header
	dtruss_header_idx = output.index("SYSCALL(args) \t\t = return")
	del output[:dtruss_header_idx+1]

	return _parse_dtruss_output(output)

#
# Parsing implementation details
#
# dtruss' output format:
# SYSCALL(arg0, arg1, arg2, ..) 		 = result, errno
#

def _parse_dtruss_output(lines):
	"""Turns non-empty strings from dtruss output into syscall tuples
	"""
	return map(_parse_syscall, filter(None, lines))

def _parse_syscall(string):
	name   = _syscall_name_from_dtruss_output(string)
	args   = _syscall_args_from_dtruss_output(string)
	# Result and errno are either decimal or hex numbers
	result = int(_syscall_result_from_dtruss_output(string), 0)
	errno  = int(_syscall_errno_from_dtruss_output(string), 0)

	return syscall(name=name, args=args, result=result, errno=errno)

def _syscall_name_from_dtruss_output(output_line):
	length = output_line.index('(')
	return output_line[:length]

def _syscall_args_from_dtruss_output(output_line):
	args_string = output_line[output_line.index('(')+1 : output_line.rfind(")")]
	args_string = args_string.replace('\0', '')
	# --HACK--
	# I noticed that a syscall arguments look like a CSV string,
	# so let's just use the built-in csv module for parsing them.
	# Why not just split(',') it? Because some arguments (strings)
	# may contain commas as well -- and I don't want to deal with this.
	#
	# But! csv won't handle fields with commas inside them without
	# skipinitialspace set to True
	parsed_rows = list(reader([args_string], skipinitialspace=True))
	# We have only one row here
	args = parsed_rows[0]
	# Remove trailing zeros from strings
	return list(x.replace('\0', '') for x in args)

def _syscall_result_from_dtruss_output(output_line):
	result_errno_tag = "\t\t = "
	from_idx = output_line.rfind(result_errno_tag) + len(result_errno_tag)
	tail = output_line[from_idx :].strip()
	return tail.split(' ')[0]

def _syscall_errno_from_dtruss_output(output_line):
	result_errno_tag = "\t\t = "
	from_idx = output_line.rfind(result_errno_tag) + len(result_errno_tag)
	tail = output_line[from_idx :].strip()
	errno = tail.split(' ')[1]
	# Also remove 'Err#' prefix from errno if any
	if errno.startswith('Err#'):
		errno = errno[len('Err#'):]
	return errno

if __name__ == "__main__":
	if len(argv) < 2:
		print "Usage: %s <target> [syscall]" % argv[0]
		exit(0)

	target = argv[1]
	optional_probe = argv[2] if len(argv) > 2 else None

	for syscall in dtruss(target, optional_probe):
		print "%s(%s) -> %#x %s" % (
			syscall.name,
			", ".join(syscall.args) if len(syscall.args) > 0 else "",
			syscall.result,
			"(errno = %s)" % syscall.errno if syscall.errno != 0 else ""
		)
<|endoftext|>


<commit_before># Copyright 2021 The Duet Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

__version__ = "0.2.7.dev"
<commit_msg>Bump version to 0.2.7 (#64)<commit_after># Copyright 2021 The Duet Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

__version__ = "0.2.7"
<|endoftext|>


<commit_before>"""
Mapper and reducer implementations for Direct TSQR.

Use the script run_dirtsqr.py to run Direct TSQR.

Austin R. Benson
David F. Gleich
Copyright (c) 2012-2014
"""

import sys
import time
import struct
import uuid
import cPickle as pickle
import gc

import numpy
import numpy.linalg

import util
import mrmc

import dumbo
import dumbo.backends.common
from dumbo import opt

@opt("getpath", "yes")
class DirTSQRMap1(mrmc.MatrixHandler):
    """
    Input: <key, value> pairs representing <row id, row> in the matrix A

    Output:
    1. R matrix: <mapper id, row>
    2. Q matrix: <mapper id, row + [row_id]>
    """

    def __init__(self):
        mrmc.MatrixHandler.__init__(self)
        self.keys = []
        self.data = []
        self.mapper_id = uuid.uuid1().hex

    def collect(self,key,value):
        if self.ncols == None:
            self.ncols = len(value)
            print >>sys.stderr, "Matrix size: %i columns"%(self.ncols)
        else:
            assert(len(value) == self.ncols)

        self.keys.append(key)
        self.data.append(value)
        self.nrows += 1

        # write status updates so Hadoop doesn't complain
        if self.nrows%50000 == 0:
            self.counters['rows processed'] += 50000

    def close(self):
        self.counters['rows processed'] += self.nrows % 50000

        # if no data was passed to this task, we just return
        if len(self.data) == 0:
            return

        QR = numpy.linalg.qr(numpy.array(self.data))

        yield ("R_%s" % str(self.mapper_id), self.mapper_id), QR[1].tolist()

        flat_Q = [entry for row in QR[0] for entry in row]
        val1 = pickle.dumps(self.keys)
        val2 = struct.pack('d'*len(flat_Q), *flat_Q)
        val = ''.join([str(len(val1)) + '_', val1, val2])
        yield ("Q_%s" % str(self.mapper_id), self.mapper_id), val

    def __call__(self,data):
        self.collect_data(data)
        for key,val in self.close():
            yield key, val

@opt("getpath", "yes")
class DirTSQRRed2(dumbo.backends.common.MapRedBase):
    """
    Takes all of the intermediate Rs

    Computes [R_1, ..., R_n] = Q2R_{final}

    Output:
    1. R_final: R in A = QR with key-value pairs <i, row>
    2. Q2: <mapper_id, row>

    where Q2 is a list of key value pairs.

    Each key corresponds to a mapperid from stage 1 and that keys value is the
    Q2 matrix corresponding to that mapper_id
    """
    def __init__(self, compute_svd=False):
        self.R_data = {}
        self.key_order = []
        self.Q2 = None
        self.compute_svd = compute_svd

    def collect(self, key, value):
        assert(key not in self.R_data)
        data = []
        for row in value:
            data.append([float(val) for val in row])
        self.R_data[key] = data

    def close_R(self):
        data = []
        for key in self.R_data:
            data += self.R_data[key]
            self.key_order.append(key)
        A = numpy.array(data)
        QR = numpy.linalg.qr(A)
        self.Q2 = QR[0].tolist()
        self.R_final = QR[1].tolist()
        for i, row in enumerate(self.R_final):
            yield ("R_final", i), row
        if self.compute_svd:
            U, S, Vt = numpy.linalg.svd(self.R_final)
            S = numpy.diag(S)
            for i, row in enumerate(U):
                yield ("U", i), row
            for i, row in enumerate(S):
                yield ("Sigma", i), row
            for i, row in enumerate(Vt):
                yield ("Vt", i), row

    def close_Q(self):
        num_rows = len(self.Q2)
        rows_to_read = num_rows / len(self.key_order)

        ind = 0
        key_ind = 0
        local_Q = []
        for row in self.Q2:
            local_Q.append(row)
            ind += 1
            if (ind == rows_to_read):
               flat_Q = [entry for row in local_Q for entry in row]
               yield ("Q2", self.key_order[key_ind]), flat_Q
               key_ind += 1
               local_Q = []
               ind = 0

    def __call__(self,data):
        for key,values in data:
                for value in values:
                    self.collect(key, value)

        for key, val in self.close_R():
            yield key, val
        for key, val in self.close_Q():
            yield key, val

class DirTSQRMap3(dumbo.backends.common.MapRedBase):
    """
    input: Q1 as <mapper_id, [row] + [row_id]>
    input: Q2 comes attached as a text file, which is then parsed on the fly

    output: Q as <row_id, row>
    """
    def __init__(self,ncols,q2path='q2.txt',upath=None):
        # TODO implement this
        self.Q1_data = {}
        self.row_keys = {}
        self.Q2_data = {}
        self.ncols = ncols
        self.q2path = q2path
        self.u_data = None
        if upath is not None:
          self.u_data = []
          for row in util.parse_matrix_txt(upath):
            self.u_data.append(row)
          self.u_data = numpy.mat(self.u_data)

    def parse_q2(self):
        try:
            f = open(self.q2path, 'r')
        except:
            # We may be expecting only the file to be distributed
            # with the script
            f = open(self.q2path.split('/')[-1], 'r')
        for line in f:
            if len(line) > 5:
                ind1 = line.find('(')
                ind2 = line.rfind(')')
                key = line[ind1+1:ind2]
                # lazy parsing: we only need the keys that we have
                if key not in self.Q1_data:
                    continue
                line = line[ind2+3:]
                line = line.lstrip('[').rstrip().rstrip(']')
                line = line.split(',')
                line = [float(v) for v in line]
                line = numpy.array(line)
                mat = numpy.reshape(line, (self.ncols, self.ncols))
                self.Q2_data[key] = mat
        f.close()

    def collect(self, key, keys, value):
        self.Q1_data[key] = (keys, value)

    def close(self):
        # parse the q2 file we were given
        self.parse_q2()
        for key in self.Q1_data:
            assert(key in self.Q2_data)
            keys, Q1 = self.Q1_data[key]
            Q2 = self.Q2_data[key]
            if self.u_data is not None:
              Q2 = Q2 * self.u_data
            Q_out = Q1 * Q2
            for i, row in enumerate(Q_out.getA()):
                yield keys[i], struct.pack('d' * len(row), *row)

    def __call__(self, data):
        for key, val in data:
            ind = val.find('_')
            val1_len = int(val[:ind])
            keys = val[ind + 1:ind + 1 + val1_len]
            matrix = val[ind +1 +val1_len:]
            keys = pickle.loads(keys)
            num_entries = len(matrix) / 8
            if num_entries % self.ncols != 0:
				raise mrmc.DataFormatException(
                'Length of value (%d) did not match number of columns (%d)' % (
						num_entries, self.ncols))
            mat = struct.unpack('d' * num_entries, matrix)
            mat = numpy.mat(mat)
            mat = numpy.reshape(mat, (num_entries / self.ncols , self.ncols))
            self.collect(key, keys, mat)

        for key, val in self.close():
            yield key, val

"""
The classes RLabeller, QGrouperMap, QGrouperReduce, and DirTSQRRed3 are
used for recursive Direct TSQR.
"""
class RLabeller(dumbo.backends.common.MapRedBase):
    def __init__(self):
        self.data = []

    def close(self):
        for pair in self.data:
            yield pair[0], pair[1]

    def __call__(self, data):
        for key, value in data:
            for i, row in enumerate(value):
                new_key = str(key) + '_' + str(i)
                row = [float(val) for val in row]
                row = struct.pack('d' * len(row), *row)
                self.data.append((new_key, row))

        for key, val in self.close():
            yield key, val

class QGrouperMap(dumbo.backends.common.MapRedBase):
    def __init__(self):
        self.data = []

    def close(self):
        for pair in self.data:
            yield pair[0], pair[1]

    def __call__(self, data):
        for key, value in data:
            new_key, num = key.split('_')
            val = pickle.dumps((value, int(num)))
            self.data.append((new_key, val))

        for key, val in self.close():
            yield key, val

class QGrouperReduce(dumbo.backends.common.MapRedBase):
    def __init__(self, ncols):
        self.ncols = ncols
        self.data = {}

    def close(self):
        for key in self.data:
            assert(None not in self.data[key])
            local_Q = self.data[key]
            flat_Q = [entry for row in local_Q for entry in row]
            val = 'Q2' + '_' + struct.pack('d' * (self.ncols ** 2), *flat_Q)
            yield key, val

    def collect(self, key, value, num):
        assert(num < self.ncols)
        if key not in self.data:
            self.data[key] = self.ncols * [None]

        row = struct.unpack('d' * self.ncols, value)
        self.data[key][num] = row

    def __call__(self, data):
        for key, values in data:
            for value in values:
                val, num = pickle.loads(value)
                self.collect(key, val, num)

        for key, val in self.close():
            yield key, val

class DirTSQRRed3(dumbo.backends.common.MapRedBase):
    def __init__(self, ncols):
        self.ncols = ncols
        self.Q1_data = None
        self.Q2_data = None

    def collect(self, key, keys, value):
        self.Q1_data = (keys, value)

    def collect_Q2(self, key, value):
        value = numpy.array(struct.unpack('d' * (self.ncols ** 2), value))
        self.Q2_data = numpy.reshape(value, (self.ncols, self.ncols))

    def flush(self):
        keys, Q1 = self.Q1_data
        Q2 = self.Q2_data
        Q_out = Q1 * Q2
        self.Q1_data = None
        self.Q2_data = None
        Q1 = None
        Q2 = None
        gc.collect()
        for i, row in enumerate(Q_out.getA()):
            yield keys[i], struct.pack('d' * len(row), *row)

    def __call__(self, data):
        for key, values in data:
            for val in values:
                ind = val.find('_')
                if val[:ind] == 'Q2':
                    mat = val[ind+1:]
                    self.collect_Q2(key, mat)
                else:
                    val1_len = int(val[:ind])
                    keys = val[ind + 1:ind + 1 + val1_len]
                    matrix = val[ind +1 +val1_len:]
                    keys = pickle.loads(keys)
                    num_entries = len(matrix) / 8
                    assert (num_entries % self.ncols == 0)
                    mat = struct.unpack('d' * num_entries, matrix)
                    mat = numpy.mat(mat)
                    mat = numpy.reshape(mat, (num_entries / self.ncols , self.ncols))
                    self.collect(key, keys, mat)

            for k, v in self.flush():
                yield k, v
<commit_msg>copyright notice<commit_after>"""
   Copyright (c) 2012-2014, Austin Benson and David Gleich
   All rights reserved.

   This file is part of MRTSQR and is under the BSD 2-Clause License,
   which can be found in the LICENSE file in the root directory, or at
   http://opensource.org/licenses/BSD-2-Clause
"""

"""
Mapper and reducer implementations for Direct TSQR.

Use the script run_dirtsqr.py to run Direct TSQR.
"""

import sys
import time
import struct
import uuid
import cPickle as pickle
import gc

import numpy
import numpy.linalg

import util
import mrmc

import dumbo
import dumbo.backends.common
from dumbo import opt

@opt("getpath", "yes")
class DirTSQRMap1(mrmc.MatrixHandler):
    """
    Input: <key, value> pairs representing <row id, row> in the matrix A

    Output:
    1. R matrix: <mapper id, row>
    2. Q matrix: <mapper id, row + [row_id]>
    """

    def __init__(self):
        mrmc.MatrixHandler.__init__(self)
        self.keys = []
        self.data = []
        self.mapper_id = uuid.uuid1().hex

    def collect(self,key,value):
        if self.ncols == None:
            self.ncols = len(value)
            print >>sys.stderr, "Matrix size: %i columns"%(self.ncols)
        else:
            assert(len(value) == self.ncols)

        self.keys.append(key)
        self.data.append(value)
        self.nrows += 1

        # write status updates so Hadoop doesn't complain
        if self.nrows%50000 == 0:
            self.counters['rows processed'] += 50000

    def close(self):
        self.counters['rows processed'] += self.nrows % 50000

        # if no data was passed to this task, we just return
        if len(self.data) == 0:
            return

        QR = numpy.linalg.qr(numpy.array(self.data))

        yield ("R_%s" % str(self.mapper_id), self.mapper_id), QR[1].tolist()

        flat_Q = [entry for row in QR[0] for entry in row]
        val1 = pickle.dumps(self.keys)
        val2 = struct.pack('d'*len(flat_Q), *flat_Q)
        val = ''.join([str(len(val1)) + '_', val1, val2])
        yield ("Q_%s" % str(self.mapper_id), self.mapper_id), val

    def __call__(self,data):
        self.collect_data(data)
        for key,val in self.close():
            yield key, val

@opt("getpath", "yes")
class DirTSQRRed2(dumbo.backends.common.MapRedBase):
    """
    Takes all of the intermediate Rs

    Computes [R_1, ..., R_n] = Q2R_{final}

    Output:
    1. R_final: R in A = QR with key-value pairs <i, row>
    2. Q2: <mapper_id, row>

    where Q2 is a list of key value pairs.

    Each key corresponds to a mapperid from stage 1 and that keys value is the
    Q2 matrix corresponding to that mapper_id
    """
    def __init__(self, compute_svd=False):
        self.R_data = {}
        self.key_order = []
        self.Q2 = None
        self.compute_svd = compute_svd

    def collect(self, key, value):
        assert(key not in self.R_data)
        data = []
        for row in value:
            data.append([float(val) for val in row])
        self.R_data[key] = data

    def close_R(self):
        data = []
        for key in self.R_data:
            data += self.R_data[key]
            self.key_order.append(key)
        A = numpy.array(data)
        QR = numpy.linalg.qr(A)
        self.Q2 = QR[0].tolist()
        self.R_final = QR[1].tolist()
        for i, row in enumerate(self.R_final):
            yield ("R_final", i), row
        if self.compute_svd:
            U, S, Vt = numpy.linalg.svd(self.R_final)
            S = numpy.diag(S)
            for i, row in enumerate(U):
                yield ("U", i), row
            for i, row in enumerate(S):
                yield ("Sigma", i), row
            for i, row in enumerate(Vt):
                yield ("Vt", i), row

    def close_Q(self):
        num_rows = len(self.Q2)
        rows_to_read = num_rows / len(self.key_order)

        ind = 0
        key_ind = 0
        local_Q = []
        for row in self.Q2:
            local_Q.append(row)
            ind += 1
            if (ind == rows_to_read):
               flat_Q = [entry for row in local_Q for entry in row]
               yield ("Q2", self.key_order[key_ind]), flat_Q
               key_ind += 1
               local_Q = []
               ind = 0

    def __call__(self,data):
        for key,values in data:
                for value in values:
                    self.collect(key, value)

        for key, val in self.close_R():
            yield key, val
        for key, val in self.close_Q():
            yield key, val

class DirTSQRMap3(dumbo.backends.common.MapRedBase):
    """
    input: Q1 as <mapper_id, [row] + [row_id]>
    input: Q2 comes attached as a text file, which is then parsed on the fly

    output: Q as <row_id, row>
    """
    def __init__(self,ncols,q2path='q2.txt',upath=None):
        # TODO implement this
        self.Q1_data = {}
        self.row_keys = {}
        self.Q2_data = {}
        self.ncols = ncols
        self.q2path = q2path
        self.u_data = None
        if upath is not None:
          self.u_data = []
          for row in util.parse_matrix_txt(upath):
            self.u_data.append(row)
          self.u_data = numpy.mat(self.u_data)

    def parse_q2(self):
        try:
            f = open(self.q2path, 'r')
        except:
            # We may be expecting only the file to be distributed
            # with the script
            f = open(self.q2path.split('/')[-1], 'r')
        for line in f:
            if len(line) > 5:
                ind1 = line.find('(')
                ind2 = line.rfind(')')
                key = line[ind1+1:ind2]
                # lazy parsing: we only need the keys that we have
                if key not in self.Q1_data:
                    continue
                line = line[ind2+3:]
                line = line.lstrip('[').rstrip().rstrip(']')
                line = line.split(',')
                line = [float(v) for v in line]
                line = numpy.array(line)
                mat = numpy.reshape(line, (self.ncols, self.ncols))
                self.Q2_data[key] = mat
        f.close()

    def collect(self, key, keys, value):
        self.Q1_data[key] = (keys, value)

    def close(self):
        # parse the q2 file we were given
        self.parse_q2()
        for key in self.Q1_data:
            assert(key in self.Q2_data)
            keys, Q1 = self.Q1_data[key]
            Q2 = self.Q2_data[key]
            if self.u_data is not None:
              Q2 = Q2 * self.u_data
            Q_out = Q1 * Q2
            for i, row in enumerate(Q_out.getA()):
                yield keys[i], struct.pack('d' * len(row), *row)

    def __call__(self, data):
        for key, val in data:
            ind = val.find('_')
            val1_len = int(val[:ind])
            keys = val[ind + 1:ind + 1 + val1_len]
            matrix = val[ind +1 +val1_len:]
            keys = pickle.loads(keys)
            num_entries = len(matrix) / 8
            if num_entries % self.ncols != 0:
				raise mrmc.DataFormatException(
                'Length of value (%d) did not match number of columns (%d)' % (
						num_entries, self.ncols))
            mat = struct.unpack('d' * num_entries, matrix)
            mat = numpy.mat(mat)
            mat = numpy.reshape(mat, (num_entries / self.ncols , self.ncols))
            self.collect(key, keys, mat)

        for key, val in self.close():
            yield key, val

"""
The classes RLabeller, QGrouperMap, QGrouperReduce, and DirTSQRRed3 are
used for recursive Direct TSQR.
"""
class RLabeller(dumbo.backends.common.MapRedBase):
    def __init__(self):
        self.data = []

    def close(self):
        for pair in self.data:
            yield pair[0], pair[1]

    def __call__(self, data):
        for key, value in data:
            for i, row in enumerate(value):
                new_key = str(key) + '_' + str(i)
                row = [float(val) for val in row]
                row = struct.pack('d' * len(row), *row)
                self.data.append((new_key, row))

        for key, val in self.close():
            yield key, val

class QGrouperMap(dumbo.backends.common.MapRedBase):
    def __init__(self):
        self.data = []

    def close(self):
        for pair in self.data:
            yield pair[0], pair[1]

    def __call__(self, data):
        for key, value in data:
            new_key, num = key.split('_')
            val = pickle.dumps((value, int(num)))
            self.data.append((new_key, val))

        for key, val in self.close():
            yield key, val

class QGrouperReduce(dumbo.backends.common.MapRedBase):
    def __init__(self, ncols):
        self.ncols = ncols
        self.data = {}

    def close(self):
        for key in self.data:
            assert(None not in self.data[key])
            local_Q = self.data[key]
            flat_Q = [entry for row in local_Q for entry in row]
            val = 'Q2' + '_' + struct.pack('d' * (self.ncols ** 2), *flat_Q)
            yield key, val

    def collect(self, key, value, num):
        assert(num < self.ncols)
        if key not in self.data:
            self.data[key] = self.ncols * [None]

        row = struct.unpack('d' * self.ncols, value)
        self.data[key][num] = row

    def __call__(self, data):
        for key, values in data:
            for value in values:
                val, num = pickle.loads(value)
                self.collect(key, val, num)

        for key, val in self.close():
            yield key, val

class DirTSQRRed3(dumbo.backends.common.MapRedBase):
    def __init__(self, ncols):
        self.ncols = ncols
        self.Q1_data = None
        self.Q2_data = None

    def collect(self, key, keys, value):
        self.Q1_data = (keys, value)

    def collect_Q2(self, key, value):
        value = numpy.array(struct.unpack('d' * (self.ncols ** 2), value))
        self.Q2_data = numpy.reshape(value, (self.ncols, self.ncols))

    def flush(self):
        keys, Q1 = self.Q1_data
        Q2 = self.Q2_data
        Q_out = Q1 * Q2
        self.Q1_data = None
        self.Q2_data = None
        Q1 = None
        Q2 = None
        gc.collect()
        for i, row in enumerate(Q_out.getA()):
            yield keys[i], struct.pack('d' * len(row), *row)

    def __call__(self, data):
        for key, values in data:
            for val in values:
                ind = val.find('_')
                if val[:ind] == 'Q2':
                    mat = val[ind+1:]
                    self.collect_Q2(key, mat)
                else:
                    val1_len = int(val[:ind])
                    keys = val[ind + 1:ind + 1 + val1_len]
                    matrix = val[ind +1 +val1_len:]
                    keys = pickle.loads(keys)
                    num_entries = len(matrix) / 8
                    assert (num_entries % self.ncols == 0)
                    mat = struct.unpack('d' * num_entries, matrix)
                    mat = numpy.mat(mat)
                    mat = numpy.reshape(mat, (num_entries / self.ncols , self.ncols))
                    self.collect(key, keys, mat)

            for k, v in self.flush():
                yield k, v
<|endoftext|>


<commit_before>from __future__ import unicode_literals, absolute_import

import sys
import os
import posixpath
from funcy import cached_property

from dvc.utils.compat import str, builtin_str, basestring, is_py2
from dvc.utils.compat import pathlib, urlparse

# On Python 2.7/Windows sys.getfilesystemencoding() is set to mbcs,
# which is lossy, thus we can't use that,
# see https://github.com/mcmtroffaes/pathlib2/issues/56.
from dvc.utils import relpath

if is_py2:
    fs_encoding = "utf-8"

class PathInfo(pathlib.PurePath):
    scheme = "local"

    def __new__(cls, *args):
        # Construct a proper subclass depending on current os
        if cls is PathInfo:
            cls = WindowsPathInfo if os.name == "nt" else PosixPathInfo
        return cls._from_parts(args)

    @classmethod
    def from_posix(cls, s):
        return cls(PosixPathInfo(s))

    def __str__(self):
        path = self.__fspath__()
        return relpath(path)

    def __repr__(self):
        return builtin_str("{}: '{}'").format(type(self).__name__, self)

    # This permits passing it to file utils directly in Python 3.6+
    # With Python 2.7, Python 3.5+ we are stuck with path_info.fspath for now
    def __fspath__(self):
        return pathlib.PurePath.__str__(self)

    @property
    def fspath(self):
        return self.__fspath__()

    url = fspath

    def relpath(self, other):
        return self.__class__(relpath(self, other))

    def isin(self, other):
        if isinstance(other, basestring):
            other = self.__class__(other)
        elif self.__class__ != other.__class__:
            return False
        return any(p == other for p in self.parents)

    # pathlib2 uses bytes internally in Python 2, and we use unicode everywhere
    # for paths in both pythons, thus we need this glue.
    if is_py2:
        __unicode__ = __str__

        def __str__(self):
            return self.__unicode__().encode(sys.getfilesystemencoding())

        @classmethod
        def _parse_args(cls, args):
            args = [
                a.encode(fs_encoding)
                if isinstance(a, unicode)  # noqa: F821
                else a
                for a in args
            ]
            return super(PathInfo, cls)._parse_args(args)

        @property
        def name(self):
            return super(PathInfo, self).name.decode(fs_encoding)

        def __fspath__(self):  # noqa: F811
            return pathlib.PurePath.__str__(self).decode(fs_encoding)

        def with_name(self, name):
            return pathlib.PurePath.with_name(self, name.encode(fs_encoding))

        def as_posix(self):
            f = self._flavour
            return str(self).replace(f.sep, "/")

class WindowsPathInfo(PathInfo, pathlib.PureWindowsPath):
    pass

class PosixPathInfo(PathInfo, pathlib.PurePosixPath):
    pass

class URLInfo(object):
    DEFAULT_PORTS = {"http": 80, "https": 443, "ssh": 22}

    def __init__(self, url):
        self.parsed = urlparse(url)
        assert self.parsed.scheme != "remote"

    @classmethod
    def from_parts(
        cls, scheme=None, netloc=None, host=None, user=None, port=None, path=""
    ):
        assert scheme and (bool(host) ^ bool(netloc))

        if netloc is None:
            netloc = host
            if user:
                netloc = user + "@" + host
            if port:
                netloc += ":" + str(port)
        return cls("{}://{}{}".format(scheme, netloc, path))

    @cached_property
    def url(self):
        p = self.parsed
        netloc = p.hostname
        if p.username:
            netloc = p.username + "@" + netloc
        if p.port and int(p.port) != self.DEFAULT_PORTS.get(p.scheme):
            netloc += ":" + str(p.port)
        return "{}://{}{}".format(p.scheme, netloc, p.path)

    def __str__(self):
        return self.url

    def __repr__(self):
        return "{}: '{}'".format(type(self).__name__, self)

    def __eq__(self, other):
        if isinstance(other, basestring):
            other = self.__class__(other)
        return str(self) == str(other)

    def __hash__(self):
        return hash(self.url)

    def __div__(self, other):
        p = self.parsed
        new_path = posixpath.join(p.path, str(other))
        if not new_path.startswith("/"):
            new_path = "/" + new_path
        new_url = "{}://{}{}".format(p.scheme, p.netloc, new_path)
        return self.__class__(new_url)

    __truediv__ = __div__

    def __getattr__(self, name):
        return getattr(self.parsed, name)

    @property
    def port(self):
        return self.parsed.port or self.DEFAULT_PORTS.get(self.parsed.scheme)

    @property
    def host(self):
        return self.parsed.hostname

    @property
    def user(self):
        return self.parsed.username

    @cached_property
    def _path(self):
        return PosixPathInfo(self.parsed.path)

    @property
    def name(self):
        return self._path.name

    @property
    def parts(self):
        return (self.scheme, self.netloc) + self._path.parts

    @property
    def bucket(self):
        return self.netloc

    @property
    def parent(self):
        return self.from_parts(
            scheme=self.scheme, netloc=self.netloc, path=self._path.parent
        )

    def relative_to(self, other):
        if isinstance(other, str):
            other = URLInfo(other)
        if self.scheme != other.scheme or self.netloc != other.netloc:
            raise ValueError(
                "'{}' does not start with '{}'".format(self, other)
            )
        return self._path.relative_to(other._path)

    def isin(self, other):
        if isinstance(other, basestring):
            other = self.__class__(other)
        elif self.__class__ != other.__class__:
            return False
        return (
            self.scheme == other.scheme
            and self.netloc == other.netloc
            and self._path.isin(other._path)
        )

class CloudURLInfo(URLInfo):
    @property
    def path(self):
        return self.parsed.path.lstrip("/")
<commit_msg>path info: optimize isin<commit_after>from __future__ import unicode_literals, absolute_import

import sys
import os
import posixpath
from funcy import cached_property

from dvc.utils.compat import str, builtin_str, basestring, is_py2
from dvc.utils.compat import pathlib, urlparse

# On Python 2.7/Windows sys.getfilesystemencoding() is set to mbcs,
# which is lossy, thus we can't use that,
# see https://github.com/mcmtroffaes/pathlib2/issues/56.
from dvc.utils import relpath

if is_py2:
    fs_encoding = "utf-8"

class PathInfo(pathlib.PurePath):
    scheme = "local"

    def __new__(cls, *args):
        # Construct a proper subclass depending on current os
        if cls is PathInfo:
            cls = WindowsPathInfo if os.name == "nt" else PosixPathInfo
        return cls._from_parts(args)

    @classmethod
    def from_posix(cls, s):
        return cls(PosixPathInfo(s))

    def __str__(self):
        path = self.__fspath__()
        return relpath(path)

    def __repr__(self):
        return builtin_str("{}: '{}'").format(type(self).__name__, self)

    # This permits passing it to file utils directly in Python 3.6+
    # With Python 2.7, Python 3.5+ we are stuck with path_info.fspath for now
    def __fspath__(self):
        return pathlib.PurePath.__str__(self)

    @property
    def fspath(self):
        return self.__fspath__()

    url = fspath

    def relpath(self, other):
        return self.__class__(relpath(self, other))

    def isin(self, other):
        if isinstance(other, basestring):
            other = self.__class__(other)
        elif self.__class__ != other.__class__:
            return False
        for p in self.parents:
            if p == other:
                return True
            if len(other.parts) >= len(p.parts):
                return False

    # pathlib2 uses bytes internally in Python 2, and we use unicode everywhere
    # for paths in both pythons, thus we need this glue.
    if is_py2:
        __unicode__ = __str__

        def __str__(self):
            return self.__unicode__().encode(sys.getfilesystemencoding())

        @classmethod
        def _parse_args(cls, args):
            args = [
                a.encode(fs_encoding)
                if isinstance(a, unicode)  # noqa: F821
                else a
                for a in args
            ]
            return super(PathInfo, cls)._parse_args(args)

        @property
        def name(self):
            return super(PathInfo, self).name.decode(fs_encoding)

        def __fspath__(self):  # noqa: F811
            return pathlib.PurePath.__str__(self).decode(fs_encoding)

        def with_name(self, name):
            return pathlib.PurePath.with_name(self, name.encode(fs_encoding))

        def as_posix(self):
            f = self._flavour
            return str(self).replace(f.sep, "/")

class WindowsPathInfo(PathInfo, pathlib.PureWindowsPath):
    pass

class PosixPathInfo(PathInfo, pathlib.PurePosixPath):
    pass

class URLInfo(object):
    DEFAULT_PORTS = {"http": 80, "https": 443, "ssh": 22}

    def __init__(self, url):
        self.parsed = urlparse(url)
        assert self.parsed.scheme != "remote"

    @classmethod
    def from_parts(
        cls, scheme=None, netloc=None, host=None, user=None, port=None, path=""
    ):
        assert scheme and (bool(host) ^ bool(netloc))

        if netloc is None:
            netloc = host
            if user:
                netloc = user + "@" + host
            if port:
                netloc += ":" + str(port)
        return cls("{}://{}{}".format(scheme, netloc, path))

    @cached_property
    def url(self):
        p = self.parsed
        netloc = p.hostname
        if p.username:
            netloc = p.username + "@" + netloc
        if p.port and int(p.port) != self.DEFAULT_PORTS.get(p.scheme):
            netloc += ":" + str(p.port)
        return "{}://{}{}".format(p.scheme, netloc, p.path)

    def __str__(self):
        return self.url

    def __repr__(self):
        return "{}: '{}'".format(type(self).__name__, self)

    def __eq__(self, other):
        if isinstance(other, basestring):
            other = self.__class__(other)
        return str(self) == str(other)

    def __hash__(self):
        return hash(self.url)

    def __div__(self, other):
        p = self.parsed
        new_path = posixpath.join(p.path, str(other))
        if not new_path.startswith("/"):
            new_path = "/" + new_path
        new_url = "{}://{}{}".format(p.scheme, p.netloc, new_path)
        return self.__class__(new_url)

    __truediv__ = __div__

    def __getattr__(self, name):
        return getattr(self.parsed, name)

    @property
    def port(self):
        return self.parsed.port or self.DEFAULT_PORTS.get(self.parsed.scheme)

    @property
    def host(self):
        return self.parsed.hostname

    @property
    def user(self):
        return self.parsed.username

    @cached_property
    def _path(self):
        return PosixPathInfo(self.parsed.path)

    @property
    def name(self):
        return self._path.name

    @property
    def parts(self):
        return (self.scheme, self.netloc) + self._path.parts

    @property
    def bucket(self):
        return self.netloc

    @property
    def parent(self):
        return self.from_parts(
            scheme=self.scheme, netloc=self.netloc, path=self._path.parent
        )

    def relative_to(self, other):
        if isinstance(other, str):
            other = URLInfo(other)
        if self.scheme != other.scheme or self.netloc != other.netloc:
            raise ValueError(
                "'{}' does not start with '{}'".format(self, other)
            )
        return self._path.relative_to(other._path)

    def isin(self, other):
        if isinstance(other, basestring):
            other = self.__class__(other)
        elif self.__class__ != other.__class__:
            return False
        return (
            self.scheme == other.scheme
            and self.netloc == other.netloc
            and self._path.isin(other._path)
        )

class CloudURLInfo(URLInfo):
    @property
    def path(self):
        return self.parsed.path.lstrip("/")
<|endoftext|>