<commit_before># -*- coding: utf-8 -*-
"""
A Theil-Sen Estimator for Multiple Linear Regression Model
"""

# Author: Florian Wilhelm <florian.wilhelm@gmail.com>
#
# License: BSD 3 clause

from __future__ import division, print_function, absolute_import

import warnings
from itertools import combinations

import numpy as np
from scipy import linalg
from scipy.special import binom
from scipy.linalg.lapack import get_lapack_funcs

from .base import LinearModel
from ..base import RegressorMixin
from ..utils import check_array, check_random_state, ConvergenceWarning
from ..utils import check_consistent_length, _get_n_jobs
from ..externals.joblib import Parallel, delayed
from ..externals.six.moves import xrange as range

def _modified_weiszfeld_step(X, x_old):
    """Modified Weiszfeld step.

    This function defines one iteration step in order to approximate the
    spatial median (L1 median). It is a form of an iteratively re-weighted
    least squares method.

    Parameters
    ----------
    X : array, shape = [n_samples, n_features]
        Training vector, where n_samples is the number of samples and
        n_features is the number of features.

    x_old : array, shape = [n_features]
        Current start vector.

    Returns
    -------
    x_new : array, shape = [n_features]
        New iteration step.

    References
    ----------
    - On Computation of Spatial Median for Robust Data Mining, 2005
      T. Kärkkäinen and S. Äyrämö
      http://users.jyu.fi/~samiayr/pdf/ayramo_eurogen05.pdf
    """
    epsilon = np.finfo(np.double).eps
    X = X.T
    diff = X.T - x_old
    diff_norm = np.sqrt(np.sum(diff ** 2, axis=1))
    mask = diff_norm >= epsilon
    if mask.sum() < X.shape[1]:  # x_old equals one of our samples
        equals_sample = 1.
    else:
        equals_sample = 0.
    diff = diff[mask, :]
    diff_norm = diff_norm[mask][:, np.newaxis]
    quotient_norm = linalg.norm(np.sum(diff / diff_norm, axis=0))
    if quotient_norm > epsilon:  # to avoid division by zero
        new_direction = (np.sum(X.T[mask, :] / diff_norm, axis=0)
                         / np.sum(1 / diff_norm, axis=0))
    else:
        new_direction = 1.
        quotient_norm = 1.
    return (max(0., 1. - equals_sample / quotient_norm) * new_direction
            + min(1., equals_sample / quotient_norm) * x_old)

def _spatial_median(X, max_iter=300, tol=1.e-3):
    """Spatial median (L1 median).

    The spatial median is member of a class of so-called M-estimators which
    are defined by an optimization problem. Given a number of p points in an
    n-dimensional space, the point x minimizing the sum of all distances to the
    p other points is called spatial median.

    Parameters
    ----------
    X : array, shape = [n_samples, n_features]
        Training vector, where n_samples is the number of samples and
        n_features is the number of features.

    max_iter : int, optional
        Maximum number of iterations.  Default is 300.

    tol : float, optional
        Stop the algorithm if spatial_median has converged. Default is 1.e-3.

    Returns
    -------
    spatial_median : array, shape = [n_features]
        Spatial median.

    n_iter: int
        Number of iterations needed.

    References
    ----------
    - On Computation of Spatial Median for Robust Data Mining, 2005
      T. Kärkkäinen and S. Äyrämö
      http://users.jyu.fi/~samiayr/pdf/ayramo_eurogen05.pdf
    """
    if X.shape[1] == 1:
        return 1, np.median(X.ravel())
    # We are computing the tol on the squared norm
    tol **= 2
    spatial_median_old = np.mean(X, axis=0)
    for n_iter in range(max_iter):
        spatial_median = _modified_weiszfeld_step(X, spatial_median_old)
        if np.sum((spatial_median_old - spatial_median) ** 2) < tol:
            break
        else:
            spatial_median_old = spatial_median
    else:
        warnings.warn("Maximum number of iterations {max_iter} reached in "
                      "spatial median for TheilSen regressor."
                      "".format(max_iter=max_iter), ConvergenceWarning)
    return n_iter, spatial_median

def _breakdown_point(n_samples, n_subsamples):
    """Approximation of the breakdown point.

    Parameters
    ----------
    n_samples : int
        Number of samples.

    n_subsamples : int
        Number of subsamples to consider.

    Returns
    -------
    breakdown_point : float
        Approximation of breakdown point.
    """
    return 1 - (0.5 ** (1 / n_subsamples) * (n_samples - n_subsamples + 1) +
                n_subsamples - 1) / n_samples

def _lstsq(X, y, indices, intercept):
    """Least Squares Estimator for TheilSenRegressor class.

    This function calculates the least squares method on a subset of rows of X
    and y defined by the indices array. Optionally, an intercept column is
    added if intercept is set to true.

    Parameters
    ----------
    X : array, shape = [n_samples, n_features]
        Design matrix, where n_samples is the number of samples and
        n_features is the number of features.

    y : array, shape = [n_samples]
        Target vector, where n_samples is the number of samples.

    indices : array, shape = [n_subpopulation, n_subsamples]
        Indices of all subsamples with respect to the chosen subpopulation.

    intercept : bool
        Fit intercept or not.

    Returns
    -------
    weights : array, shape = [n_subpopulation, n_features + intercept]
        Solution matrix of n_subpopulation solved least square problems.
    """
    first_elem = 1 if intercept else 0
    n_dim = X.shape[1] + first_elem
    n_subsamples = indices.shape[1]
    weights = np.empty((indices.shape[0], n_dim))
    X_subpopulation = np.ones((n_subsamples, n_dim))
    # gelss need to pad y_subpopulation to be of the max dim of X_subpopulation
    y_subpopulation = np.zeros((max(n_subsamples, n_dim)))
    lstsq, = get_lapack_funcs(('gelss',), (X_subpopulation, y_subpopulation))
    for iter, subset in enumerate(indices):
        X_subpopulation[:, first_elem:] = X[subset, :]
        y_subpopulation[:n_subsamples] = y[subset]
        weights[iter, :] = lstsq(X_subpopulation, y_subpopulation)[1][:n_dim]
    return weights

class TheilSenRegressor(LinearModel, RegressorMixin):
    """Theil-Sen Estimator: robust multivariate regression model.

    The algorithm calculates least square solutions on subsets with size
    n_subsamples of the samples in X. Any value of n_subsamples between the
    number of features and samples leads to an estimator with a compromise
    between robustness and efficiency. Since the number of least square
    solutions is "n_samples choose n_subsamples", it can be extremely large
    and can therefore be limited with max_subpopulation. If this limit is
    reached, the subsets are chosen randomly. In a final step, the spatial
    median (or L1 median) is calculated of all least square solutions.

    Parameters
    ----------
    fit_intercept : boolean, optional, default True
        Whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations.

    copy_X : boolean, optional, default True
        If True, X will be copied; else, it may be overwritten.

    max_subpopulation : int, optional, default 1e4
        Instead of computing with a set of cardinality 'n choose k', where n is
        the number of samples and k is the number of subsamples (at least
        number of features), consider only a stochastic subpopulation of a
        given maximal size if 'n choose k' is larger than max_subpopulation.
        For other than small problem sizes this parameter will determine
        memory usage and runtime if n_subsamples is not changed.

    n_subsamples : int, optional, default None
        Number of samples to calculate the parameters. This is at least the
        number of features (plus 1 if fit_intercept=True) and the number of
        samples as a maximum. A lower number leads to a higher breakdown
        point and a low efficiency while a high number leads to a low
        breakdown point and a high efficiency. If None, take the
        minimum number of subsamples leading to maximal robustness.
        If n_subsamples is set to n_samples, Theil-Sen is identical to least
        squares.

    max_iter : int, optional, default 300
        Maximum number of iterations for the calculation of spatial median.

    tol : float, optional, default 1.e-3
        Tolerance when calculating spatial median.

    random_state : RandomState or an int seed, optional, default None
        A random number generator instance to define the state of the
        random permutations generator.

    n_jobs : integer, optional, default 1
        Number of CPUs to use during the cross validation. If ``-1``, use
        all the CPUs.

    verbose : boolean, optional, default False
        Verbose mode when fitting the model.

    Attributes
    ----------
    `coef_` : array, shape = (n_features)
        Coefficients of the regression model (median of distribution).

    `intercept_` : float
        Estimated intercept of regression model.

    `breakdown_` : float
        Approximated breakdown point.

    `n_iter_` : int
        Number of iterations needed for the spatial median.

    References
    ----------
    - Theil-Sen Estimators in a Multiple Linear Regression Model, 2009
      Xin Dang, Hanxiang Peng, Xueqin Wang and Heping Zhang
      http://www.math.iupui.edu/~hpeng/MTSE_0908.pdf
    """

    def __init__(self, fit_intercept=True, copy_X=True,
                 max_subpopulation=1e4, n_subsamples=None, max_iter=300,
                 tol=1.e-3, random_state=None, n_jobs=1, verbose=False):
        self.fit_intercept = fit_intercept
        self.copy_X = copy_X
        self.max_subpopulation = int(max_subpopulation)
        self.n_subsamples = n_subsamples
        self.max_iter = max_iter
        self.tol = tol
        self.random_state = random_state
        self.n_jobs = n_jobs
        self.verbose = verbose

    def _check_subparams(self, n_samples, n_features):
        if self.fit_intercept:
            n_dim = n_features + 1
        else:
            n_dim = n_features
        n_subsamples = self.n_subsamples
        if n_subsamples is not None:
            if n_subsamples > n_samples:
                raise ValueError("Invalid parameter since n_subsamples > "
                                 "n_samples ({0} > {1}).".format(n_subsamples,
                                                                 n_samples))
            if n_samples >= n_features:
                if n_dim > n_subsamples:
                    plus_1 = "+1" if self.fit_intercept else ""
                    raise ValueError("Invalid parameter since n_features{0} "
                                     "> n_subsamples ({1} > {2})."
                                     "".format(plus_1, n_dim, n_samples))
            else:  # if n_samples < n_features
                if n_subsamples != n_samples:
                    raise ValueError("Invalid parameter since n_subsamples != "
                                     "n_samples ({0} != {1}) while n_samples "
                                     "< n_features.".format(n_subsamples,
                                                            n_samples))
        else:
            n_subsamples = min(n_dim, n_samples)
        if self.max_subpopulation <= 0:
            raise ValueError("Subpopulation must be strictly positive "
                             "({0} <= 0).".format(self.max_subpopulation))
        all_combinations = max(1, np.rint(binom(n_samples, n_subsamples)))
        n_subpopulation = int(min(self.max_subpopulation, all_combinations))
        return n_subsamples, n_subpopulation

    def fit(self, X, y):
        """Fit linear model.

        Parameters
        ----------
        X : numpy array of shape [n_samples, n_features]
            Training data
        y : numpy array of shape [n_samples]
            Target values

        Returns
        -------
        self : returns an instance of self.
        """
        random_state = check_random_state(self.random_state)
        X = check_array(X)
        y = check_array(y, ensure_2d=False)
        check_consistent_length(X, y)
        n_samples, n_features = X.shape
        n_subsamples, n_subpopulation = self._check_subparams(n_samples,
                                                              n_features)
        self.breakdown_ = _breakdown_point(n_samples, n_subsamples)
        if self.verbose:
            print("Breakdown point: {0}".format(self.breakdown_))
            print("Number of samples: {0}".format(n_samples))
            tol_outliers = int(self.breakdown_ * n_samples)
            print("Tolerable outliers: {0}".format(tol_outliers))
            print("Number of subpopulations: {0}".format(n_subpopulation))
        # Determine indices of subpopulation
        if np.rint(binom(n_samples, n_subsamples)) <= self.max_subpopulation:
            indices = list(combinations(range(n_samples), n_subsamples))
        else:
            indices = [random_state.randint(0, n_samples, n_subsamples)
                       for _ in range(n_subpopulation)]
        n_jobs = _get_n_jobs(self.n_jobs)
        index_list = np.array_split(indices, n_jobs)
        weights = Parallel(n_jobs=n_jobs,
                           verbose=self.verbose)(
            delayed(_lstsq)(X, y, index_list[job], self.fit_intercept)
            for job in range(n_jobs))
        weights = np.vstack(weights)
        self.n_iter_, coefs = _spatial_median(weights,
                                              max_iter=self.max_iter,
                                              tol=self.tol)
        if self.fit_intercept:
            self.intercept_ = coefs[0]
            self.coef_ = coefs[1:]
        else:
            self.intercept_ = 0.
            self.coef_ = coefs

        return self
<commit_msg>COSMIT: Moved epsilon to module level<commit_after># -*- coding: utf-8 -*-
"""
A Theil-Sen Estimator for Multiple Linear Regression Model
"""

# Author: Florian Wilhelm <florian.wilhelm@gmail.com>
#
# License: BSD 3 clause

from __future__ import division, print_function, absolute_import

import warnings
from itertools import combinations

import numpy as np
from scipy import linalg
from scipy.special import binom
from scipy.linalg.lapack import get_lapack_funcs

from .base import LinearModel
from ..base import RegressorMixin
from ..utils import check_array, check_random_state, ConvergenceWarning
from ..utils import check_consistent_length, _get_n_jobs
from ..externals.joblib import Parallel, delayed
from ..externals.six.moves import xrange as range

_epsilon = np.finfo(np.double).eps

def _modified_weiszfeld_step(X, x_old):
    """Modified Weiszfeld step.

    This function defines one iteration step in order to approximate the
    spatial median (L1 median). It is a form of an iteratively re-weighted
    least squares method.

    Parameters
    ----------
    X : array, shape = [n_samples, n_features]
        Training vector, where n_samples is the number of samples and
        n_features is the number of features.

    x_old : array, shape = [n_features]
        Current start vector.

    Returns
    -------
    x_new : array, shape = [n_features]
        New iteration step.

    References
    ----------
    - On Computation of Spatial Median for Robust Data Mining, 2005
      T. Kärkkäinen and S. Äyrämö
      http://users.jyu.fi/~samiayr/pdf/ayramo_eurogen05.pdf
    """
    X = X.T
    diff = X.T - x_old
    diff_norm = np.sqrt(np.sum(diff ** 2, axis=1))
    mask = diff_norm >= _epsilon
    if mask.sum() < X.shape[1]:  # x_old equals one of our samples
        equals_sample = 1.
    else:
        equals_sample = 0.
    diff = diff[mask, :]
    diff_norm = diff_norm[mask][:, np.newaxis]
    quotient_norm = linalg.norm(np.sum(diff / diff_norm, axis=0))
    if quotient_norm > _epsilon:  # to avoid division by zero
        new_direction = (np.sum(X.T[mask, :] / diff_norm, axis=0)
                         / np.sum(1 / diff_norm, axis=0))
    else:
        new_direction = 1.
        quotient_norm = 1.
    return (max(0., 1. - equals_sample / quotient_norm) * new_direction
            + min(1., equals_sample / quotient_norm) * x_old)

def _spatial_median(X, max_iter=300, tol=1.e-3):
    """Spatial median (L1 median).

    The spatial median is member of a class of so-called M-estimators which
    are defined by an optimization problem. Given a number of p points in an
    n-dimensional space, the point x minimizing the sum of all distances to the
    p other points is called spatial median.

    Parameters
    ----------
    X : array, shape = [n_samples, n_features]
        Training vector, where n_samples is the number of samples and
        n_features is the number of features.

    max_iter : int, optional
        Maximum number of iterations.  Default is 300.

    tol : float, optional
        Stop the algorithm if spatial_median has converged. Default is 1.e-3.

    Returns
    -------
    spatial_median : array, shape = [n_features]
        Spatial median.

    n_iter: int
        Number of iterations needed.

    References
    ----------
    - On Computation of Spatial Median for Robust Data Mining, 2005
      T. Kärkkäinen and S. Äyrämö
      http://users.jyu.fi/~samiayr/pdf/ayramo_eurogen05.pdf
    """
    if X.shape[1] == 1:
        return 1, np.median(X.ravel())
    # We are computing the tol on the squared norm
    tol **= 2
    spatial_median_old = np.mean(X, axis=0)
    for n_iter in range(max_iter):
        spatial_median = _modified_weiszfeld_step(X, spatial_median_old)
        if np.sum((spatial_median_old - spatial_median) ** 2) < tol:
            break
        else:
            spatial_median_old = spatial_median
    else:
        warnings.warn("Maximum number of iterations {max_iter} reached in "
                      "spatial median for TheilSen regressor."
                      "".format(max_iter=max_iter), ConvergenceWarning)
    return n_iter, spatial_median

def _breakdown_point(n_samples, n_subsamples):
    """Approximation of the breakdown point.

    Parameters
    ----------
    n_samples : int
        Number of samples.

    n_subsamples : int
        Number of subsamples to consider.

    Returns
    -------
    breakdown_point : float
        Approximation of breakdown point.
    """
    return 1 - (0.5 ** (1 / n_subsamples) * (n_samples - n_subsamples + 1) +
                n_subsamples - 1) / n_samples

def _lstsq(X, y, indices, intercept):
    """Least Squares Estimator for TheilSenRegressor class.

    This function calculates the least squares method on a subset of rows of X
    and y defined by the indices array. Optionally, an intercept column is
    added if intercept is set to true.

    Parameters
    ----------
    X : array, shape = [n_samples, n_features]
        Design matrix, where n_samples is the number of samples and
        n_features is the number of features.

    y : array, shape = [n_samples]
        Target vector, where n_samples is the number of samples.

    indices : array, shape = [n_subpopulation, n_subsamples]
        Indices of all subsamples with respect to the chosen subpopulation.

    intercept : bool
        Fit intercept or not.

    Returns
    -------
    weights : array, shape = [n_subpopulation, n_features + intercept]
        Solution matrix of n_subpopulation solved least square problems.
    """
    first_elem = 1 if intercept else 0
    n_dim = X.shape[1] + first_elem
    n_subsamples = indices.shape[1]
    weights = np.empty((indices.shape[0], n_dim))
    X_subpopulation = np.ones((n_subsamples, n_dim))
    # gelss need to pad y_subpopulation to be of the max dim of X_subpopulation
    y_subpopulation = np.zeros((max(n_subsamples, n_dim)))
    lstsq, = get_lapack_funcs(('gelss',), (X_subpopulation, y_subpopulation))
    for iter, subset in enumerate(indices):
        X_subpopulation[:, first_elem:] = X[subset, :]
        y_subpopulation[:n_subsamples] = y[subset]
        weights[iter, :] = lstsq(X_subpopulation, y_subpopulation)[1][:n_dim]
    return weights

class TheilSenRegressor(LinearModel, RegressorMixin):
    """Theil-Sen Estimator: robust multivariate regression model.

    The algorithm calculates least square solutions on subsets with size
    n_subsamples of the samples in X. Any value of n_subsamples between the
    number of features and samples leads to an estimator with a compromise
    between robustness and efficiency. Since the number of least square
    solutions is "n_samples choose n_subsamples", it can be extremely large
    and can therefore be limited with max_subpopulation. If this limit is
    reached, the subsets are chosen randomly. In a final step, the spatial
    median (or L1 median) is calculated of all least square solutions.

    Parameters
    ----------
    fit_intercept : boolean, optional, default True
        Whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations.

    copy_X : boolean, optional, default True
        If True, X will be copied; else, it may be overwritten.

    max_subpopulation : int, optional, default 1e4
        Instead of computing with a set of cardinality 'n choose k', where n is
        the number of samples and k is the number of subsamples (at least
        number of features), consider only a stochastic subpopulation of a
        given maximal size if 'n choose k' is larger than max_subpopulation.
        For other than small problem sizes this parameter will determine
        memory usage and runtime if n_subsamples is not changed.

    n_subsamples : int, optional, default None
        Number of samples to calculate the parameters. This is at least the
        number of features (plus 1 if fit_intercept=True) and the number of
        samples as a maximum. A lower number leads to a higher breakdown
        point and a low efficiency while a high number leads to a low
        breakdown point and a high efficiency. If None, take the
        minimum number of subsamples leading to maximal robustness.
        If n_subsamples is set to n_samples, Theil-Sen is identical to least
        squares.

    max_iter : int, optional, default 300
        Maximum number of iterations for the calculation of spatial median.

    tol : float, optional, default 1.e-3
        Tolerance when calculating spatial median.

    random_state : RandomState or an int seed, optional, default None
        A random number generator instance to define the state of the
        random permutations generator.

    n_jobs : integer, optional, default 1
        Number of CPUs to use during the cross validation. If ``-1``, use
        all the CPUs.

    verbose : boolean, optional, default False
        Verbose mode when fitting the model.

    Attributes
    ----------
    `coef_` : array, shape = (n_features)
        Coefficients of the regression model (median of distribution).

    `intercept_` : float
        Estimated intercept of regression model.

    `breakdown_` : float
        Approximated breakdown point.

    `n_iter_` : int
        Number of iterations needed for the spatial median.

    References
    ----------
    - Theil-Sen Estimators in a Multiple Linear Regression Model, 2009
      Xin Dang, Hanxiang Peng, Xueqin Wang and Heping Zhang
      http://www.math.iupui.edu/~hpeng/MTSE_0908.pdf
    """

    def __init__(self, fit_intercept=True, copy_X=True,
                 max_subpopulation=1e4, n_subsamples=None, max_iter=300,
                 tol=1.e-3, random_state=None, n_jobs=1, verbose=False):
        self.fit_intercept = fit_intercept
        self.copy_X = copy_X
        self.max_subpopulation = int(max_subpopulation)
        self.n_subsamples = n_subsamples
        self.max_iter = max_iter
        self.tol = tol
        self.random_state = random_state
        self.n_jobs = n_jobs
        self.verbose = verbose

    def _check_subparams(self, n_samples, n_features):
        if self.fit_intercept:
            n_dim = n_features + 1
        else:
            n_dim = n_features
        n_subsamples = self.n_subsamples
        if n_subsamples is not None:
            if n_subsamples > n_samples:
                raise ValueError("Invalid parameter since n_subsamples > "
                                 "n_samples ({0} > {1}).".format(n_subsamples,
                                                                 n_samples))
            if n_samples >= n_features:
                if n_dim > n_subsamples:
                    plus_1 = "+1" if self.fit_intercept else ""
                    raise ValueError("Invalid parameter since n_features{0} "
                                     "> n_subsamples ({1} > {2})."
                                     "".format(plus_1, n_dim, n_samples))
            else:  # if n_samples < n_features
                if n_subsamples != n_samples:
                    raise ValueError("Invalid parameter since n_subsamples != "
                                     "n_samples ({0} != {1}) while n_samples "
                                     "< n_features.".format(n_subsamples,
                                                            n_samples))
        else:
            n_subsamples = min(n_dim, n_samples)
        if self.max_subpopulation <= 0:
            raise ValueError("Subpopulation must be strictly positive "
                             "({0} <= 0).".format(self.max_subpopulation))
        all_combinations = max(1, np.rint(binom(n_samples, n_subsamples)))
        n_subpopulation = int(min(self.max_subpopulation, all_combinations))
        return n_subsamples, n_subpopulation

    def fit(self, X, y):
        """Fit linear model.

        Parameters
        ----------
        X : numpy array of shape [n_samples, n_features]
            Training data
        y : numpy array of shape [n_samples]
            Target values

        Returns
        -------
        self : returns an instance of self.
        """
        random_state = check_random_state(self.random_state)
        X = check_array(X)
        y = check_array(y, ensure_2d=False)
        check_consistent_length(X, y)
        n_samples, n_features = X.shape
        n_subsamples, n_subpopulation = self._check_subparams(n_samples,
                                                              n_features)
        self.breakdown_ = _breakdown_point(n_samples, n_subsamples)
        if self.verbose:
            print("Breakdown point: {0}".format(self.breakdown_))
            print("Number of samples: {0}".format(n_samples))
            tol_outliers = int(self.breakdown_ * n_samples)
            print("Tolerable outliers: {0}".format(tol_outliers))
            print("Number of subpopulations: {0}".format(n_subpopulation))
        # Determine indices of subpopulation
        if np.rint(binom(n_samples, n_subsamples)) <= self.max_subpopulation:
            indices = list(combinations(range(n_samples), n_subsamples))
        else:
            indices = [random_state.randint(0, n_samples, n_subsamples)
                       for _ in range(n_subpopulation)]
        n_jobs = _get_n_jobs(self.n_jobs)
        index_list = np.array_split(indices, n_jobs)
        weights = Parallel(n_jobs=n_jobs,
                           verbose=self.verbose)(
            delayed(_lstsq)(X, y, index_list[job], self.fit_intercept)
            for job in range(n_jobs))
        weights = np.vstack(weights)
        self.n_iter_, coefs = _spatial_median(weights,
                                              max_iter=self.max_iter,
                                              tol=self.tol)
        if self.fit_intercept:
            self.intercept_ = coefs[0]
            self.coef_ = coefs[1:]
        else:
            self.intercept_ = 0.
            self.coef_ = coefs

        return self
<|endoftext|>


<commit_before>import os
import re
import logging
import tempfile

from subprocess import check_output

from collections import defaultdict

from django.db import models
from django.core.files import File
from django.core.urlresolvers import reverse

from ckeditor_uploader.fields import RichTextUploadingField

from imagekit.models import ImageSpecField
from imagekit.processors import Thumbnail
from imagekit.processors import Transpose

from model_utils.models import TimeStampedModel

from slugify import slugify

from ..users.models import User

logger = logging.getLogger(__name__)

class Blog(TimeStampedModel):
    user = models.ForeignKey(User)
    title = models.CharField(max_length=255)
    description = models.CharField(max_length=500)
    slug = models.SlugField(max_length=50)

    def __str__(self):
        return self.title

def image_spec_thumbnail(size):
    processors = [Transpose(), Thumbnail(size, size, crop=False)]
    return ImageSpecField(source='original', processors=processors,
                          format='JPEG', options={'quality': 60})

class BlogImage(TimeStampedModel):
    user = models.ForeignKey(User)

    original = models.ImageField(
        upload_to='blogs_images/originals',
        height_field='original_height',
        width_field='original_width')
    original_height = models.PositiveIntegerField(blank=True, null=True)
    original_width = models.PositiveIntegerField(blank=True, null=True)

    img_full = ImageSpecField(source='original', processors=[Transpose()],
                              format='JPEG', options={'quality': 60})

    img_xl = image_spec_thumbnail(2200)
    img_lg = image_spec_thumbnail(1100)
    img_md = image_spec_thumbnail(768)
    img_sm = image_spec_thumbnail(500)
    img_xs = image_spec_thumbnail(300)

    sizes = [
        (None, 'img_full'),
        (2200, 'img_xl'),
        (1100, 'img_lg'),
        (768, 'img_md'),
        (500, 'img_sm'),
        (300, 'img_xs'),
    ]

    blogpost_context_key = 'image'

    def get_all_paths(self):
        paths = set()
        paths.add(self.original.name)
        for size, attr_name in self.sizes:
            paths.add(getattr(self, attr_name).name)
        return paths

    def __str__(self):
        return self.original.name

    def get_srcset(self):
        sources = []
        for size, attr_name in self.sizes:
            img = getattr(self, attr_name)
            width = self.original_width if size is None else size
            url = img.url
            sources.append(url)
            sources.append('{}w,'.format(width))
        return ' '.join(sources)

    @property
    def srcset(self):
        return self.get_srcset()

    @property
    def thumbnail_src(self):
        return self.img_xs.url

    @property
    def full_src(self):
        return self.full.url

class BlogVideo(TimeStampedModel):
    user = models.ForeignKey(User)
    original = models.FileField(upload_to='blogs_videos/')
    poster = models.ImageField(upload_to='blogs_videos/poster/', null=True, blank=True)
    poster_seconds = models.FloatField(default=1)

    poster_thumbnail = ImageSpecField(source='poster',
                                      processors=[Thumbnail(300, 300, crop=False)],
                                      format='JPEG',
                                      options={'quality': 60})

    blogpost_context_key = 'video'
    calc_poster = True

    def _create_poster(self):
        """Moved into own method to make it mockable in tests."""
        fp, tmp_path = tempfile.mkstemp(prefix='poster_', suffix='.jpg')
        original = self.original.open()
        logger.info(original)
        logger.info('original url: {}'.format(self.original.url))
        logger.info('original path: {}'.format(self.original.path))
        video_url = self.original.url
        if not video_url.startswith('http'):
            video_url = self.original.path
        poster_cmd = (
            'ffmpeg -i "{video_path}" -ss {seconds} -vframes 1'
            ' -y -f image2 {poster_path}'
        ).format(video_path=video_url, seconds=self.poster_seconds,
                 poster_path=tmp_path)
        logger.info(poster_cmd)
        check_output(poster_cmd, shell=True)
        self.poster.save(
            os.path.basename(tmp_path), File(open(tmp_path, 'rb')),
            save=False)
        os.unlink(tmp_path)
        logger.info(self.pk)
        logger.info(self.poster)

    def create_poster(self):
        if self.poster or not self.calc_poster:
            # poster is not null
            logger.info('skip creating poster')
        else:
            try:
                self._create_poster()
            except Exception as e:
                logger.info('create poster failed')

    def get_all_paths(self):
        paths = set()
        paths.add(self.original.name)
        if self.poster:
            paths.add(self.poster.name)
            try:
                if self.poster_thumbnail:
                    paths.add(self.poster_thumbnail.name)
            except FileNotFoundError as e:
                pass
        return paths

    def save(self, *args, **kwargs):
        generate_poster = kwargs.pop('poster', True)
        # need to save original first - django file handling is driving me nuts
        result = super().save(*args, **kwargs)
        if generate_poster:
            logger.info('generate video poster')
            # generate poster thumbnail by default, but make it optional
            # for recalc management command
            self.create_poster()
            # save again after adding poster
            result = super().save(*args, **kwargs)
        return result

class BlogGallery(TimeStampedModel):
    user = models.ForeignKey(User)
    images = models.ManyToManyField(BlogImage)
    blogpost_context_key = 'gallery'

    @property
    def image_ids(self):
        return set([i.pk for i in self.images.all()])

class BlogPost(TimeStampedModel):
    author = models.ForeignKey(User)
    blog = models.ForeignKey(Blog)
    title = models.CharField(max_length=255)
    published = models.BooleanField(default=False)

    content = RichTextUploadingField()
    slug = models.SlugField(max_length=50)

    images = models.ManyToManyField(BlogImage)
    videos = models.ManyToManyField(BlogVideo)
    galleries = models.ManyToManyField(BlogGallery)

    media_model_lookup = {
        'image': BlogImage,
        'video': BlogVideo,
        'gallery': BlogGallery,
    }

    def __str__(self):
        return self.title

    def get_absolute_url(self):
        params = {
            "slug": self.slug,
            "blog_slug": self.blog.slug,
        }
        return reverse("blogs:blogpost_detail", kwargs=params)

    def get_slug(self):
        return slugify(self.title)

    @property
    def media_lookup_old(self):
        lookup = defaultdict(dict)
        media = list(self.media.all().prefetch_related('content_object'))
        for item in media:
            obj = item.content_object
            lookup[obj.blogpost_context_key][obj.pk] = obj
        return lookup

    @property
    def media_lookup(self):
        return {
            "image": {i.pk: i for i in self.images.all()},
            "video": {v.pk: v for v in self.videos.all()},
            "gallery": {g.pk: g for g in self.galleries.all()}
        }

    @property
    def media_from_content(self):
        regex = re.compile('{% blog_(\w+) (\d+) %}')
        groups = regex.findall(self.content)
        media = []
        for name, pk in groups:
            media.append((name, int(pk)))
        return media

    def add_missing_media_objects(self):
        media_attr_lookup = {
            'image': self.images,
            'video': self.videos,
            'gallery': self.galleries,
        }

        media_lookup = self.media_lookup
        model_lookup = self.media_model_lookup
        for model_name, model_pk in self.media_from_content:
            try:
                model = media_lookup[model_name][model_pk]
                logger.info("found: {} {} {}".format(model_name, model_pk, model))
            except KeyError:
                media_object = model_lookup[model_name].objects.get(pk=model_pk)
                print(model_name, media_object, media_attr_lookup[model_name])
                # bm = BlogMedia.objects.create(blogpost=self, content_object=media_object)  # noqa
                media_attr_lookup[model_name].add(media_object)
                logger.info('added: {} {} {}'.format(model_name, model_pk, media_object))

    def save(self, *args, **kwargs):
        save_return = super().save(*args, **kwargs)
        self.add_missing_media_objects()
        return save_return
<commit_msg>determine video orientation and get poster image accordingly<commit_after>import os
import re
import logging
import tempfile
import subprocess

from subprocess import check_output

from collections import defaultdict

from django.db import models
from django.core.files import File
from django.core.urlresolvers import reverse

from ckeditor_uploader.fields import RichTextUploadingField

from imagekit.models import ImageSpecField
from imagekit.processors import Thumbnail
from imagekit.processors import Transpose

from model_utils.models import TimeStampedModel

from slugify import slugify

from ..users.models import User

logger = logging.getLogger(__name__)

class Blog(TimeStampedModel):
    user = models.ForeignKey(User)
    title = models.CharField(max_length=255)
    description = models.CharField(max_length=500)
    slug = models.SlugField(max_length=50)

    def __str__(self):
        return self.title

def image_spec_thumbnail(size):
    processors = [Transpose(), Thumbnail(size, size, crop=False)]
    return ImageSpecField(source='original', processors=processors,
                          format='JPEG', options={'quality': 60})

class BlogImage(TimeStampedModel):
    user = models.ForeignKey(User)

    original = models.ImageField(
        upload_to='blogs_images/originals',
        height_field='original_height',
        width_field='original_width')
    original_height = models.PositiveIntegerField(blank=True, null=True)
    original_width = models.PositiveIntegerField(blank=True, null=True)

    img_full = ImageSpecField(source='original', processors=[Transpose()],
                              format='JPEG', options={'quality': 60})

    img_xl = image_spec_thumbnail(2200)
    img_lg = image_spec_thumbnail(1100)
    img_md = image_spec_thumbnail(768)
    img_sm = image_spec_thumbnail(500)
    img_xs = image_spec_thumbnail(300)

    sizes = [
        (None, 'img_full'),
        (2200, 'img_xl'),
        (1100, 'img_lg'),
        (768, 'img_md'),
        (500, 'img_sm'),
        (300, 'img_xs'),
    ]

    blogpost_context_key = 'image'

    def get_all_paths(self):
        paths = set()
        paths.add(self.original.name)
        for size, attr_name in self.sizes:
            paths.add(getattr(self, attr_name).name)
        return paths

    def __str__(self):
        return self.original.name

    def get_srcset(self):
        sources = []
        for size, attr_name in self.sizes:
            img = getattr(self, attr_name)
            width = self.original_width if size is None else size
            url = img.url
            sources.append(url)
            sources.append('{}w,'.format(width))
        return ' '.join(sources)

    @property
    def srcset(self):
        return self.get_srcset()

    @property
    def thumbnail_src(self):
        return self.img_xs.url

    @property
    def full_src(self):
        return self.full.url

class BlogVideo(TimeStampedModel):
    user = models.ForeignKey(User)
    original = models.FileField(upload_to='blogs_videos/')
    poster = models.ImageField(upload_to='blogs_videos/poster/', null=True, blank=True)
    poster_seconds = models.FloatField(default=1)

    poster_thumbnail = ImageSpecField(source='poster',
                                      processors=[Thumbnail(300, 300, crop=False)],
                                      format='JPEG',
                                      options={'quality': 60})

    blogpost_context_key = 'video'
    calc_poster = True

    def _get_video_dimensions(self, video_url):
        ffprobe_cmd = 'ffprobe -i "{}"'.format(video_url)
        result = subprocess.check_output(ffprobe_cmd, shell=True, stderr=subprocess.STDOUT)
        lines = result.decode('utf8').split("\n")
        width, height = None, None
        for line in lines:
            if 'SAR' in line:
                data = line.split(', ')[3]
                r1, r2 = map(int, data.split(' ')[0].split('x'))
                o1, o2 = map(int, data.rstrip(']').split(' ')[-1].split(':'))
                portrait = o1 < o2
                width, height = (r2, r1) if portrait else (r1, r2)
                break
        return width, height

    def _create_poster(self):
        """Moved into own method to make it mockable in tests."""
        fp, tmp_path = tempfile.mkstemp(prefix='poster_', suffix='.jpg')
        logger.info('original url: {}'.format(self.original.url))
        logger.info('original path: {}'.format(self.original.path))
        video_url = self.original.url
        if not video_url.startswith('http'):
            video_url = self.original.path
        width, height = self._get_video_dimensions(video_url)
        poster_cmd = (
            'ffmpeg -ss {seconds} -i "{video_path}" -vframes 1'
            ' -y -f image2 -s {width}x{height} {poster_path}'
        ).format(video_path=video_url, seconds=self.poster_seconds,
                 poster_path=tmp_path, width=width, height=height)
        logger.info(poster_cmd)
        check_output(poster_cmd, shell=True)
        self.poster.save(
            os.path.basename(tmp_path), File(open(tmp_path, 'rb')),
            save=False)
        os.unlink(tmp_path)
        logger.info(self.pk)
        logger.info(self.poster)

    def create_poster(self):
        if self.poster or not self.calc_poster:
            # poster is not null
            logger.info('skip creating poster')
        else:
            try:
                self._create_poster()
            except Exception as e:
                logger.info(e)
                logger.info('create poster failed')

    def get_all_paths(self):
        paths = set()
        paths.add(self.original.name)
        if self.poster:
            paths.add(self.poster.name)
            try:
                if self.poster_thumbnail:
                    paths.add(self.poster_thumbnail.name)
            except FileNotFoundError as e:
                pass
        return paths

    def save(self, *args, **kwargs):
        generate_poster = kwargs.pop('poster', True)
        # need to save original first - django file handling is driving me nuts
        result = super().save(*args, **kwargs)
        if generate_poster:
            logger.info('generate video poster')
            # generate poster thumbnail by default, but make it optional
            # for recalc management command
            self.create_poster()
            # save again after adding poster
            result = super().save(*args, **kwargs)
        return result

class BlogGallery(TimeStampedModel):
    user = models.ForeignKey(User)
    images = models.ManyToManyField(BlogImage)
    blogpost_context_key = 'gallery'

    @property
    def image_ids(self):
        return set([i.pk for i in self.images.all()])

class BlogPost(TimeStampedModel):
    author = models.ForeignKey(User)
    blog = models.ForeignKey(Blog)
    title = models.CharField(max_length=255)
    published = models.BooleanField(default=False)

    content = RichTextUploadingField()
    slug = models.SlugField(max_length=50)

    images = models.ManyToManyField(BlogImage)
    videos = models.ManyToManyField(BlogVideo)
    galleries = models.ManyToManyField(BlogGallery)

    media_model_lookup = {
        'image': BlogImage,
        'video': BlogVideo,
        'gallery': BlogGallery,
    }

    def __str__(self):
        return self.title

    def get_absolute_url(self):
        params = {
            "slug": self.slug,
            "blog_slug": self.blog.slug,
        }
        return reverse("blogs:blogpost_detail", kwargs=params)

    def get_slug(self):
        return slugify(self.title)

    @property
    def media_lookup_old(self):
        lookup = defaultdict(dict)
        media = list(self.media.all().prefetch_related('content_object'))
        for item in media:
            obj = item.content_object
            lookup[obj.blogpost_context_key][obj.pk] = obj
        return lookup

    @property
    def media_lookup(self):
        return {
            "image": {i.pk: i for i in self.images.all()},
            "video": {v.pk: v for v in self.videos.all()},
            "gallery": {g.pk: g for g in self.galleries.all()}
        }

    @property
    def media_from_content(self):
        regex = re.compile('{% blog_(\w+) (\d+) %}')
        groups = regex.findall(self.content)
        media = []
        for name, pk in groups:
            media.append((name, int(pk)))
        return media

    def add_missing_media_objects(self):
        media_attr_lookup = {
            'image': self.images,
            'video': self.videos,
            'gallery': self.galleries,
        }

        media_lookup = self.media_lookup
        model_lookup = self.media_model_lookup
        for model_name, model_pk in self.media_from_content:
            try:
                model = media_lookup[model_name][model_pk]
                logger.info("found: {} {} {}".format(model_name, model_pk, model))
            except KeyError:
                media_object = model_lookup[model_name].objects.get(pk=model_pk)
                print(model_name, media_object, media_attr_lookup[model_name])
                # bm = BlogMedia.objects.create(blogpost=self, content_object=media_object)  # noqa
                media_attr_lookup[model_name].add(media_object)
                logger.info('added: {} {} {}'.format(model_name, model_pk, media_object))

    def save(self, *args, **kwargs):
        save_return = super().save(*args, **kwargs)
        self.add_missing_media_objects()
        return save_return
<|endoftext|>


<commit_before><commit_msg>fix imports in ensembles<commit_after>from rakeld import RakelD
from rakelo import RakelO
from fixed import FixedLabelPartitionClassifier, LabelSpacePartitioningClassifier
from partition import LabelSpacePartitioningClassifier<|endoftext|>


<commit_before>from abc import ABCMeta, abstractmethod

class _TuneDefinition(metaclass=ABCMeta):
    def __init__(self, target, domain=None):
        self.domain = domain
        self._target = target

    def in_domain(self, value):
        if self.domain is None:
            return True
        else:
            lower_bound, upper_bound = self.domain
            return ((lower_bound is None or lower_bound <= value)
                    and (upper_bound is None or value <= upper_bound))

    def wrap_into_domain(self, value):
        if self._domain is None:
            return value
        else:
            lower_bound, upper_bound = self.domain
            if lower_bound is not None and value < lower_bound:
                return lower_bound
            elif upper_bound is not None and value > upper_bound:
                return upper_bound
            else:
                return value

    @property
    def x(self):
        return self._get_x()

    @property
    def max_x(self):
        if self.domain is None:
            return None
        else:
            return self.domain[1]

    @property
    def min_x(self):
        if self.domain is None:
            return None
        else:
            return self.domain[0]

    @x.setter
    def x(self, value):
        return self._set_x(self.wrap_into_domain(value))

    @property
    def y(self):
        return self._get_y()

    @property
    def target(self):
        return self._get_target()

    @target.setter
    def target(self, value):
        self._set_target(value)

    @abstractmethod
    def _get_x(self):
        pass

    @abstractmethod
    def _set_x(self):
        pass

    @abstractmethod
    def _get_y(self):
        pass

    def _get_target(self):
        return self._target

    def _set_target(self, value):
        self._target = value

    @property
    def domain(self):
        if self._domain is not None:
            return tuple(self._domain)
        else:
            return None

    @domain.setter
    def domain(self, value):
        if value is not None and not len(value) == 2:
            raise ValueError("domain must be a sequence of length two.")
        self._domain = value

    def __hash__(self):
        raise NotImplementedError("This object is not hashable.")

    def __eq__(self, other):
        raise NotImplementedError("This object is not equatable.")

class ManualTuneDefinition(metaclass=ABCMeta):
    def __init__(self, get_y, target_y, get_x, set_x, domain=None):
        self.__get_x = get_x
        self.__set_x = set_x
        self.__get_y = get_y
        self._target = target_y
        if domain is not None and not len(domain) == 2:
            raise ValueError("domain must be a sequence of length two.")
        self._domain = domain

    def _get_x(self):
        return self.__get_x()

    def _set_x(self, value):
        return self.__set_x(value)

    def _get_y(self):
        return self.__get_y()

    def _get_target(self):
        return self._target

    def _set_target(self, value):
        self._target = value

    def __hash__(self):
        return hash((self.__get_x, self.__set_x, self.__get_y, self._target))

    def __eq__(self, other):
        return (self.__get_x == other.__get_x
                and self.__set_x == other.__set_x
                and self.__get_y == other.__get_y
                and self._target == other._target)

class Solver(metaclass=ABCMeta):
    @abstractmethod
    def _solve_one(self, tunable):
        pass

    def solve(self, tunables):
        for tunable in tunables:
            self._solve_one(tunable)

    def tuned(self, tunables, tol):
        return all(abs(t.target - t.y) <= tol for t in tunables)

class ScaleSolver(Solver):
    def __init__(self, max_scale=2.0, gamma=2.0):
        self.max_scale = max_scale
        self.gamma = gamma

    def _solve_one(self, tunable):
        x, y, target = tunable.x, tunable.y, tunable.target
        if y > 0:
            scale = ((1.0 + self.gamma) * y) / (target + (self.gamma * y))
        else:
            # y was zero. Try a value an order of magnitude smaller
            scale = 0.1
        if (scale > self.max_scale):
            scale = self.max_scale
        # Ensures we stay within the tunable's domain (i.e. we don't take on
        # values to high or low).
        tunable.x = tunable.wrap_into_domain(scale * x)

class SecantSolver(Solver):
    def __init__(self, gamma=0.9):
        self._gamma = gamma
        self._previous_pair = dict()

    def _solve_one(self, tunable):
        x, y, target = tunable.x, tunable.y, tunable.target
        if tunable not in self._previous_pair:
            # We must perturb x some to get a second point to find the correct
            # root.
            new_x = tunable.wrap_into_domain(x * 1.1)
            if new_x == x:
                new_x = tunable.wrap_into_domain(x * 0.9)
                if new_x == x:
                    raise RuntimeError("Unable to perturb x for secant solver.")
        else:
            # standard secant formula. A brief note, we use f(x) = y - target
            # since this is the root we are searching for.
            old_x, old_f_x = self._previous_pair[tunable]
            self._previous_pair[tunable] = (x, y - target)
            f_x = y - target
            dxdf = (x - old_x) / (f_x - old_f_x)
            new_x = x - (self._gamma * f_x * dxdf)

        self._previous_pair[tunable] = (x, y - target)
        tunable.x = new_x
<commit_msg>Update Solvers to return convergence status<commit_after>from abc import ABCMeta, abstractmethod

class _TuneDefinition(metaclass=ABCMeta):
    def __init__(self, target, domain=None):
        self.domain = domain
        self._target = target

    def in_domain(self, value):
        if self.domain is None:
            return True
        else:
            lower_bound, upper_bound = self.domain
            return ((lower_bound is None or lower_bound <= value)
                    and (upper_bound is None or value <= upper_bound))

    def wrap_into_domain(self, value):
        if self._domain is None:
            return value
        else:
            lower_bound, upper_bound = self.domain
            if lower_bound is not None and value < lower_bound:
                return lower_bound
            elif upper_bound is not None and value > upper_bound:
                return upper_bound
            else:
                return value

    @property
    def x(self):
        return self._get_x()

    @property
    def max_x(self):
        if self.domain is None:
            return None
        else:
            return self.domain[1]

    @property
    def min_x(self):
        if self.domain is None:
            return None
        else:
            return self.domain[0]

    @x.setter
    def x(self, value):
        return self._set_x(self.wrap_into_domain(value))

    @property
    def y(self):
        return self._get_y()

    @property
    def target(self):
        return self._get_target()

    @target.setter
    def target(self, value):
        self._set_target(value)

    @abstractmethod
    def _get_x(self):
        pass

    @abstractmethod
    def _set_x(self):
        pass

    @abstractmethod
    def _get_y(self):
        pass

    def _get_target(self):
        return self._target

    def _set_target(self, value):
        self._target = value

    @property
    def domain(self):
        if self._domain is not None:
            return tuple(self._domain)
        else:
            return None

    @domain.setter
    def domain(self, value):
        if value is not None and not len(value) == 2:
            raise ValueError("domain must be a sequence of length two.")
        self._domain = value

    def __hash__(self):
        raise NotImplementedError("This object is not hashable.")

    def __eq__(self, other):
        raise NotImplementedError("This object is not equatable.")

class ManualTuneDefinition(metaclass=ABCMeta):
    def __init__(self, get_y, target_y, get_x, set_x, domain=None):
        self.__get_x = get_x
        self.__set_x = set_x
        self.__get_y = get_y
        self._target = target_y
        if domain is not None and not len(domain) == 2:
            raise ValueError("domain must be a sequence of length two.")
        self._domain = domain

    def _get_x(self):
        return self.__get_x()

    def _set_x(self, value):
        return self.__set_x(value)

    def _get_y(self):
        return self.__get_y()

    def _get_target(self):
        return self._target

    def _set_target(self, value):
        self._target = value

    def __hash__(self):
        return hash((self.__get_x, self.__set_x, self.__get_y, self._target))

    def __eq__(self, other):
        return (self.__get_x == other.__get_x
                and self.__set_x == other.__set_x
                and self.__get_y == other.__get_y
                and self._target == other._target)

class Solver(metaclass=ABCMeta):
    @abstractmethod
    def _solve_one(self, tunable):
        pass

    def solve(self, tunables):
        return all(self._solve_one(tunable) for tunable in tunables)

class ScaleSolver(Solver):
    def __init__(self, max_scale=2.0, gamma=2.0, tol=1e-5):
        self.max_scale = max_scale
        self.gamma = gamma
        self.tol = tol

    def _solve_one(self, tunable):
        x, y, target = tunable.x, tunable.y, tunable.target
        if abs(y - target) <= self.tol:
            return True

        if y > 0:
            scale = ((1.0 + self.gamma) * y) / (target + (self.gamma * y))
        else:
            # y was zero. Try a value an order of magnitude smaller
            scale = 0.1
        if (scale > self.max_scale):
            scale = self.max_scale
        # Ensures we stay within the tunable's domain (i.e. we don't take on
        # values to high or low).
        tunable.x = tunable.wrap_into_domain(scale * x)
        return False

class SecantSolver(Solver):
    def __init__(self, gamma=0.9, tol=1e-5):
        self.gamma = gamma
        self._previous_pair = dict()
        self.tol = tol

    def _solve_one(self, tunable):
        x, y, target = tunable.x, tunable.y, tunable.target
        if abs(y - target) <= self.tol:
            return True

        if tunable not in self._previous_pair:
            # We must perturb x some to get a second point to find the correct
            # root.
            new_x = tunable.wrap_into_domain(x * 1.1)
            if new_x == x:
                new_x = tunable.wrap_into_domain(x * 0.9)
                if new_x == x:
                    raise RuntimeError("Unable to perturb x for secant solver.")
        else:
            # standard secant formula. A brief note, we use f(x) = y - target
            # since this is the root we are searching for.
            old_x, old_f_x = self._previous_pair[tunable]
            self._previous_pair[tunable] = (x, y - target)
            f_x = y - target
            dxdf = (x - old_x) / (f_x - old_f_x)
            new_x = x - (self.gamma * f_x * dxdf)

        self._previous_pair[tunable] = (x, y - target)
        tunable.x = new_x
        return False
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
# TODO use swagger based autogenerated library for implementing these functions
# TODO use colors for heading and contests
from ...models import Judge
from ...utils.logging import logger
import click
import requests
import json
from .utils import get_data, login_oauth
from beautifultable import BeautifulTable
from ...utils.Errors import AuthenticationError
from .models import Problem, Contest, Testcase
from ...utils import style

class Codechef(Judge):
    def __init__(self, session_data=None):
        logger.debug("Initializing class Codechef with session_data:\n%s"
                     % session_data)
        self.name = "codechef"
        self.url = "https://www.codechef.com"
        self.api_url = "https://api.codechef.com"
        self.session_data = session_data
        if(session_data is not None):
            self._update_session()

    def check_login(self):
        logger.debug("Checking Login")
        if(self.session is None):
            logger.debug("No session object initialized")
            return False
        me_url = self._make_url(self.api_url, 'users/me')
        try:
            r = self._request_api(me_url)
            logger.debug(r)
            return True
        except requests.RequestException:
            return False

    def login(self):
        token = login_oauth()
        self.session_data = token
        self._update_session()
        assert(self.check_login())

    def logout(self):
        logger.warning("Logout of CodeChef.")
        click.confirm("Are you sure?", default=True, abort=True)
        self.session_data = None

    def get_running_contests(self):
        logger.debug('get running contests')
        contests = get_data.running_contests(self)
        table = BeautifulTable()
        table.width_exceed_policy = BeautifulTable.WEP_WRAP
        # TODO: use map style.headers instead of str
        # requires change with beautifultable. we may try dev version
        # TODO: sort running and upcoming
        table.column_headers = list(
            map(style.header, ['code', 'name', 'end', 'start']))
        for contest in contests:
            table.append_row(
                [
                    contest['code'], contest['name'],
                    str(contest['startDate']), str(contest['endDate'])
                ]
            )
        return table

    def get_problem(self, problem_code, contest_code):
        logger.info('fetching problem %s' % problem_code)
        if(problem_code is not None):
            if(contest_code is None):
                contest_code = 'PRACTICE'
            contest_code = contest_code.upper()
            problem_code = problem_code.upper()
            problem_url = self._make_url(
                self.api_url, 'contests', contest_code, 'problems', problem_code)
            problem_data = self._request_api(problem_url)
            problem = Problem(data=problem_data)
            problem.judge_name = self.name
            problem.contest_code = contest_code
            return problem

    def get_contest(self, contest_code):
        logger.info('fetching contest %s' % contest_code)
        contest_code = contest_code.upper()
        contest_url = self._make_url(self.api_url, 'contests', contest_code)
        contest_data = self._request_api(contest_url)
        contest = Contest(data=contest_data)
        logger.debug(contest.problem_codes)
        for problem_code in contest.problem_codes:
            contest.problems.append(
                self.get_problem(
                    contest_code=contest.code, problem_code=problem_code))
        contest.judge_name = self.name
        return contest

    def get_problem_url(self, problem_code, contest_code):
        if (contest_code is None):
            contest_code = 'PRACTICE'
        contest_code = contest_code.upper()
        problem_code = problem_code.upper()
        if(contest_code == 'PRACTICE'):
            return self._make_url(self.url, 'problems', problem_code)
        return self._make_url(self.url, contest_code,
                              'problems', problem_code)

    def submit(self, problem, code_text, extension):
        # TODO : correct this
        logger.warn("Codechef api doesn't support the submissions yet.\n"
                    "Please contact them to support this.\n"
                    "You can also try codechef-web implementation.\n"
                    "https://github.com/termicoder/termicoder-codechef-web\n"
                    )
        import pyperclip
        pyperclip.copy(code_text)
        if(problem.contest_code != 'PRACTICE'):
            url = self._make_url(
                self.url, problem.contest_code, 'submit', problem.code)
        else:
            url = self._make_url(
                self.url, 'submit', problem.code)
        logger.warn("By the time,"
                    "We have copied the code to clipboard "
                    "and are launching the submit url\n%s\n"
                    "Please paste your code there and submit" % url)
        click.confirm("Continue?", default=True)
        click.launch(url)

    def get_contest_url(self, contest_code):
        if (contest_code is None):
            contest_code = 'PRACTICE'
        contest_code = contest_code.upper()
        if(contest_code == 'PRACTICE'):
            return self._make_url(self.url, 'problems', 'school')
        return self._make_url(self.url, contest_code)

    def get_contests_list_url(self):
        return self._make_url(self.url, 'contests')

    def _update_session(self):
        self.session = requests.Session()

        def debug_url(r, *args, **kwargs):
            logger.debug('Getting url %s' % r.url)

        def debug_data(r, *args, **kwargs):
            try:
                response = json.dumps(r.json(), indent=1)
            except json.JSONDecodeError:
                response = r.text
            logger.debug('Response %s' % response)

        self.session.hooks['response'].append(debug_url)
        self.session.hooks['response'].append(debug_data)
        token = self.session_data['result']['data']['access_token']
        logger.debug('Token: ' + token)
        OAuth2_Header = {
            'Authorization': 'Bearer %s' % token
        }
        self.session.headers.update(OAuth2_Header)

    def get_testcase(self, inp, ans, code):
        return Testcase(inp=inp, ans=ans, code=code)

    def _make_url(self, base_url, *rel_urls):
        logger.debug(base_url)
        logger.debug(rel_urls)
        base_url = base_url.strip('/')
        join_urls = [base_url]
        for rel_url in rel_urls:
            join_urls.append(rel_url.strip('/'))
        return "/".join(join_urls)

    def _request_api(self, url):
        logger.debug('fetching url %s' % url)
        try:
            with self.session as s:
                r = s.get(url)
                if(r.status_code == 401):
                    logger.error("Authentication failed trying refreshing token")
                    self.refresh_login()
                    logger.debug(r)
                    r = s.get(url)
                r.raise_for_status()
            return r.json()
        except AttributeError:
            raise AuthenticationError(
                'The endpoint %s requires authorization\n'
                'Try `termicoder setup --login -j codechef` first'
                'and grant appropriate rights' % url)

    def refresh_login(self):
        logger.debug('refreshing token')
        url = 'http://termicoder.diveshuttam.me/refresh_token'
        # TODO implement this on server side
        logger.debug(self.session_data)
        try:
            r = requests.get(url, params={'data': json.dumps(self.session_data)})
            logger.debug("response for refresh:")
            logger.debug(r.text)
            r.raise_for_status()
            self.session_data = r.json()
            logger.debug(r.text)
        except requests.exceptions.HTTPError:
            self.session_data = None
            logger.error("Refreshing token failed, please try :"
                         "`termicoder setup -j codechef --login`")
<commit_msg>Resolved session error finally.. yay!<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
# TODO use swagger based autogenerated library for implementing these functions
# TODO use colors for heading and contests
from ...models import Judge
from ...utils.logging import logger
import click
import requests
import json
from .utils import get_data, login_oauth
from beautifultable import BeautifulTable
from ...utils.Errors import AuthenticationError
from .models import Problem, Contest, Testcase
from ...utils import style

class Codechef(Judge):
    def __init__(self, session_data=None):
        logger.debug("Initializing class Codechef with session_data:\n%s"
                     % session_data)
        self.name = "codechef"
        self.url = "https://www.codechef.com"
        self.api_url = "https://api.codechef.com"
        self.session_data = session_data
        if(session_data is not None):
            self._update_session()

    def check_login(self):
        logger.debug("Checking Login")
        if(self.session is None):
            logger.debug("No session object initialized")
            return False
        me_url = self._make_url(self.api_url, 'users/me')
        try:
            r = self._request_api(me_url)
            logger.debug(r)
            return True
        except requests.RequestException:
            return False

    def login(self):
        token = login_oauth()
        self.session_data = token
        self._update_session()
        assert(self.check_login())

    def logout(self):
        logger.warning("Logout of CodeChef.")
        click.confirm("Are you sure?", default=True, abort=True)
        self.session_data = None

    def get_running_contests(self):
        logger.debug('get running contests')
        contests = get_data.running_contests(self)
        table = BeautifulTable()
        table.width_exceed_policy = BeautifulTable.WEP_WRAP
        # TODO: use map style.headers instead of str
        # requires change with beautifultable. we may try dev version
        # TODO: sort running and upcoming
        table.column_headers = list(
            map(style.header, ['code', 'name', 'end', 'start']))
        for contest in contests:
            table.append_row(
                [
                    contest['code'], contest['name'],
                    str(contest['startDate']), str(contest['endDate'])
                ]
            )
        return table

    def get_problem(self, problem_code, contest_code):
        logger.info('fetching problem %s' % problem_code)
        if(problem_code is not None):
            if(contest_code is None):
                contest_code = 'PRACTICE'
            contest_code = contest_code.upper()
            problem_code = problem_code.upper()
            problem_url = self._make_url(
                self.api_url, 'contests', contest_code, 'problems', problem_code)
            problem_data = self._request_api(problem_url)
            problem = Problem(data=problem_data)
            problem.judge_name = self.name
            problem.contest_code = contest_code
            return problem

    def get_contest(self, contest_code):
        logger.info('fetching contest %s' % contest_code)
        contest_code = contest_code.upper()
        contest_url = self._make_url(self.api_url, 'contests', contest_code)
        contest_data = self._request_api(contest_url)
        contest = Contest(data=contest_data)
        logger.debug(contest.problem_codes)
        for problem_code in contest.problem_codes:
            contest.problems.append(
                self.get_problem(
                    contest_code=contest.code, problem_code=problem_code))
        contest.judge_name = self.name
        return contest

    def get_problem_url(self, problem_code, contest_code):
        if (contest_code is None):
            contest_code = 'PRACTICE'
        contest_code = contest_code.upper()
        problem_code = problem_code.upper()
        if(contest_code == 'PRACTICE'):
            return self._make_url(self.url, 'problems', problem_code)
        return self._make_url(self.url, contest_code,
                              'problems', problem_code)

    def submit(self, problem, code_text, extension):
        # TODO : correct this
        logger.warn("Codechef api doesn't support the submissions yet.\n"
                    "Please contact them to support this.\n"
                    "You can also try codechef-web implementation.\n"
                    "https://github.com/termicoder/termicoder-codechef-web\n"
                    )
        import pyperclip
        pyperclip.copy(code_text)
        if(problem.contest_code != 'PRACTICE'):
            url = self._make_url(
                self.url, problem.contest_code, 'submit', problem.code)
        else:
            url = self._make_url(
                self.url, 'submit', problem.code)
        logger.warn("By the time,"
                    "We have copied the code to clipboard "
                    "and are launching the submit url\n%s\n"
                    "Please paste your code there and submit" % url)
        click.confirm("Continue?", default=True)
        click.launch(url)

    def get_contest_url(self, contest_code):
        if (contest_code is None):
            contest_code = 'PRACTICE'
        contest_code = contest_code.upper()
        if(contest_code == 'PRACTICE'):
            return self._make_url(self.url, 'problems', 'school')
        return self._make_url(self.url, contest_code)

    def get_contests_list_url(self):
        return self._make_url(self.url, 'contests')

    def _update_session(self):
        self.session = requests.Session()

        def debug_url(r, *args, **kwargs):
            logger.debug('Getting url %s' % r.url)

        def debug_data(r, *args, **kwargs):
            try:
                response = json.dumps(r.json(), indent=1)
            except json.JSONDecodeError:
                response = r.text
            logger.debug('Response %s' % response)

        self.session.hooks['response'].append(debug_url)
        self.session.hooks['response'].append(debug_data)
        token = self.session_data['result']['data']['access_token']
        logger.debug('Token: ' + token)
        OAuth2_Header = {
            'Authorization': 'Bearer %s' % token
        }
        self.session.headers.update(OAuth2_Header)

    def get_testcase(self, inp, ans, code):
        return Testcase(inp=inp, ans=ans, code=code)

    def _make_url(self, base_url, *rel_urls):
        logger.debug(base_url)
        logger.debug(rel_urls)
        base_url = base_url.strip('/')
        join_urls = [base_url]
        for rel_url in rel_urls:
            join_urls.append(rel_url.strip('/'))
        return "/".join(join_urls)

    def _request_api(self, url):
        logger.debug('fetching url %s' % url)
        try:
            with self.session as s:
                r = s.get(url)
                if(r.status_code == 401):
                    logger.error("Authentication failed trying refreshing token")
                    self.refresh_login()
                    logger.debug(r)
                    # s is still bound to older session object so use self.session
                    r = self.session.get(url)
                r.raise_for_status()
            return r.json()
        except AttributeError:
            raise AuthenticationError(
                'The endpoint %s requires authorization\n'
                'Try `termicoder setup --login -j codechef` first'
                'and grant appropriate rights' % url)

    def refresh_login(self):
        logger.debug('refreshing token')
        url = 'http://termicoder.diveshuttam.me/refresh_token'
        # TODO implement this on server side
        logger.debug(self.session_data)
        try:
            r = requests.get(url, params={'data': json.dumps(self.session_data)})
            logger.debug("response for refresh:")
            logger.debug(r.text)
            r.raise_for_status()
            self.session_data = r.json()
            self._update_session()
            logger.debug(r.text)
        except requests.exceptions.HTTPError:
            self.session_data = None
            logger.error("Refreshing token failed, please try :"
                         "`termicoder setup -j codechef --login`")
<|endoftext|>


<commit_before># Scrapy settings for synscrape project
#
# For simplicity, this file contains only the most important settings by
# default. All the other settings are documented here:
#
#     http://doc.scrapy.org/topics/settings.html
#
# Or you can copy and paste them from where they're defined in Scrapy:
#
#     scrapy/conf/default_settings.py
#

BOT_NAME = 'synscrape'
BOT_VERSION = '1.0'

SPIDER_MODULES = ['synscrape.spiders']
NEWSPIDER_MODULE = 'synscrape.spiders'
DEFAULT_ITEM_CLASS = 'synscrape.items.SynscrapeItem'
USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION)

<commit_msg>Updated item class in synscrape settings<commit_after># Scrapy settings for synscrape project
#
# For simplicity, this file contains only the most important settings by
# default. All the other settings are documented here:
#
#     http://doc.scrapy.org/topics/settings.html
#
# Or you can copy and paste them from where they're defined in Scrapy:
#
#     scrapy/conf/default_settings.py
#

BOT_NAME = 'synscrape'
BOT_VERSION = '1.0'

SPIDER_MODULES = ['synscrape.spiders']
NEWSPIDER_MODULE = 'synscrape.spiders'
DEFAULT_ITEM_CLASS = 'synscrape.items.WordSenseItem'
USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION)

<|endoftext|>


<commit_before>#
# ------------------------------------------------------------
# Copyright (c) SILAB , Physics Institute of Bonn University
# ------------------------------------------------------------
#
# SVN revision information:
#  $Rev::                       $:
#  $Author::                    $:
#  $Date::                      $:
#

from HL.HardwareLayer import HardwareLayer
from struct import pack, unpack

class seq_gen(HardwareLayer):

    def __init__(self, intf, conf):
        HardwareLayer.__init__(self, intf, conf)

    def init(self):
        self.reset()

    def reset(self):
        self._intf.write(self._conf['base_addr'], [0])

    def start(self):
        self._intf.write(self._conf['base_addr'] + 1, [0])

    def set_size(self, value):
        self._intf.write(self._conf['base_addr'] + 3, unpack('BBBB', pack('>L', value))[2:4])

    def get_size(self):
        ret = self._intf.read(self._conf['base_addr'] + 3, 2)
        return ret[0] * (2 ** 8) + ret[1]

    def set_clk_divide(self, value):
        self._intf.write(self._conf['base_addr'] + 2, [value])

    def get_clk_divide(self):
        return self._intf.read(self._conf['base_addr'] + 2, 1)[0]

    def set_data(self, addr, data):
        self._intf.write(self._conf['base_addr'] + 16 + addr, data)

    def get_data(self, addr, size):
        return self._intf.read(self._conf['base_addr'] + 16 + addr, size)

    def set_repeat_start(self, value):
        self._intf.write(self._conf['base_addr'] + 8, unpack('BBBB', pack('>L', value))[2:4])

    def get_repeat_start(self):
        ret = self._intf.read(self._conf['base_addr'] + 8, 2)
        return ret[0] * (2 ** 8) + ret[1]
<commit_msg>ENH: new driver for cmd_seq module<commit_after>#
# ------------------------------------------------------------
# Copyright (c) SILAB , Physics Institute of Bonn University
# ------------------------------------------------------------
#
# SVN revision information:
#  $Rev::                       $:
#  $Author::                    $:
#  $Date::                      $:
#

from HL.HardwareLayer import HardwareLayer
from struct import pack, unpack, unpack_from
from array import array

class seq_gen(HardwareLayer):

    def __init__(self, intf, conf):
        HardwareLayer.__init__(self, intf, conf)
        self._cmd_mem_offset = 16  # in bytes
        self._cmd_mem_size = conf['cmd_mem_size'] - self._cmd_mem_offset  # in bytes

    def init(self):
        self.reset()

    def reset(self):
        self._intf.write(self._conf['base_addr'], (0,))

    def start(self):
        self._intf.write(self._conf['base_addr'] + 1, (0,))

    @property
    def is_ready(self):
        return (self._intf.read(self._conf['base_addr'] + 1, size=1)[0] & 0x01) == 1

    def set_ext_trigger(self, value):
        ret = self._intf.read(self._conf['base_addr'] + 2, size=1)
        reg = unpack_from('B', ret)[0]
        if value:
            reg |= 0x01
        else:
            reg &= ~0x01
        self._intf.write(self._conf['base_addr'] + + 2, data=(reg,))

    def set_output_mode(self, value):
        if value > 3 or value < 0:
            raise ValueError('Output mode not existing')
        ret = self._intf.read(self._conf['base_addr'] + 2, size=1)
        reg = unpack_from('B', ret)[0]
        reg = ((value & 0x03) << 1) | (reg & 0xf9)
        self._intf.write(self._conf['base_addr'] + + 2, data=(reg,))

    def set_clock_gate(self, value):
        ret = self._intf.read(self._conf['base_addr'] + 2, size=1)
        reg = unpack_from('B', ret)[0]
        if not value:
            reg |= 0x08
        else:
            reg &= ~0x08
        self._intf.write(self._conf['base_addr'] + + 2, data=(reg,))

    def set_cmd_pulse(self, value):
        ret = self._intf.read(self._conf['base_addr'] + 2, size=1)
        reg = unpack_from('B', ret)[0]
        if value:
            reg |= 0x10
        else:
            reg &= ~0x10
        self._intf.write(self._conf['base_addr'] + + 2, data=(reg,))

    def get_size(self):
        ret = self._intf.read(self._conf['base_addr'] + 3, size=2)
        return unpack_from('H', ret)[0]

    def set_size(self, value):
        self._intf.write(self._conf['base_addr'] + 3, array.array('B', pack('H', value)))  # alternatively: unpack('BB', pack('H', value))

    def get_repeat(self):
        ret = self._intf.read(self._conf['base_addr'] + 5, size=4)
        return unpack_from('L', ret)[0]

    def set_repeat(self, value):
        self._intf.write(self._conf['base_addr'] + 5, array.array('B', pack('L', value)))

    def get_start_seq_length(self):
        ret = self._intf.read(self._conf['base_addr'] + 9, size=2)
        return unpack_from('H', ret)[0]

    def set_start_seq_length(self, value):
        if value < 2:
            raise ValueError('Length is too short')  # bug in FPGA module
        self._intf.write(self._conf['base_addr'] + 9, array.array('B', pack('H', value)))

    def get_stop_seq_length(self):
        ret = self._intf.read(self._conf['base_addr'] + 11, size=2)
        return unpack_from('H', ret)[0]

    def set_stop_seq_length(self, value):
        if value < 2:
            raise ValueError('Length is too short')  # bug in FPGA module
        self._intf.write(self._conf['base_addr'] + 1, array.array('B', pack('H', value)))

    def set_data(self, addr=0, data):
        if self._cmd_mem_size < len(data):
            raise ValueError('Size of data is too big')
        self._intf.write(self._conf['base_addr'] + 16 + addr, data)

    def get_data(self, addr=0, size):
        if self._cmd_mem_size < size:
            raise ValueError('Size is too big')
        return self._intf.read(self._conf['base_addr'] + 16 + addr, size)
<|endoftext|>


<commit_before>import numpy as np
import h5py

# value is a list
# converts anything numeric into a float
def processMetadataValue(value):
    if len(value) < 2:
        return ''
    try:
        return float(value[1])
    except ValueError:
        return value[1]

# a fuction to parse txtXY files
# will parse files with or without metadata
# probably pretty inefficient, but easy to read :)
def parseTxtXY(filename):
    with open(filename, "r") as file:
        rawdata=file.readlines()
        # strip all whitespace except tabs
        rawdata = [s.replace(" ", "") for s in rawdata]
        # get metadata fields
        splitNumeric = [s.split('\t') for s in rawdata if s[0] == '-' or ord(s[0]) in range(ord('0'), ord('9'))]
        splitMetadata = [s[1:].replace("\n", "").split(':') for s in rawdata if s[0] == '$']
        data = np.asarray(splitNumeric).astype(np.double)
        metadataKeys = [line[0] for line in splitMetadata]
        metadataValues = [processMetadataValue(line) for line in splitMetadata]
        metadata = dict(zip(metadataKeys, metadataValues))
        return {'data': data, 'metadata': metadata}

# will save a sample file, assuming that the first row is x and subsequent rows are Y
# numData is numpy array, metadata is dictionary
def saveSampleFile(filename, numData, metadata):
    # clean metadata of empty entries:
    metadata = {key: value for key, value in metadata.items() if value != ''}
    with h5py.File(filename, 'w') as outfile:
        ut
        outfile.create_dataset('x', data=numData[:, 0].astype(np.double))
        outfile.create_dataset('Y', data=numData[:, 1:].astype(np.double))
        for key, value in metadata.items():
            if isinstance(value, str):
                outfile.attrs[key] = np.string_(value)
            else:
                outfile.attrs[key] = value

<commit_msg>Fix error in parsetxtxy.py<commit_after>import numpy as np
import h5py

# value is a list
# converts anything numeric into a float
def processMetadataValue(value):
    if len(value) < 2:
        return ''
    try:
        return float(value[1])
    except ValueError:
        return value[1]

# a fuction to parse txtXY files
# will parse files with or without metadata
# probably pretty inefficient, but easy to read :)
def parseTxtXY(filename):
    with open(filename, "r") as file:
        rawdata=file.readlines()
        # strip all whitespace except tabs
        rawdata = [s.replace(" ", "") for s in rawdata]
        # get metadata fields
        splitNumeric = [s.split('\t') for s in rawdata if s[0] == '-' or ord(s[0]) in range(ord('0'), ord('9'))]
        splitMetadata = [s[1:].replace("\n", "").split(':') for s in rawdata if s[0] == '$']
        data = np.asarray(splitNumeric).astype(np.double)
        metadataKeys = [line[0] for line in splitMetadata]
        metadataValues = [processMetadataValue(line) for line in splitMetadata]
        metadata = dict(zip(metadataKeys, metadataValues))
        return {'data': data, 'metadata': metadata}

# will save a sample file, assuming that the first row is x and subsequent rows are Y
# numData is numpy array, metadata is dictionary
def saveSampleFile(filename, numData, metadata):
    # clean metadata of empty entries:
    metadata = {key: value for key, value in metadata.items() if value != ''}
    with h5py.File(filename, 'w') as outfile:
        outfile.create_dataset('x', data=numData[:, 0].astype(np.double))
        outfile.create_dataset('Y', data=numData[:, 1:].astype(np.double))
        for key, value in metadata.items():
            if isinstance(value, str):
                outfile.attrs[key] = np.string_(value)
            else:
                outfile.attrs[key] = value

<|endoftext|>


<commit_before>import os
from collections import namedtuple
from skylines.lib import files
from skylines.lib.xcsoar.path import helper_path

FlightPathFix = namedtuple('FlightPathFix', ['seconds_of_day', 'latitude', 'longitude', 'altitude', 'enl'])

def flight_path(igc_file, max_points=1000):
    path = files.filename_to_path(igc_file.filename)
    f = os.popen(helper_path('FlightPath') + ' --max-points=' + str(max_points) + ' "' + path + '"')

    path = []
    for line in f:
        line = line.split()
        path.append(FlightPathFix(int(line[0]), float(line[1]), float(line[2]), int(line[3]), int(line[4])))
    return path
<commit_msg>flightpath: Replaced deprecated os.popen() by subprocess.Popen()<commit_after>from subprocess import Popen, PIPE
from collections import namedtuple
from skylines.lib import files
from skylines.lib.xcsoar.path import helper_path

FlightPathFix = namedtuple('FlightPathFix', ['seconds_of_day', 'latitude', 'longitude', 'altitude', 'enl'])

def flight_path(igc_file, max_points=1000):
    args = [
        helper_path('FlightPath'),
        '--max-points=' + str(max_points),
        files.filename_to_path(igc_file.filename),
    ]

    p = Popen(args, stdout=PIPE)

    path = []
    for line in p.stdout:
        line = line.split()
        path.append(FlightPathFix(int(line[0]), float(line[1]), float(line[2]), int(line[3]), int(line[4])))

    return path
<|endoftext|>