<commit_before>"""
Interface to minimization algorithms

Functions
---------
- minimize : unconstrained minimization of a function of several variables.
- con_minimize : constrained minimization of a function of several variables.
- minimize1d: minimization of a scalar function.

"""

__all__ = ['minimize']

from warnings import warn
from numpy import Inf
from optimize import _epsilon
# unconstrained minimization
from optimize import fmin, fmin_powell, fmin_cg, fmin_bfgs, fmin_ncg

def minimize(fun, x0, args=(), method='Nelder-Mead', jac=None, hess=None,
             hessp=None, options=dict(), full_output=False, callback=None,
             retall=False):
    """
    Minimization of scalar function of several variables.

    Parameters
    ----------
    fun : callable
        Objective function.
    x0 : ndarray
        Initial guess.
    args : tuple
        Extra arguments passed to the objective function and its
        derivatives (Jacobian, Hessian).
    method : str
        Type of solver amongst:
            'Nelder-Mead', 'Powell', 'CG', 'BFGS', 'Newton-CG'.
    jac : callable
        Jacobian of objective function (if None, Jacobian will be
        estimate numerically). Only for CG, BFGS, Newton-CG.
    hess, hessp : callable
        Hessian of objective function or Hessian of objective function
        times and arbitrary vector p. Only for Newton-CG.
        Only one of `hessp` or `hess` need to be given.  If `hess` is
        provided, then `hessp` will be ignored.  If neither `hess` nor
        `hessp` is provided, then the hessian product will be approximated
        using finite differences on `jac`. `hessp` must compute the hessian
        times an arbitrary vector. If it is not given, finite-differences
        on `jac` are used to compute it.
    options : dict
        A dictionary of solver options with the keys:
            disp : int
                If positive, information on the progress of the
                optimization is displayed. Different level of details are
                available depending on the solver. Most solvers consider
                either 0 or 1 (i.e. Boolean) values for `disp`.
            xtol : float
                Relative error in xopt acceptable for convergence.
            ftol : float
                Relative error in fun(xopt) acceptable for convergence.
            maxit : int
                Maximum number of iterations to perform.
            maxfev : int
                Maximum number of function evaluations to make.
            gtol : float
                Gradient norm must be less than gtol before successful
                termination.
            norm : float
                Order of norm (Inf is max, -Inf is min)
            eps : int or ndarray
                If `jac` is approximated, use this value for the step size.
    full_output : bool
        If True, return optional outputs.
    callback : callable
        Called after each iteration, as callback(xk), where xk is the
        current parameter vector.
    retall : bool
        If True, return a list of the solution at each iteration. (Implies
        full_output=True, ignored otherwise.)

    Returns
    -------
    x : ndarray
        The solution.
    info : dict
        A dictionary of optional outputs with the keys:
            status : int
                An integer flag. Set to 0 if a solution was found,
                otherwise refer to message for more information.
            message : str
                If no solution is found, message details the cause of failure.
            fun, jac, hess : ndarray
                Values of objective function, Jacobian and Hessian (if
                available).
            nfev, njev, nhev: int
                Number of evaluations of the objective functions and of its
                jacobian and hessian.
            nit: int
                Number of iterations.
            direc: array
                Current direction set.
            allvecs : list
                Solution at each iteration (if retall==True).

    Notes
    -----
    This section describes the available solver to be selected by the
    'method' parameter. Respective solver options are also listed here.

    By default, this function uses the *Nelder-Mead* simplex algorithm
    [1]_,[2]_,[3]_ to find the a minimum of function of one or more
    variables. This algorithm has a long history of successful use in
    applications. But it will usually be slower than an algorithm that uses
    first or second derivative information. In practice it can have poor
    performance in high-dimensional problems and is not robust to
    minimizing complicated functions. Additionally, there currently is no
    complete theory describing when the algorithm will successfully
    converge to the minimum, or how fast it will if it does.

    Relevant `options` are: `xtol`, `ftol`, `maxit`, `maxfev`, `disp`.

    Method *Powell* is a modification of Powell's method [4]_, [5]_ to find
    the minimum of a function of N variables. Powell's method is a
    conjugate direction method.

    The algorithm has two loops. The outer loop merely iterates over the
    inner loop. The inner loop minimizes over each current direction in the
    direction set. At the end of the inner loop, if certain conditions are
    met, the direction that gave the largest decrease is dropped and
    replaced with the difference between the current estiamted x and the
    estimated x from the beginning of the inner-loop.

    The technical conditions for replacing the direction of greatest
    increase amount to checking that

    1. No further gain can be made along the direction of greatest increase
       from that iteration.
    2. The direction of greatest increase accounted for a large sufficient
       fraction of the decrease in the function value from that iteration of
       the inner loop.

    Relevant `options` are: `xtol`, `ftol`, `maxit`, `maxfev`, `disp`.

    Method *CG* uses a nonlinear conjugate gradient algorithm by Polak and
    Ribiere as described in [6]_ pp. 120-122.

    Relevant `options` are: `gtol`, `norm`, `maxit`, `eps`, `disp`.

    Method *BFGS* uses the quasi-Newton method of Broyden, Fletcher,
    Goldfarb, and Shanno (BFGS) [6]_ pp. 198

    Relevant `options` are: `gtol`, `norm`, `maxit`, `eps`, `disp`.

    Method *Newton-CG* uses a Newton-CG algorithm [6]_ pp.140. Newton-CG
    methods are also called truncated Newton methods. See also `fmincon`
    with method='TNC' for a box-constrained minimization with a similar
    algorithm.

    Relevant `options` are: `xtol`, `maxit`, `eps`, `disp`.

    References
    ----------
    .. [1] Nelder, J.A. and Mead, R. (1965), "A simplex method for function
       minimization", The Computer Journal, 7, pp. 308-313
    .. [2] Wright, M.H. (1996), "Direct Search Methods: Once Scorned, Now
       Respectable", in Numerical Analysis 1995, Proceedings of the
       1995 Dundee Biennial Conference in Numerical Analysis, D.F.
    .. [3] Griffiths and G.A. Watson (Eds.), Addison Wesley Longman,
       Harlow, UK, pp. 191-208.
    .. [4] Powell M.J.D. (1964) An efficient method for finding the minimum of a
       function of several variables without calculating derivatives,
       Computer Journal, 7 (2):155-162.
    .. [5] Press W., Teukolsky S.A., Vetterling W.T., and Flannery B.P.:
       Numerical Recipes (any edition), Cambridge University Press
    .. [6] Wright & Nocedal, 'Numerical Optimization', 1999.
    """
    if full_output:
        info = dict()
    elif retall: # retall implies full_output, issue a warning otherwise
        warn('retall is ignored since full_output=False.', RuntimeWarning)

    if method.lower() == 'nelder-mead':
        # set default options and update using input values
        opts = {'xtol': 1e-4,
                'ftol': 1e-4,
                'disp': 1}
        opts.update(**options)

        out = fmin(fun, x0, args=args, xtol=opts.get('xtol'),
                   ftol=opts.get('ftol'), maxiter=opts.get('maxit'),
                   maxfun=opts.get('maxfev'), full_output=full_output,
                   disp=opts.get('disp'), retall=retall, callback=callback)

        if full_output:
            x, info['fun'], info['nit'], info['nfev'], info['status'] = \
                    out[:5]
            if retall:
                info['allvecs'] = out[5]
        else:
            if retall:
                x = out[0]
            else:
                x = out

    elif method.lower() == 'powell':
        # set default options and update using input values
        opts = {'xtol': 1e-4,
                'ftol': 1e-4,
                'disp': 1}
        opts.update(**options)
        out = fmin_powell(fun, x0, args=args, xtol=opts.get('xtol'),
                          ftol=opts.get('ftol'), maxiter=opts.get('maxit'),
                          maxfun=opts.get('maxfev'),
                          full_output=full_output, disp=opts.get('disp'),
                          retall=retall, callback=callback)

        if full_output:
            x, info['fun'], info['direc'], info['nit'], info['nfev'], \
                    info['status'] = out[:6]
            if retall:
                info['allvecs'] = out[6]
        else:
            if retall:
                x = out[0]
            else:
                x = out

    elif method.lower() == 'cg':
        # set default options and update using input values
        opts = {'gtol': 1e-5,
                'norm': Inf,
                'eps': _epsilon,
                'disp': 1}
        opts.update(**options)
        out = fmin_cg(fun, x0, fprime=jac, args=args,
                      gtol=opts.get('gtol'), norm=opts.get('norm'),
                      epsilon=opts.get('eps'), maxiter=opts.get('maxit'),
                      full_output=full_output, disp=opts.get('disp'),
                      retall=retall, callback=callback)

        if full_output:
            x, info['fun'], info['nfev'], info['njev'], info['status'] = \
                    out[:5]
            if retall:
                info['allvecs'] = out[5]
        else:
            if retall:
                x = out[0]
            else:
                x = out

    elif method.lower() == 'bfgs':
        # set default options and update using input values
        opts = {'gtol': 1e-5,
                'norm': Inf,
                'eps': _epsilon,
                'disp': 1}
        opts.update(**options)
        out = fmin_bfgs(fun, x0, fprime=jac, args=args,
                        gtol=opts.get('gtol'), norm=opts.get('norm'),
                        epsilon=opts.get('eps'), maxiter=opts.get('maxit'),
                        full_output=full_output, disp=opts.get('disp'),
                        retall=retall, callback=callback)

        if full_output:
            x, info['fun'], info['jac'], info['hess'], info['nfev'], \
                    info['njev'], info['status'] = out[:7]
            if retall:
                info['allvecs'] = out[7]
        else:
            if retall:
                x = out[0]
            else:
                x = out

    elif method.lower() == 'newton-cg':
        if jac == None:
            raise ValueError('Jacobian is required for Newton-CG method')
        # set default options and update using input values
        opts = {'xtol': 1e-5,
                'eps': _epsilon,
                'disp': 1}
        opts.update(**options)
        out = fmin_ncg(fun, x0, jac, fhess_p=hessp, fhess=hess, args=args,
                       avextol=opts.get('xtol'), epsilon=opts.get('eps'),
                       maxiter=opts.get('maxit'), full_output=full_output,
                       disp=opts.get('disp'), retall=retall,
                       callback=callback)

        if full_output:
            x, info['fun'], info['nfev'], info['njev'], info['nhev'], \
                    info['status'] = out[:6]
            if retall:
                info['allvecs'] = out[6]
        else:
            if retall:
                x = out[0]
            else:
                x = out

    else:
        raise ValueError('Unknown solver %s' % method)

    if full_output:
        return x, info
    else:
        return x
<commit_msg>optimize: use internal functions in minimize interface<commit_after>"""
Interface to minimization algorithms

Functions
---------
- minimize : unconstrained minimization of a function of several variables.
- con_minimize : constrained minimization of a function of several variables.
- minimize1d: minimization of a scalar function.

"""

__all__ = ['minimize']

from warnings import warn
from numpy import Inf
from optimize import _epsilon
# unconstrained minimization
from optimize import _minimize_neldermead, _minimize_powell, \
        _minimize_cg, _minimize_bfgs, _minimize_ncg

def minimize(fun, x0, args=(), method='Nelder-Mead', jac=None, hess=None,
             hessp=None, options=dict(), full_output=False, callback=None,
             retall=False):
    """
    Minimization of scalar function of several variables.

    Parameters
    ----------
    fun : callable
        Objective function.
    x0 : ndarray
        Initial guess.
    args : tuple
        Extra arguments passed to the objective function and its
        derivatives (Jacobian, Hessian).
    method : str
        Type of solver amongst:
            'Nelder-Mead', 'Powell', 'CG', 'BFGS', 'Newton-CG'.
    jac : callable
        Jacobian of objective function (if None, Jacobian will be
        estimate numerically). Only for CG, BFGS, Newton-CG.
    hess, hessp : callable
        Hessian of objective function or Hessian of objective function
        times and arbitrary vector p. Only for Newton-CG.
        Only one of `hessp` or `hess` need to be given.  If `hess` is
        provided, then `hessp` will be ignored.  If neither `hess` nor
        `hessp` is provided, then the hessian product will be approximated
        using finite differences on `jac`. `hessp` must compute the hessian
        times an arbitrary vector. If it is not given, finite-differences
        on `jac` are used to compute it.
    options : dict
        A dictionary of solver options with the keys:
            disp : int
                If positive, information on the progress of the
                optimization is displayed. Different level of details are
                available depending on the solver. Most solvers consider
                either 0 or 1 (i.e. Boolean) values for `disp`.
            xtol : float
                Relative error in xopt acceptable for convergence.
            ftol : float
                Relative error in fun(xopt) acceptable for convergence.
            maxit : int
                Maximum number of iterations to perform.
            maxfev : int
                Maximum number of function evaluations to make.
            gtol : float
                Gradient norm must be less than gtol before successful
                termination.
            norm : float
                Order of norm (Inf is max, -Inf is min)
            eps : int or ndarray
                If `jac` is approximated, use this value for the step size.
    full_output : bool
        If True, return optional outputs.
    callback : callable
        Called after each iteration, as callback(xk), where xk is the
        current parameter vector.
    retall : bool
        If True, return a list of the solution at each iteration. (Implies
        full_output=True, ignored otherwise.)

    Returns
    -------
    x : ndarray
        The solution.
    info : dict
        A dictionary of optional outputs with the keys:
            status : int
                An integer flag. Set to 0 if a solution was found,
                otherwise refer to message for more information.
            message : str
                If no solution is found, message details the cause of failure.
            fun, jac, hess : ndarray
                Values of objective function, Jacobian and Hessian (if
                available).
            nfev, njev, nhev: int
                Number of evaluations of the objective functions and of its
                jacobian and hessian.
            nit: int
                Number of iterations.
            direc: array
                Current direction set.
            allvecs : list
                Solution at each iteration (if retall==True).

    Notes
    -----
    This section describes the available solver to be selected by the
    'method' parameter. Respective solver options are also listed here.

    By default, this function uses the *Nelder-Mead* simplex algorithm
    [1]_,[2]_,[3]_ to find the a minimum of function of one or more
    variables. This algorithm has a long history of successful use in
    applications. But it will usually be slower than an algorithm that uses
    first or second derivative information. In practice it can have poor
    performance in high-dimensional problems and is not robust to
    minimizing complicated functions. Additionally, there currently is no
    complete theory describing when the algorithm will successfully
    converge to the minimum, or how fast it will if it does.

    Relevant `options` are: `xtol`, `ftol`, `maxit`, `maxfev`, `disp`.

    Method *Powell* is a modification of Powell's method [4]_, [5]_ to find
    the minimum of a function of N variables. Powell's method is a
    conjugate direction method.

    The algorithm has two loops. The outer loop merely iterates over the
    inner loop. The inner loop minimizes over each current direction in the
    direction set. At the end of the inner loop, if certain conditions are
    met, the direction that gave the largest decrease is dropped and
    replaced with the difference between the current estiamted x and the
    estimated x from the beginning of the inner-loop.

    The technical conditions for replacing the direction of greatest
    increase amount to checking that

    1. No further gain can be made along the direction of greatest increase
       from that iteration.
    2. The direction of greatest increase accounted for a large sufficient
       fraction of the decrease in the function value from that iteration of
       the inner loop.

    Relevant `options` are: `xtol`, `ftol`, `maxit`, `maxfev`, `disp`.

    Method *CG* uses a nonlinear conjugate gradient algorithm by Polak and
    Ribiere as described in [6]_ pp. 120-122.

    Relevant `options` are: `gtol`, `norm`, `maxit`, `eps`, `disp`.

    Method *BFGS* uses the quasi-Newton method of Broyden, Fletcher,
    Goldfarb, and Shanno (BFGS) [6]_ pp. 198

    Relevant `options` are: `gtol`, `norm`, `maxit`, `eps`, `disp`.

    Method *Newton-CG* uses a Newton-CG algorithm [6]_ pp.140. Newton-CG
    methods are also called truncated Newton methods. See also `fmincon`
    with method='TNC' for a box-constrained minimization with a similar
    algorithm.

    Relevant `options` are: `xtol`, `maxit`, `eps`, `disp`.

    References
    ----------
    .. [1] Nelder, J.A. and Mead, R. (1965), "A simplex method for function
       minimization", The Computer Journal, 7, pp. 308-313
    .. [2] Wright, M.H. (1996), "Direct Search Methods: Once Scorned, Now
       Respectable", in Numerical Analysis 1995, Proceedings of the
       1995 Dundee Biennial Conference in Numerical Analysis, D.F.
    .. [3] Griffiths and G.A. Watson (Eds.), Addison Wesley Longman,
       Harlow, UK, pp. 191-208.
    .. [4] Powell M.J.D. (1964) An efficient method for finding the minimum of a
       function of several variables without calculating derivatives,
       Computer Journal, 7 (2):155-162.
    .. [5] Press W., Teukolsky S.A., Vetterling W.T., and Flannery B.P.:
       Numerical Recipes (any edition), Cambridge University Press
    .. [6] Wright & Nocedal, 'Numerical Optimization', 1999.
    """
    if full_output:
        info = dict()
    elif retall: # retall implies full_output, issue a warning otherwise
        warn('retall is ignored since full_output=False.', RuntimeWarning)

    if method.lower() == 'nelder-mead':
        out = _minimize_neldermead(fun, x0, args, options, full_output,
                                   retall, callback)

        if full_output:
            x, info['fun'], info['nit'], info['nfev'], info['status'] = \
                    out[:5]
            if retall:
                info['allvecs'] = out[5]
        else:
            if retall:
                x = out[0]
            else:
                x = out

    elif method.lower() == 'powell':
        out = _minimize_powell(fun, x0, args, options, full_output, retall,
                               callback)

        if full_output:
            x, info['fun'], info['direc'], info['nit'], info['nfev'], \
                    info['status'] = out[:6]
            if retall:
                info['allvecs'] = out[6]
        else:
            if retall:
                x = out[0]
            else:
                x = out

    elif method.lower() == 'cg':
        out = _minimize_cg(fun, x0, args, jac, options, full_output,
                           retall, callback)

        if full_output:
            x, info['fun'], info['nfev'], info['njev'], info['status'] = \
                    out[:5]
            if retall:
                info['allvecs'] = out[5]
        else:
            if retall:
                x = out[0]
            else:
                x = out

    elif method.lower() == 'bfgs':
        out = _minimize_bfgs(fun, x0, args, jac, options, full_output,
                             retall, callback)

        if full_output:
            x, info['fun'], info['jac'], info['hess'], info['nfev'], \
                    info['njev'], info['status'] = out[:7]
            if retall:
                info['allvecs'] = out[7]
        else:
            if retall:
                x = out[0]
            else:
                x = out

    elif method.lower() == 'newton-cg':
        if jac == None:
            raise ValueError('Jacobian is required for Newton-CG method')
        out = _minimize_ncg(fun, x0, args, jac, hess, hessp, options,
                            full_output, retall, callback)

        if full_output:
            x, info['fun'], info['nfev'], info['njev'], info['nhev'], \
                    info['status'] = out[:6]
            if retall:
                info['allvecs'] = out[6]
        else:
            if retall:
                x = out[0]
            else:
                x = out

    else:
        raise ValueError('Unknown solver %s' % method)

    if full_output:
        return x, info
    else:
        return x
<|endoftext|>


<commit_before>"""Some functions for working with contingency tables (i.e. cross tabulations).
"""

from __future__ import division, print_function, absolute_import

from functools import reduce
import numpy as np
from .stats import power_divergence

__all__ = ['margins', 'expected_freq', 'chi2_contingency']

def margins(a):
    """Return a list of the marginal sums of the array `a`.

    Parameters
    ----------
    a : ndarray
        The array for which to compute the marginal sums.

    Returns
    -------
    margsums : list of ndarrays
        A list of length `a.ndim`.  `margsums[k]` is the result
        of summing `a` over all axes except `k`; it has the same
        number of dimensions as `a`, but the length of each axis
        except axis `k` will be 1.

    Examples
    --------
    >>> a = np.arange(12).reshape(2, 6)
    >>> a
    array([[ 0,  1,  2,  3,  4,  5],
           [ 6,  7,  8,  9, 10, 11]])
    >>> m0, m1 = margins(a)
    >>> m0
    array([[15],
           [51]])
    >>> m1
    array([[ 6,  8, 10, 12, 14, 16]])

    >>> b = np.arange(24).reshape(2,3,4)
    >>> m0, m1, m2 = margins(b)
    >>> m0
    array([[[ 66]],
           [[210]]])
    >>> m1
    array([[[ 60],
            [ 92],
            [124]]])
    >>> m2
    array([[[60, 66, 72, 78]]])
    """
    margsums = []
    ranged = list(range(a.ndim))
    for k in ranged:
        marg = np.apply_over_axes(np.sum, a, [j for j in ranged if j != k])
        margsums.append(marg)
    return margsums

def expected_freq(observed):
    """
    Compute the expected frequencies from a contingency table.

    Given an n-dimensional contingency table of observed frequencies,
    compute the expected frequencies for the table based on the marginal
    sums under the assumption that the groups associated with each
    dimension are independent.

    Parameters
    ----------
    observed : array_like
        The table of observed frequencies.  (While this function can handle
        a 1-D array, that case is trivial.  Generally `observed` is at
        least 2-D.)

    Returns
    -------
    expected : ndarray of float64
        The expected frequencies, based on the marginal sums of the table.
        Same shape as `observed`.

    Examples
    --------
    >>> observed = np.array([[10, 10, 20],[20, 20, 20]])
    >>> from scipy.stats.contingency import expected_freq
    >>> expected_freq(observed)
    array([[ 12.,  12.,  16.],
           [ 18.,  18.,  24.]])

    """
    # Typically `observed` is an integer array. If `observed` has a large
    # number of dimensions or holds large values, some of the following
    # computations may overflow, so we first switch to floating point.
    observed = np.asarray(observed, dtype=np.float64)

    # Create a list of the marginal sums.
    margsums = margins(observed)

    # Create the array of expected frequencies.  The shapes of the
    # marginal sums returned by apply_over_axes() are just what we
    # need for broadcasting in the following product.
    d = observed.ndim
    expected = reduce(np.multiply, margsums) / observed.sum() ** (d - 1)
    return expected

def chi2_contingency(observed, correction=True, lambda_=None):
    """Chi-square test of independence of variables in a contingency table.

    This function computes the chi-square statistic and p-value for the
    hypothesis test of independence of the observed frequencies in the
    contingency table [1]_ `observed`.  The expected frequencies are computed
    based on the marginal sums under the assumption of independence; see
    `scipy.stats.contingency.expected_freq`.  The number of degrees of
    freedom is (expressed using numpy functions and attributes)::

        dof = observed.size - sum(observed.shape) + observed.ndim - 1

    Parameters
    ----------
    observed : array_like
        The contingency table. The table contains the observed frequencies
        (i.e. number of occurrences) in each category.  In the two-dimensional
        case, the table is often described as an "R x C table".
    correction : bool, optional
        If True, *and* the degrees of freedom is 1, apply Yates' correction
        for continuity.  The effect of the correction is to adjust each
        observed value by 0.5 towards the corresponding expected value.
    lambda_ : float or str, optional.
        By default, the statistic computed in this test is Pearson's
        chi-squared statistic [2]_.  `lambda_` allows a statistic from the
        Cressie-Read power divergence family [3]_ to be used instead.  See
        `power_divergence` for details.

    Returns
    -------
    chi2 : float
        The test statistic.
    p : float
        The p-value of the test
    dof : int
        Degrees of freedom
    expected : ndarray, same shape as `observed`
        The expected frequencies, based on the marginal sums of the table.

    See Also
    --------
    contingency.expected_freq
    fisher_exact
    chisquare
    power_divergence

    Notes
    -----
    An often quoted guideline for the validity of this calculation is that
    the test should be used only if the observed and expected frequencies
    in each cell are at least 5.

    This is a test for the independence of different categories of a
    population. The test is only meaningful when the dimension of
    `observed` is two or more.  Applying the test to a one-dimensional
    table will always result in `expected` equal to `observed` and a
    chi-square statistic equal to 0.

    This function does not handle masked arrays, because the calculation
    does not make sense with missing values.

    Like stats.chisquare, this function computes a chi-square statistic;
    the convenience this function provides is to figure out the expected
    frequencies and degrees of freedom from the given contingency table.
    If these were already known, and if the Yates' correction was not
    required, one could use stats.chisquare.  That is, if one calls::

        chi2, p, dof, ex = chi2_contingency(obs, correction=False)

    then the following is true::

        (chi2, p) == stats.chisquare(obs.ravel(), f_exp=ex.ravel(),
                                     ddof=obs.size - 1 - dof)

    The `lambda_` argument was added in version 0.13.0 of scipy.

    References
    ----------
    .. [1] "Contingency table",
           https://en.wikipedia.org/wiki/Contingency_table
    .. [2] "Pearson's chi-squared test",
           https://en.wikipedia.org/wiki/Pearson%27s_chi-squared_test
    .. [3] Cressie, N. and Read, T. R. C., "Multinomial Goodness-of-Fit
           Tests", J. Royal Stat. Soc. Series B, Vol. 46, No. 3 (1984),
           pp. 440-464.

    Examples
    --------
    A two-way example (2 x 3):

    >>> from scipy.stats import chi2_contingency
    >>> obs = np.array([[10, 10, 20], [20, 20, 20]])
    >>> chi2_contingency(obs)
    (2.7777777777777777,
     0.24935220877729619,
     2,
     array([[ 12.,  12.,  16.],
            [ 18.,  18.,  24.]]))

    Perform the test using the log-likelihood ratio (i.e. the "G-test")
    instead of Pearson's chi-squared statistic.

    >>> g, p, dof, expctd = chi2_contingency(obs, lambda_="log-likelihood")
    >>> g, p
    (2.7688587616781319, 0.25046668010954165)

    A four-way example (2 x 2 x 2 x 2):

    >>> obs = np.array(
    ...     [[[[12, 17],
    ...        [11, 16]],
    ...       [[11, 12],
    ...        [15, 16]]],
    ...      [[[23, 15],
    ...        [30, 22]],
    ...       [[14, 17],
    ...        [15, 16]]]])
    >>> chi2_contingency(obs)
    (8.7584514426741897,
     0.64417725029295503,
     11,
     array([[[[ 14.15462386,  14.15462386],
              [ 16.49423111,  16.49423111]],
             [[ 11.2461395 ,  11.2461395 ],
              [ 13.10500554,  13.10500554]]],
            [[[ 19.5591166 ,  19.5591166 ],
              [ 22.79202844,  22.79202844]],
             [[ 15.54012004,  15.54012004],
              [ 18.10873492,  18.10873492]]]]))
    """
    observed = np.asarray(observed)
    if np.any(observed < 0):
        raise ValueError("All values in `observed` must be nonnegative.")
    if observed.size == 0:
        raise ValueError("No data; `observed` has size 0.")

    expected = expected_freq(observed)
    if np.any(expected == 0):
        # Include one of the positions where expected is zero in
        # the exception message.
        zeropos = list(zip(*np.nonzero(expected == 0)))[0]
        raise ValueError("The internally computed table of expected "
                         "frequencies has a zero element at %s." % (zeropos,))

    # The degrees of freedom
    dof = expected.size - sum(expected.shape) + expected.ndim - 1

    if dof == 0:
        # Degenerate case; this occurs when `observed` is 1D (or, more
        # generally, when it has only one nontrivial dimension).  In this
        # case, we also have observed == expected, so chi2 is 0.
        chi2 = 0.0
        p = 1.0
    else:
        if dof == 1 and correction:
            # Adjust `observed` according to Yates' correction for continuity.
            observed = observed + 0.5 * np.sign(expected - observed)

        chi2, p = power_divergence(observed, expected,
                                   ddof=observed.size - 1 - dof, axis=None,
                                   lambda_=lambda_)

    return chi2, p, dof, expected
<commit_msg>DOC: fix import in stats.contingency refguide-check<commit_after>"""Some functions for working with contingency tables (i.e. cross tabulations).
"""

from __future__ import division, print_function, absolute_import

from functools import reduce
import numpy as np
from .stats import power_divergence

__all__ = ['margins', 'expected_freq', 'chi2_contingency']

def margins(a):
    """Return a list of the marginal sums of the array `a`.

    Parameters
    ----------
    a : ndarray
        The array for which to compute the marginal sums.

    Returns
    -------
    margsums : list of ndarrays
        A list of length `a.ndim`.  `margsums[k]` is the result
        of summing `a` over all axes except `k`; it has the same
        number of dimensions as `a`, but the length of each axis
        except axis `k` will be 1.

    Examples
    --------
    >>> a = np.arange(12).reshape(2, 6)
    >>> a
    array([[ 0,  1,  2,  3,  4,  5],
           [ 6,  7,  8,  9, 10, 11]])
    >>> from scipy.stats.contingency import margins
    >>> m0, m1 = margins(a)
    >>> m0
    array([[15],
           [51]])
    >>> m1
    array([[ 6,  8, 10, 12, 14, 16]])

    >>> b = np.arange(24).reshape(2,3,4)
    >>> m0, m1, m2 = margins(b)
    >>> m0
    array([[[ 66]],
           [[210]]])
    >>> m1
    array([[[ 60],
            [ 92],
            [124]]])
    >>> m2
    array([[[60, 66, 72, 78]]])
    """
    margsums = []
    ranged = list(range(a.ndim))
    for k in ranged:
        marg = np.apply_over_axes(np.sum, a, [j for j in ranged if j != k])
        margsums.append(marg)
    return margsums

def expected_freq(observed):
    """
    Compute the expected frequencies from a contingency table.

    Given an n-dimensional contingency table of observed frequencies,
    compute the expected frequencies for the table based on the marginal
    sums under the assumption that the groups associated with each
    dimension are independent.

    Parameters
    ----------
    observed : array_like
        The table of observed frequencies.  (While this function can handle
        a 1-D array, that case is trivial.  Generally `observed` is at
        least 2-D.)

    Returns
    -------
    expected : ndarray of float64
        The expected frequencies, based on the marginal sums of the table.
        Same shape as `observed`.

    Examples
    --------
    >>> observed = np.array([[10, 10, 20],[20, 20, 20]])
    >>> from scipy.stats.contingency import expected_freq
    >>> expected_freq(observed)
    array([[ 12.,  12.,  16.],
           [ 18.,  18.,  24.]])

    """
    # Typically `observed` is an integer array. If `observed` has a large
    # number of dimensions or holds large values, some of the following
    # computations may overflow, so we first switch to floating point.
    observed = np.asarray(observed, dtype=np.float64)

    # Create a list of the marginal sums.
    margsums = margins(observed)

    # Create the array of expected frequencies.  The shapes of the
    # marginal sums returned by apply_over_axes() are just what we
    # need for broadcasting in the following product.
    d = observed.ndim
    expected = reduce(np.multiply, margsums) / observed.sum() ** (d - 1)
    return expected

def chi2_contingency(observed, correction=True, lambda_=None):
    """Chi-square test of independence of variables in a contingency table.

    This function computes the chi-square statistic and p-value for the
    hypothesis test of independence of the observed frequencies in the
    contingency table [1]_ `observed`.  The expected frequencies are computed
    based on the marginal sums under the assumption of independence; see
    `scipy.stats.contingency.expected_freq`.  The number of degrees of
    freedom is (expressed using numpy functions and attributes)::

        dof = observed.size - sum(observed.shape) + observed.ndim - 1

    Parameters
    ----------
    observed : array_like
        The contingency table. The table contains the observed frequencies
        (i.e. number of occurrences) in each category.  In the two-dimensional
        case, the table is often described as an "R x C table".
    correction : bool, optional
        If True, *and* the degrees of freedom is 1, apply Yates' correction
        for continuity.  The effect of the correction is to adjust each
        observed value by 0.5 towards the corresponding expected value.
    lambda_ : float or str, optional.
        By default, the statistic computed in this test is Pearson's
        chi-squared statistic [2]_.  `lambda_` allows a statistic from the
        Cressie-Read power divergence family [3]_ to be used instead.  See
        `power_divergence` for details.

    Returns
    -------
    chi2 : float
        The test statistic.
    p : float
        The p-value of the test
    dof : int
        Degrees of freedom
    expected : ndarray, same shape as `observed`
        The expected frequencies, based on the marginal sums of the table.

    See Also
    --------
    contingency.expected_freq
    fisher_exact
    chisquare
    power_divergence

    Notes
    -----
    An often quoted guideline for the validity of this calculation is that
    the test should be used only if the observed and expected frequencies
    in each cell are at least 5.

    This is a test for the independence of different categories of a
    population. The test is only meaningful when the dimension of
    `observed` is two or more.  Applying the test to a one-dimensional
    table will always result in `expected` equal to `observed` and a
    chi-square statistic equal to 0.

    This function does not handle masked arrays, because the calculation
    does not make sense with missing values.

    Like stats.chisquare, this function computes a chi-square statistic;
    the convenience this function provides is to figure out the expected
    frequencies and degrees of freedom from the given contingency table.
    If these were already known, and if the Yates' correction was not
    required, one could use stats.chisquare.  That is, if one calls::

        chi2, p, dof, ex = chi2_contingency(obs, correction=False)

    then the following is true::

        (chi2, p) == stats.chisquare(obs.ravel(), f_exp=ex.ravel(),
                                     ddof=obs.size - 1 - dof)

    The `lambda_` argument was added in version 0.13.0 of scipy.

    References
    ----------
    .. [1] "Contingency table",
           https://en.wikipedia.org/wiki/Contingency_table
    .. [2] "Pearson's chi-squared test",
           https://en.wikipedia.org/wiki/Pearson%27s_chi-squared_test
    .. [3] Cressie, N. and Read, T. R. C., "Multinomial Goodness-of-Fit
           Tests", J. Royal Stat. Soc. Series B, Vol. 46, No. 3 (1984),
           pp. 440-464.

    Examples
    --------
    A two-way example (2 x 3):

    >>> from scipy.stats import chi2_contingency
    >>> obs = np.array([[10, 10, 20], [20, 20, 20]])
    >>> chi2_contingency(obs)
    (2.7777777777777777,
     0.24935220877729619,
     2,
     array([[ 12.,  12.,  16.],
            [ 18.,  18.,  24.]]))

    Perform the test using the log-likelihood ratio (i.e. the "G-test")
    instead of Pearson's chi-squared statistic.

    >>> g, p, dof, expctd = chi2_contingency(obs, lambda_="log-likelihood")
    >>> g, p
    (2.7688587616781319, 0.25046668010954165)

    A four-way example (2 x 2 x 2 x 2):

    >>> obs = np.array(
    ...     [[[[12, 17],
    ...        [11, 16]],
    ...       [[11, 12],
    ...        [15, 16]]],
    ...      [[[23, 15],
    ...        [30, 22]],
    ...       [[14, 17],
    ...        [15, 16]]]])
    >>> chi2_contingency(obs)
    (8.7584514426741897,
     0.64417725029295503,
     11,
     array([[[[ 14.15462386,  14.15462386],
              [ 16.49423111,  16.49423111]],
             [[ 11.2461395 ,  11.2461395 ],
              [ 13.10500554,  13.10500554]]],
            [[[ 19.5591166 ,  19.5591166 ],
              [ 22.79202844,  22.79202844]],
             [[ 15.54012004,  15.54012004],
              [ 18.10873492,  18.10873492]]]]))
    """
    observed = np.asarray(observed)
    if np.any(observed < 0):
        raise ValueError("All values in `observed` must be nonnegative.")
    if observed.size == 0:
        raise ValueError("No data; `observed` has size 0.")

    expected = expected_freq(observed)
    if np.any(expected == 0):
        # Include one of the positions where expected is zero in
        # the exception message.
        zeropos = list(zip(*np.nonzero(expected == 0)))[0]
        raise ValueError("The internally computed table of expected "
                         "frequencies has a zero element at %s." % (zeropos,))

    # The degrees of freedom
    dof = expected.size - sum(expected.shape) + expected.ndim - 1

    if dof == 0:
        # Degenerate case; this occurs when `observed` is 1D (or, more
        # generally, when it has only one nontrivial dimension).  In this
        # case, we also have observed == expected, so chi2 is 0.
        chi2 = 0.0
        p = 1.0
    else:
        if dof == 1 and correction:
            # Adjust `observed` according to Yates' correction for continuity.
            observed = observed + 0.5 * np.sign(expected - observed)

        chi2, p = power_divergence(observed, expected,
                                   ddof=observed.size - 1 - dof, axis=None,
                                   lambda_=lambda_)

    return chi2, p, dof, expected
<|endoftext|>


<commit_before>from lxml import etree

import schematicTransformations
import eagleLibrary

class SchematicParsingException(Exception):
  pass

class EaglePart:
  def __init__(self, partNode, schematic):
    self.attributes = {}

    self.xml = partNode
    self.name = partNode.get('name')
    self.value = partNode.get('value')

    self.libraryName = partNode.get('library')
    self.devicesetName = partNode.get('deviceset')
    self.deviceName = partNode.get('device')
    self.technologyName = partNode.get('technology')

    self.attributes = self._extractAttributes(schematic)
    self.netMap = self._extractNetMap(schematic)
    self.gates = self._extractGates(schematic)

    pass

  def _extractGates(self, schematic):
    result = {}
    queryString = ".//instance[@part='{name}']".format(name=self.name)
    for element in schematic.findall(queryString):
      result[element.get('gate')] = element
    return result

  def _extractAttributes(self, schematic):
    technology = self.getTechnologyValue(schematic, self.technologyName)

    if technology == None:
      technology = self.getTechnologyValue(schematic, "DEFAULT")

    if technology == None:
      technology = self.getTechnologyValue(schematic, "")

    if technology == None:
      raise SchematicParsingException("Can't find correct attributes for part {}".format(self.name))

    attributes = {}
    for element in technology.findall('attribute'):
      attributes[element.get('name')] = element.get('value')

    return attributes

  def getTechnologyValue(self, schematic, technologyName):
    technologyPathString = "./libraries/library[@name='{library}']/devicesets/deviceset[@name='{deviceset}']/devices/device[@name='{device}']/technologies/technology[@name='{technology}']"
    return schematic.find(
        technologyPathString.format(library=self.libraryName, deviceset=self.devicesetName, device=self.deviceName,
                                    technology=technologyName))

  def _extractNetMap(self, schematic):
    nets = schematic.findall('.//net')
    netMap = {}

    for net in nets:
      pinrefs = net.findall('.//pinref')
      for pinref in pinrefs:
        if pinref.get('part') == self.name:
          netMap[pinref.get('pin')] = net.get('name')

    return netMap

  def getSpiceNetlist(self):
    spicePrefix = self.getAttribute("SV_SPICE_PREFIX")

    if spicePrefix == None:
      return None

    spiceOrder = self.getAttribute("SV_SPICE_ORDER").split(';')

    netList = ''
    for pin in spiceOrder:
      netList += self.netMap[pin] + ' '

    return spicePrefix + self.name + ' ' + netList + self.value

  def getSpiceModel(self):
    spiceModel = self.getAttribute("SV_SPICE_MODEL")

    if spiceModel == None:
      return None

    return '.model ' + self.value + ' ' + spiceModel

  def getSpiceSupply(self):

    if self.libraryName not in ['supply1', 'supply2']:
      return None

    if 'GND' in self.devicesetName:
      return None

    voltage = self.devicesetName
    name = voltage.replace('+', 'P').replace('-', 'M')
    connection = ' '.join(self.netMap.values())

    return 'V' + name + ' ' + connection + ' GND dc ' + voltage + ' ac 0V'

  def getSpiceSubCircuit(self):
    subCircuitData = self.getAttribute("SV_SPICE_SUBCKT")

    if subCircuitData == None:
      return None

    subCircuitData = subCircuitData.replace("\\n", "\n")
    subCircuitData = subCircuitData.replace('%NAME%', self.value)
    return subCircuitData

  def getAttribute(self, attributeName):
    try:
      return self.attributes[attributeName]
    except KeyError:
      pass
    return None

  def rename(self, newName):
    self.name = newName
    self.xml.set('name', newName)
    for gate in self.gates.values():
      gate.set('part', newName)

  def getRotation(self):
    rotation = 0
    for gate in self.gates.values():
      rot = gate.get('rot')
      if rot != None:
        rotation = int(rot[1:])
    return rotation

  def getPosition(self):
    position = (0,0)
    for gate in self.gates.values():
      x = float(gate.get('x'))
      y = float(gate.get('y'))
      position = (x,y)
    return position

class EagleSchematic:
  def __init__(self, xmlString):
    tree =  etree.fromstring(xmlString, etree.XMLParser(remove_blank_text=True))

    self.xml = tree

    self.settings = tree.find('./drawing/settings')
    self.grid = tree.find('./drawing/grid')
    self.layers = tree.find('./drawing/layers')

    schematic = tree.find('./drawing/schematic')
    self.libraries = eagleLibrary.parseLibraries(self.xml)
    self.parts = self._parseParts(schematic)

  def _parseParts(self, rootNode):
    parts = {}
    for partNode in rootNode.findall('.//part'):
      part = EaglePart(partNode, rootNode)
      parts[part.name] = part

    return parts

  def getSpiceData(self):
    netLists = set()
    models = set()
    voltageSources = set()
    subCircuits = set()

    for part in self.parts.values():
      self.addIfNotNone(voltageSources, part.getSpiceSupply())
      self.addIfNotNone(netLists, part.getSpiceNetlist())
      self.addIfNotNone(models, part.getSpiceModel())
      self.addIfNotNone(subCircuits, part.getSpiceSubCircuit())

    netList = "\n".join(netLists)
    models = "\n".join(models)
    voltageSources = "\n".join(voltageSources)
    subCircuits = "\n".join(subCircuits)

    return voltageSources + "\n" + netList + "\n" + models + "\n" + subCircuits + "\n.end"

  def toString(self):
    return etree.tostring(self.xml, pretty_print=True)

  def replace(self, deviceSet, replacementSchematic):
    self._replaceLibraries(replacementSchematic.libraries)
    for oldPart in self.parts.values():
      if oldPart.devicesetName == deviceSet:
        newSchematic = replacementSchematic.deepCopy()

        rotation = schematicTransformations.SchematicRotation(oldPart.getRotation())
        translation = schematicTransformations.SchematicTranslation(oldPart.getPosition())

        newSchematic = rotation.transform(newSchematic)
        newSchematic = translation.transform(newSchematic)

        newSchematic.prefixParts(oldPart.name + '-')

        self._replaceDocu(newSchematic)
        self._replaceSinglePart(newSchematic, oldPart)
        self._replaceNets(newSchematic, oldPart)

    pass

  def prefixParts(self, prefix):
    for part in self.parts.values():
      oldName = part.name
      newName = prefix + oldName

      partRefs =  self.xml.findall(".//pinref[@part='{partName}']".format(partName=oldName))
      for ref in partRefs:
        ref.set('part', newName)
      part.rename(newName)

    self._rebuildPartTree()

  def _replaceLibraries(self, newLibraries):
    self.libraries = eagleLibrary.mergeLibraryLists(self.libraries, newLibraries)

    libraryNode = self.xml.find('./drawing/schematic/libraries')
    libraryNode.clear()
    for library in self.libraries.values():
      library = library.deepCopy()
      libraryNode.append(library.xml)

  def _replaceSinglePart(self, replacementSchematic, oldPart):
    self.parts.pop(oldPart.name)
    for newPart in replacementSchematic.parts.values():
      self.parts[newPart.name] = newPart

    self._rebuildPartTree()

  def _rebuildPartTree(self):
    partsNode = self.xml.find('./drawing/schematic/parts')
    partsNode.clear()
    for part in self.parts.values():
      partsNode.append(part.xml)
    self._replaceInstanceTree()

  def _replaceInstanceTree(self):
    instanceNode = self.xml.find('.//instances')
    instanceNode.clear()
    for part in self.parts.values():
      for instance in part.gates.values():
        instanceNode.append(instance)

  def _replaceNets(self, replacementSchematic, oldPart):
    replacedNets = []
    pinRefs =  self.xml.findall(".//pinref[@part='{partName}']".format(partName=oldPart.name))
    for refNode in pinRefs:
      pin = refNode.get('pin')
      replacementNet = replacementSchematic.getNet(pin)
      if replacementNet != None:
        replacementSegments = replacementNet.findall('.//segment')
        for segment in replacementSegments:
          for connection in segment.getchildren():
            refNode.getparent().append(connection)

        refNode.getparent().remove(refNode)
        replacedNets.append(pin)

    newNets = replacementSchematic.getNets()
    for net in newNets:
      oldName = net.get('name')
      if oldName not in replacedNets:
        newName = oldPart.name + '-' + oldName
        net.set('name', newName)
        self.addNet(net)

  def _replaceDocu(self, replacementSchematic):
    self.addDocu(replacementSchematic.getDocu())

  def getDocu(self):
    return self.xml.findall(".//plain/")

  def addDocu(self, newDocu):
    docuNode = self.xml.find(".//plain")
    for newNode in newDocu:
      docuNode.append(newNode)

  def getNets(self):
    return self.xml.findall(".//net")

  def getNet(self, netName):
    return self.xml.find(".//net[@name='{partName}']".format(partName=netName))

  def addNet(self, netNode):
    parent = self.xml.find(".//nets")
    parent.append(netNode)

  def getLibraries(self):
    return self.libraries

  def deepCopy(self):
    originalXMLString = self.toString()
    return EagleSchematic(originalXMLString)

  @staticmethod
  def addIfNotNone(collection, value):
    if value != None:
      collection.add(value)

<commit_msg>fixed spice netlist creation for nets with dashes in the name<commit_after>from lxml import etree

import schematicTransformations
import eagleLibrary

class SchematicParsingException(Exception):
  pass

class EaglePart:
  def __init__(self, partNode, schematic):
    self.attributes = {}

    self.xml = partNode
    self.name = partNode.get('name')
    self.value = partNode.get('value')

    self.libraryName = partNode.get('library')
    self.devicesetName = partNode.get('deviceset')
    self.deviceName = partNode.get('device')
    self.technologyName = partNode.get('technology')

    self.attributes = self._extractAttributes(schematic)
    self.netMap = self._extractNetMap(schematic)
    self.gates = self._extractGates(schematic)

    pass

  def _extractGates(self, schematic):
    result = {}
    queryString = ".//instance[@part='{name}']".format(name=self.name)
    for element in schematic.findall(queryString):
      result[element.get('gate')] = element
    return result

  def _extractAttributes(self, schematic):
    technology = self.getTechnologyValue(schematic, self.technologyName)

    if technology == None:
      technology = self.getTechnologyValue(schematic, "DEFAULT")

    if technology == None:
      technology = self.getTechnologyValue(schematic, "")

    if technology == None:
      raise SchematicParsingException("Can't find correct attributes for part {}".format(self.name))

    attributes = {}
    for element in technology.findall('attribute'):
      attributes[element.get('name')] = element.get('value')

    return attributes

  def getTechnologyValue(self, schematic, technologyName):
    technologyPathString = "./libraries/library[@name='{library}']/devicesets/deviceset[@name='{deviceset}']/devices/device[@name='{device}']/technologies/technology[@name='{technology}']"
    return schematic.find(
        technologyPathString.format(library=self.libraryName, deviceset=self.devicesetName, device=self.deviceName,
                                    technology=technologyName))

  def _extractNetMap(self, schematic):
    nets = schematic.findall('.//net')
    netMap = {}

    for net in nets:
      pinrefs = net.findall('.//pinref')
      for pinref in pinrefs:
        if pinref.get('part') == self.name:
          netMap[pinref.get('pin')] = net.get('name')

    return netMap

  def getSpiceNetlist(self):
    spicePrefix = self.getAttribute("SV_SPICE_PREFIX")

    if spicePrefix == None:
      return None

    spiceOrder = self.getAttribute("SV_SPICE_ORDER").split(';')

    netList = ''
    for pin in spiceOrder:
      net = self.netMap[pin]
      net = net.replace('-', '_')
      netList += net + ' '

    name = self.name.replace('-', '_')

    return spicePrefix + name + ' ' + netList + self.value

  def getSpiceModel(self):
    spiceModel = self.getAttribute("SV_SPICE_MODEL")

    if spiceModel == None:
      return None

    return '.model ' + self.value + ' ' + spiceModel

  def getSpiceSupply(self):

    if self.libraryName not in ['supply1', 'supply2']:
      return None

    if 'GND' in self.devicesetName:
      return None

    voltage = self.devicesetName
    name = voltage.replace('+', 'P').replace('-', 'M')
    connection = ' '.join(self.netMap.values())

    return 'V' + name + ' ' + connection + ' GND dc ' + voltage + ' ac 0V'

  def getSpiceSubCircuit(self):
    subCircuitData = self.getAttribute("SV_SPICE_SUBCKT")

    if subCircuitData == None:
      return None

    subCircuitData = subCircuitData.replace("\\n", "\n")
    subCircuitData = subCircuitData.replace('%NAME%', self.value)
    return subCircuitData

  def getAttribute(self, attributeName):
    try:
      return self.attributes[attributeName]
    except KeyError:
      pass
    return None

  def rename(self, newName):
    self.name = newName
    self.xml.set('name', newName)
    for gate in self.gates.values():
      gate.set('part', newName)

  def getRotation(self):
    rotation = 0
    for gate in self.gates.values():
      rot = gate.get('rot')
      if rot != None:
        rotation = int(rot[1:])
    return rotation

  def getPosition(self):
    position = (0,0)
    for gate in self.gates.values():
      x = float(gate.get('x'))
      y = float(gate.get('y'))
      position = (x,y)
    return position

class EagleSchematic:
  def __init__(self, xmlString):
    tree =  etree.fromstring(xmlString, etree.XMLParser(remove_blank_text=True))

    self.xml = tree

    self.settings = tree.find('./drawing/settings')
    self.grid = tree.find('./drawing/grid')
    self.layers = tree.find('./drawing/layers')

    schematic = tree.find('./drawing/schematic')
    self.libraries = eagleLibrary.parseLibraries(self.xml)
    self.parts = self._parseParts(schematic)

  def _parseParts(self, rootNode):
    parts = {}
    for partNode in rootNode.findall('.//part'):
      part = EaglePart(partNode, rootNode)
      parts[part.name] = part

    return parts

  def getSpiceData(self):
    netLists = set()
    models = set()
    voltageSources = set()
    subCircuits = set()

    for part in self.parts.values():
      self.addIfNotNone(voltageSources, part.getSpiceSupply())
      self.addIfNotNone(netLists, part.getSpiceNetlist())
      self.addIfNotNone(models, part.getSpiceModel())
      self.addIfNotNone(subCircuits, part.getSpiceSubCircuit())

    netList = "\n".join(netLists)
    models = "\n".join(models)
    voltageSources = "\n".join(voltageSources)
    subCircuits = "\n".join(subCircuits)

    return voltageSources + "\n" + netList + "\n" + models + "\n" + subCircuits + "\n.end"

  def toString(self):
    return etree.tostring(self.xml, pretty_print=True)

  def replace(self, deviceSet, replacementSchematic):
    self._replaceLibraries(replacementSchematic.libraries)
    for oldPart in self.parts.values():
      if oldPart.devicesetName == deviceSet:
        newSchematic = replacementSchematic.deepCopy()

        rotation = schematicTransformations.SchematicRotation(oldPart.getRotation())
        translation = schematicTransformations.SchematicTranslation(oldPart.getPosition())

        newSchematic = rotation.transform(newSchematic)
        newSchematic = translation.transform(newSchematic)

        newSchematic.prefixParts(oldPart.name + '-')

        self._replaceDocu(newSchematic)
        self._replaceSinglePart(newSchematic, oldPart)
        self._replaceNets(newSchematic, oldPart)

    pass

  def prefixParts(self, prefix):
    for part in self.parts.values():
      oldName = part.name
      newName = prefix + oldName

      partRefs =  self.xml.findall(".//pinref[@part='{partName}']".format(partName=oldName))
      for ref in partRefs:
        ref.set('part', newName)
      part.rename(newName)

    self._rebuildPartTree()

  def _replaceLibraries(self, newLibraries):
    self.libraries = eagleLibrary.mergeLibraryLists(self.libraries, newLibraries)

    libraryNode = self.xml.find('./drawing/schematic/libraries')
    libraryNode.clear()
    for library in self.libraries.values():
      library = library.deepCopy()
      libraryNode.append(library.xml)

  def _replaceSinglePart(self, replacementSchematic, oldPart):
    self.parts.pop(oldPart.name)
    for newPart in replacementSchematic.parts.values():
      self.parts[newPart.name] = newPart

    self._rebuildPartTree()

  def _rebuildPartTree(self):
    partsNode = self.xml.find('./drawing/schematic/parts')
    partsNode.clear()
    for part in self.parts.values():
      partsNode.append(part.xml)
    self._replaceInstanceTree()

  def _replaceInstanceTree(self):
    instanceNode = self.xml.find('.//instances')
    instanceNode.clear()
    for part in self.parts.values():
      for instance in part.gates.values():
        instanceNode.append(instance)

  def _replaceNets(self, replacementSchematic, oldPart):
    replacedNets = []
    pinRefs =  self.xml.findall(".//pinref[@part='{partName}']".format(partName=oldPart.name))
    for refNode in pinRefs:
      pin = refNode.get('pin')
      replacementNet = replacementSchematic.getNet(pin)
      if replacementNet != None:
        replacementSegments = replacementNet.findall('.//segment')
        for segment in replacementSegments:
          for connection in segment.getchildren():
            refNode.getparent().append(connection)

        refNode.getparent().remove(refNode)
        replacedNets.append(pin)

    newNets = replacementSchematic.getNets()
    for net in newNets:
      oldName = net.get('name')
      if oldName not in replacedNets:
        newName = oldPart.name + '-' + oldName
        net.set('name', newName)
        self.addNet(net)

  def _replaceDocu(self, replacementSchematic):
    self.addDocu(replacementSchematic.getDocu())

  def getDocu(self):
    return self.xml.findall(".//plain/")

  def addDocu(self, newDocu):
    docuNode = self.xml.find(".//plain")
    for newNode in newDocu:
      docuNode.append(newNode)

  def getNets(self):
    return self.xml.findall(".//net")

  def getNet(self, netName):
    return self.xml.find(".//net[@name='{partName}']".format(partName=netName))

  def addNet(self, netNode):
    parent = self.xml.find(".//nets")
    parent.append(netNode)

  def getLibraries(self):
    return self.libraries

  def deepCopy(self):
    originalXMLString = self.toString()
    return EagleSchematic(originalXMLString)

  @staticmethod
  def addIfNotNone(collection, value):
    if value != None:
      collection.add(value)

<|endoftext|>


<commit_before>from base import *

########## TEST SETTINGS
TEST_RUNNER = 'discover_runner.DiscoverRunner'
TEST_DISCOVER_TOP_LEVEL = SITE_ROOT
TEST_DISCOVER_ROOT = SITE_ROOT
TEST_DISCOVER_PATTERN = "test_*.py"
########## IN-MEMORY TEST DATABASE
DATABASES = {
    "default": {
        "ENGINE": "django.db.backends.sqlite3",
        "NAME": ":memory:",
        "USER": "",
        "PASSWORD": "",
        "HOST": "",
        "PORT": "",
    },
}

INSTALLED_APPS += (
    'django_jenkins',
)

JENKINS_TASKS = (
    'django_jenkins.tasks.run_pylint',
    'django_jenkins.tasks.with_coverage',
    'django_jenkins.tasks.django_tests',
)
<commit_msg>disable lint testing because of bug in astroid<commit_after>from base import *

########## TEST SETTINGS
TEST_RUNNER = 'discover_runner.DiscoverRunner'
TEST_DISCOVER_TOP_LEVEL = SITE_ROOT
TEST_DISCOVER_ROOT = SITE_ROOT
TEST_DISCOVER_PATTERN = "test_*.py"
########## IN-MEMORY TEST DATABASE
DATABASES = {
    "default": {
        "ENGINE": "django.db.backends.sqlite3",
        "NAME": ":memory:",
        "USER": "",
        "PASSWORD": "",
        "HOST": "",
        "PORT": "",
    },
}

INSTALLED_APPS += (
    'django_jenkins',
)

JENKINS_TASKS = (
#    'django_jenkins.tasks.run_pylint',
    'django_jenkins.tasks.with_coverage',
    'django_jenkins.tasks.django_tests',
)
<|endoftext|>


<commit_before>"""
A Lake Winnipeg Basin Information Network (BIN) harvester for the SHARE project

Example API request: http://130.179.67.140/api/3/action/package_search?q= (problematic)
http://130.179.67.140/api/3/action/current_package_list_with_resources (currently using)
It oddly returns 5 more datasets than all searchable ones on LWBIN data hub.

Known issues:
1 -- Five datasets can be searched but cannot be accessed via LWBIN.
Clicking on the searching result would result in linking to a redirected page like this:
http://130.179.67.140/user/login?came_from=http://130.179.67.140/dataset/mpca-surface-water-data-access-interactive-map
Within each dataset there are resouces that contain urls to source pages. For future work considering using resources
urls as canonical urls.
2 -- Resouces properties contained in raw metadata of the datasets are not added to the normalized metadata at this
point.
3 -- Single name contributors can be used as filters or an invalid query will be returned. Has nothing to do with scrapi but the frontend.
"""

from __future__ import unicode_literals

import json
import logging

from nameparser import HumanName
from dateutil.parser import parse

from scrapi import requests
from scrapi.base import JSONHarvester
from scrapi.linter.document import RawDocument
from scrapi.base.helpers import build_properties

logger = logging.getLogger(__name__)

organization_dict = [
    "organization", "Organization", "fund", "Fund", "canada", "Canada", "agriculture", "Agriculture", "commitee",
    "Commitee", "international", "International", "council", "Council", "office", "Office", "of", "Of"
]

customized_organization_dict = ["LWBIN", "CoCoRaHS", "USGS"]

def is_organization(name):
    """Return a boolean to indicate if the name passed to the function is an organization
    """
    words = name.split(' ')
    while len(words) > 0:
        word = words.pop()
        if word in organization_dict or word in customized_organization_dict:
            return True

def process_contributors(authors, emails):
    """Process authors and add author emails
    If multiple authors and one email, put email in a new author
    """

    authors = authors.strip().replace('', '').replace('', '')
    authors = authors.split(',')
    emails = emails.split(',')

    for author in authors:
        if is_organization(author):
            break
    else:
        if ' and ' in authors[-1] or ' et al.' in authors[-1]:
            split_name = authors.pop(-1).replace(' et al.', '').split(' and ')
            authors.extend(split_name)

    contributor_list = []
    append_emails = True
    if len(authors) > len(emails):
        append_emails = False  # if there are more authors than emails, do no append emails to authors; instead create new authors.

    for ind, person in enumerate(authors):
        name = HumanName(person)
        contributor = {
            'name': person,
            'givenName': name.first,
            'additionalName': name.middle,
            'familyName': name.last,
            'email': emails[ind] if append_emails else '',
            'sameAs': [],
        }
        contributor_list.append(contributor)

    if not append_emails:
        for email in emails:
            contributor = {
                'name': '',
                'givenName': '',
                'additionalName': '',
                'familyName': '',
                'email': email,
                'sameAs': [],
            }
            contributor_list.append(contributor)

    return contributor_list

def process_licenses(license_title, license_url, license_id):
    """Process licenses to comply with the noormalized schema
    """

    if not license_url:
        return []
    else:
        license = {
            'uri': license_url,
            'description': "{} ({})".format(license_title, license_id) or ""
        }

        return [license]

def construct_url(url, dataset_path, end_point):
    """Return a url that directs back to the page on LBWIN Data Hub instead of the source page.

    Keyword arguments:
    url -- host url
    dataset_path -- parent path of all datasets
    end_point -- name of datasets
    """

    return "/".join([url, dataset_path, end_point])

def process_object_uris(url, extras):
    """Extract doi from /extras, and return a list or object uris including /url and doi if it exists.
    """
    doi = ""
    for d in extras:
        if d['key'] == "DOI" or d['key'] == "DOI:":
            doi = d['value']
            break
    if doi == "":
        return [url]
    else:
        return [url, doi]

class LWBINHarvester(JSONHarvester):
    short_name = 'lwbin'
    long_name = 'Lake Winnipeg Basin Information Network'
    url = 'http://130.179.67.140'
    dataset_path = "dataset"   # dataset base url for constructing urls that go back to LWBIN instead of source pages.

    DEFAULT_ENCODING = 'UTF-8'

    record_encoding = None

    @property
    def schema(self):
        return {
            'title': ('/title', lambda x: x if x else ''),
            'description': ('/notes', lambda x: x[0] if (isinstance(x, list) and x) else x or ''),
            'providerUpdatedDateTime': ('/metadata_modified', lambda x: parse(x).isoformat()),
            'uris': {
                'canonicalUri': ('/name', lambda x: construct_url(self.url, self.dataset_path, x)),  # Construct new urls directing to LWBIN
                'objectUris': ('/url', '/extras', process_object_uris)  # Default urls from the metadata directing to source pages
            },
            'contributors': ('/author', '/author_email', process_contributors),
            'licenses': ('/license_title', '/license_url', '/license_id', process_licenses),
            'tags': ('/tags', lambda x: [tag['name'].lower() for tag in (x or [])]),
            'freeToRead': {
                'startDate': ('/isopen', '/metadata_created', lambda x, y: parse(y).date().isoformat() if x else None)
            },
            'otherProperties': build_properties(
                ('maintainer', '/maintainer'),
                ('maintainerEmail', '/maintainer_email'),
                ('revisionTimestamp', ('/revision_timestamp', lambda x: parse(x).isoformat())),
                ('id', '/id'),
                ('metadataCreated', ('/metadata_created', lambda x: parse(x).isoformat())),
                ('state', '/state'),
                ('version', '/version'),
                ('creatorUserId', '/creator_user_id'),
                ('type', '/type'),
                ('numberOfResources', '/num_resources'),
                ('numberOfTags', '/num_tags'),
                ('name', '/name'),
                ('groups', '/groups'),
            )
        }

    def harvest(self, start_date=None, end_date=None):
        """Returns a list of Rawdocuments (metadata)
        Searching by time is not supported by LWBIN CKAN API. all datasets have to be scanned each time.
        """

        base_url = 'http://130.179.67.140/api/3/action/current_package_list_with_resources'

        doc_list = []
        records = requests.get(base_url).json()['result']
        total = len(records)  # Total number of documents
        logger.info('{} documents to be harvested'.format(total))

        for record in records:
            doc_id = record['id']
            doc_list.append(RawDocument({
                'doc': json.dumps(record),
                'source': self.short_name,
                'docID': doc_id,
                'filetype': 'json'
            }))

        return doc_list
<commit_msg>update check organzations function<commit_after>"""
A Lake Winnipeg Basin Information Network (BIN) harvester for the SHARE project

Example API request: http://130.179.67.140/api/3/action/package_search?q= (problematic)
http://130.179.67.140/api/3/action/current_package_list_with_resources (currently using)
It oddly returns 5 more datasets than all searchable ones on LWBIN data hub.

Known issues:
1 -- Five datasets can be searched but cannot be accessed via LWBIN.
Clicking on the searching result would result in linking to a redirected page like this:
http://130.179.67.140/user/login?came_from=http://130.179.67.140/dataset/mpca-surface-water-data-access-interactive-map
Within each dataset there are resouces that contain urls to source pages. For future work considering using resources
urls as canonical urls.
2 -- Resouces properties contained in raw metadata of the datasets are not added to the normalized metadata at this
point.
3 -- Single name contributors can be used as filters or an invalid query will be returned. Has nothing to do with scrapi but the frontend.
"""

from __future__ import unicode_literals

import json
import logging

from nameparser import HumanName
from dateutil.parser import parse

from scrapi import requests
from scrapi.base import JSONHarvester
from scrapi.linter.document import RawDocument
from scrapi.base.helpers import build_properties

logger = logging.getLogger(__name__)

ORGANIZATIONS = (
    "organization", "Organization", "fund", "Fund", "canada", "Canada", "agriculture", "Agriculture", "commitee",
    "Commitee", "international", "International", "council", "Council", "office", "Office", "of", "Of",
    "observation", "Observation", "LWBIN", "CoCoRaHS", "USGS", "NSIDC"
)

def is_organization(name):
    """Return a boolean to indicate if the name passed to the function is an organization
    """
    words = name.split(' ')
    return any(word.strip(";") in ORGANIZATIONS for word in words)

def process_contributors(authors, emails):
    """Process authors and add author emails
    If multiple authors and one email, put email in a new author
    """

    authors = authors.strip().replace('', '').replace('', '')
    authors = authors.split(',')
    emails = emails.split(',')

    for author in authors:
        if is_organization(author):
            break
    else:
        if ' and ' in authors[-1] or ' et al.' in authors[-1]:
            split_name = authors.pop(-1).replace(' et al.', '').split(' and ')
            authors.extend(split_name)

    contributor_list = []
    append_emails = True
    if len(authors) > len(emails):
        append_emails = False  # if there are more authors than emails, do no append emails to authors; instead create new authors.

    for ind, person in enumerate(authors):
        name = HumanName(person)
        contributor = {
            'name': person,
            'givenName': name.first,
            'additionalName': name.middle,
            'familyName': name.last,
            'email': emails[ind] if append_emails else '',
            'sameAs': [],
        }
        contributor_list.append(contributor)

    if not append_emails:
        for email in emails:
            contributor = {
                'name': '',
                'givenName': '',
                'additionalName': '',
                'familyName': '',
                'email': email,
                'sameAs': [],
            }
            contributor_list.append(contributor)

    return contributor_list

def process_licenses(license_title, license_url, license_id):
    """Process licenses to comply with the noormalized schema
    """

    if not license_url:
        return []
    else:
        license = {
            'uri': license_url,
            'description': "{} ({})".format(license_title, license_id) or ""
        }

        return [license]

def construct_url(url, dataset_path, end_point):
    """Return a url that directs back to the page on LBWIN Data Hub instead of the source page.

    Keyword arguments:
    url -- host url
    dataset_path -- parent path of all datasets
    end_point -- name of datasets
    """

    return "/".join([url, dataset_path, end_point])

def process_object_uris(url, extras):
    """Extract doi from /extras, and return a list or object uris including /url and doi if it exists.
    """
    doi = ""
    for d in extras:
        if d['key'] == "DOI" or d['key'] == "DOI:":
            doi = d['value']
            break
    if doi == "":
        return [url]
    else:
        return [url, doi]

class LWBINHarvester(JSONHarvester):
    short_name = 'lwbin'
    long_name = 'Lake Winnipeg Basin Information Network'
    url = 'http://130.179.67.140'
    dataset_path = "dataset"   # dataset base url for constructing urls that go back to LWBIN instead of source pages.

    DEFAULT_ENCODING = 'UTF-8'

    record_encoding = None

    @property
    def schema(self):
        return {
            'title': ('/title', lambda x: x if x else ''),
            'description': ('/notes', lambda x: x[0] if (isinstance(x, list) and x) else x or ''),
            'providerUpdatedDateTime': ('/metadata_modified', lambda x: parse(x).isoformat()),
            'uris': {
                'canonicalUri': ('/name', lambda x: construct_url(self.url, self.dataset_path, x)),  # Construct new urls directing to LWBIN
                'objectUris': ('/url', '/extras', process_object_uris)  # Default urls from the metadata directing to source pages
            },
            'contributors': ('/author', '/author_email', process_contributors),
            'licenses': ('/license_title', '/license_url', '/license_id', process_licenses),
            'tags': ('/tags', lambda x: [tag['name'].lower() for tag in (x or [])]),
            'freeToRead': {
                'startDate': ('/isopen', '/metadata_created', lambda x, y: parse(y).date().isoformat() if x else None)
            },
            'otherProperties': build_properties(
                ('maintainer', '/maintainer'),
                ('maintainerEmail', '/maintainer_email'),
                ('revisionTimestamp', ('/revision_timestamp', lambda x: parse(x).isoformat())),
                ('id', '/id'),
                ('metadataCreated', ('/metadata_created', lambda x: parse(x).isoformat())),
                ('state', '/state'),
                ('version', '/version'),
                ('creatorUserId', '/creator_user_id'),
                ('type', '/type'),
                ('numberOfResources', '/num_resources'),
                ('numberOfTags', '/num_tags'),
                ('name', '/name'),
                ('groups', '/groups'),
            )
        }

    def harvest(self, start_date=None, end_date=None):
        """Returns a list of Rawdocuments (metadata)
        Searching by time is not supported by LWBIN CKAN API. all datasets have to be scanned each time.
        """

        base_url = 'http://130.179.67.140/api/3/action/current_package_list_with_resources'

        doc_list = []
        records = requests.get(base_url).json()['result']
        total = len(records)  # Total number of documents
        logger.info('{} documents to be harvested'.format(total))

        for record in records:
            doc_id = record['id']
            doc_list.append(RawDocument({
                'doc': json.dumps(record),
                'source': self.short_name,
                'docID': doc_id,
                'filetype': 'json'
            }))

        return doc_list
<|endoftext|>


<commit_before>import sys
import os
import json
from urllib2 import urlopen, Request, HTTPError
from urllib import quote

## You will need a API request token from GitHub:
## https://github.com/settings/tokens

if "GITHUB_MAP_TOKEN" in os.environ:
    TOKEN = os.environ['GITHUB_MAP_TOKEN']
else:
    ## Environment variables are better than hardcoding
    ## but if you want to you can put it here
    TOKEN = ""

def request(url, token):
    """ Send a HTTP request """
    try:
        req = Request(url)
        req.add_header('Authorization', 'token %s' % token)
        response = urlopen(req)
    except HTTPError as err:
        print "There was an HTTP error: " + str(err.code)
        print err.read()
        return
    return json.load(response)

def city_users(variation, country, token):
    """ Find the number of users in a city """

    github_url = "https://api.github.com/search/users?q=+location:"
    city_address = quote('"' + variation + ", " + country + '"')
    url = github_url + city_address
    print ""
    print "Getting...", variation + ", " + country
    print "URL: " + url

    total = int(request(url, token)["total_count"])

    return total

if __name__ == "__main__":
    if sys.argv[1]:
        print TOKEN
        PLACE = sys.argv[1].split(",")
        CITY = PLACE[0]
        COUNTRY = PLACE[1]
        print city_users(CITY, COUNTRY, TOKEN)

    else:
        print "No city/country passed in"

<commit_msg>Remove print token<commit_after>import sys
import os
import json
from urllib2 import urlopen, Request, HTTPError
from urllib import quote

## You will need a API request token from GitHub:
## https://github.com/settings/tokens

if "GITHUB_MAP_TOKEN" in os.environ:
    TOKEN = os.environ['GITHUB_MAP_TOKEN']
else:
    ## Environment variables are better than hardcoding
    ## but if you want to you can put it here
    TOKEN = ""

def request(url, token):
    """ Send a HTTP request """
    try:
        req = Request(url)
        req.add_header('Authorization', 'token %s' % token)
        response = urlopen(req)
    except HTTPError as err:
        print "There was an HTTP error: " + str(err.code)
        print err.read()
        return
    return json.load(response)

def city_users(variation, country, token):
    """ Find the number of users in a city """

    github_url = "https://api.github.com/search/users?q=+location:"
    city_address = quote('"' + variation + ", " + country + '"')
    url = github_url + city_address
    print ""
    print "Getting...", variation + ", " + country
    print "URL: " + url

    total = int(request(url, token)["total_count"])

    return total

if __name__ == "__main__":
    if sys.argv[1]:
        PLACE = sys.argv[1].split(",")
        CITY = PLACE[0]
        COUNTRY = PLACE[1]
        print city_users(CITY, COUNTRY, TOKEN)

    else:
        print "No city/country passed in"

<|endoftext|>