<commit_before># Copyright 2014-2016 The ODL development group
#
# This file is part of ODL.
#
# ODL is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# ODL is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with ODL.  If not, see <http://www.gnu.org/licenses/>.

"""Standardized tests for `Operator`'s."""

# Imports for common Python 2/3 codebase
from __future__ import print_function, division, absolute_import
from future import standard_library
standard_library.install_aliases()
from builtins import object

import numpy as np

from odl.diagnostics.examples import samples
from odl.operator import power_method_opnorm
from odl.util.testutils import FailCounter

__all__ = ('OperatorTest',)

class OperatorTest(object):

    """Automated tests for `Operator` implementations.

    This class allows users to automatically test various
    features of an Operator such as linearity, the adjoint definition and
    definition of the derivative.
    """

    def __init__(self, operator, operator_norm=None, verbose=True):
        """Initialize a new instance.

        Parameters
        ----------
        operator : `Operator`
            The operator to run tests on
        operator_norm : `float`
            The norm of the operator, used for error estimates
            can be estimated otherwise.
        """
        self.operator = operator
        self.verbose = False
        if operator_norm is None:
            self.operator_norm = self.norm()
        else:
            self.operator_norm = float(operator_norm)

        self.verbose = bool(verbose)

    def norm(self):
        """Estimate the operator norm of the operator.

        The norm is estimated by calculating

        ``A(x).norm() / x.norm()``

        for some nonzero ``x``

        Returns
        -------
        norm : `float`
            Estimate of operator norm

        References
        ----------
        Wikipedia article on `Operator norm
        <https://en.wikipedia.org/wiki/Operator_norm>`_.
        """
        if self.verbose:
            print('\n== Calculating operator norm ==\n')

        operator_norm = max(power_method_opnorm(self.operator, niter=2,
                                                xstart=x)
                            for name, x in samples(self.operator.domain)
                            if name != 'Zero')

        if self.verbose:
            print('Norm is at least: {}'.format(operator_norm))
        self.operator_norm = operator_norm
        return operator_norm

    def self_adjoint(self):
        """Verify ``<Ax, y> == <x, Ay>``."""
        left_inner_vals = []
        right_inner_vals = []

        with FailCounter(
                test_name='Verifying the identity <Ax, y> = <x, Ay>',
                err_msg='error = |<Ax, y> - <x, Ay>| / ||A|| ||x|| ||y||',
                verbose=self.verbose) as counter:

            for [name_x, x], [name_y, y] in samples(self.operator.domain,
                                                    self.operator.range):
                x_norm = x.norm()
                y_norm = y.norm()

                l_inner = self.operator(x).inner(y)
                r_inner = x.inner(self.operator(y))

                denom = self.operator_norm * x_norm * y_norm
                error = 0 if denom == 0 else abs(l_inner - r_inner) / denom

                # TODO: allow specification of tolerance
                if error > 0.00001:
                    counter.fail('x={:25s} y={:25s} : error={:6.5f}'
                                 ''.format(name_x, name_y, error))

                left_inner_vals.append(l_inner)
                right_inner_vals.append(r_inner)

        scale = np.polyfit(left_inner_vals, right_inner_vals, 1)[0]
        if self.verbose:
            print('\nThe adjoint seems to be scaled according to:')
            print('(x, Ay) / (Ax, y) = {}. Should be 1.0'.format(scale))

    def _adjoint_definition(self):
        """Verify ``<Ax, y> == <x, A^* y>``."""
        left_inner_vals = []
        right_inner_vals = []

        with FailCounter(
                test_name='Verifying the identity <Ax, y> = <x, A^T y>',
                err_msg='error = |<Ax, y< - <x, A^* y>| / ||A|| ||x|| ||y||',
                verbose=self.verbose) as counter:

            for [name_x, x], [name_y, y] in samples(self.operator.domain,
                                                    self.operator.range):
                x_norm = x.norm()
                y_norm = y.norm()

                l_inner = self.operator(x).inner(y)
                r_inner = x.inner(self.operator.adjoint(y))

                denom = self.operator_norm * x_norm * y_norm
                error = 0 if denom == 0 else abs(l_inner - r_inner) / denom

                # TODO: allow specification of tolerance
                if error > 0.00001:
                    counter.fail('x={:25s} y={:25s} : error={:6.5f}'
                                 ''.format(name_x, name_y, error))

                left_inner_vals.append(l_inner)
                right_inner_vals.append(r_inner)

        scale = np.polyfit(left_inner_vals, right_inner_vals, 1)[0]
        if self.verbose:
            print('\nThe adjoint seems to be scaled according to:')
            print('(x, A^T y) / (Ax, y) = {}. Should be 1.0'.format(scale))

    def _adjoint_of_adjoint(self):
        """Verify ``(A^*)^* == A``"""
        try:
            self.operator.adjoint.adjoint
        except AttributeError:
            print('A^* has no adjoint')
            return

        if self.operator.adjoint.adjoint is self.operator:
            if self.verbose:
                print('(A^*)^* == A')
            return

        with FailCounter(
                test_name='\nVerifying the identity Ax = (A^*)^* x',
                err_msg='error = ||Ax - (A^*)^* x|| / ||A|| ||x||',
                verbose=self.verbose) as counter:
            for [name_x, x] in self.operator.domain.examples:
                opx = self.operator(x)
                op_adj_adj_x = self.operator.adjoint.adjoint(x)

                denom = self.operator_norm * x.norm()
                if denom == 0:
                    error = 0
                else:
                    error = (opx - op_adj_adj_x).norm() / denom
                # TODO: allow specification of tolerance
                if error > 0.00001:
                    counter.fail('x={:25s} : error={:6.5f}'
                                 ''.format(name_x, error))

    def adjoint(self):
        """Verify that `Operator.adjoint` works appropriately.

        References
        ----------
        Wikipedia article on `Adjoint
        <https://en.wikipedia.org/wiki/Adjoint>`_.
        """
        try:
            self.operator.adjoint
        except NotImplementedError:
            print('Operator has no adjoint')
            return

        if self.verbose:
            print('\n== Verifying operator adjoint ==\n')

        domain_range_ok = True
        if self.operator.domain != self.operator.adjoint.range:
            print('*** ERROR: A.domain != A.adjoint.range ***')
            domain_range_ok = False

        if self.operator.range != self.operator.adjoint.domain:
            print('*** ERROR: A.range != A.adjoint.domain ***')
            domain_range_ok = False

        if domain_range_ok:
            if self.verbose:
                print('Domain and range of adjoint are OK.')
        else:
            print('Domain and range of adjoint are not OK, exiting.')
            return

        self._adjoint_definition()
        self._adjoint_of_adjoint()

    def _derivative_convergence(self):
        """Verify that the derivative is a first-order approximation.

        The code verifies if

            ``||A(x+c*p) - A(x) - A'(x)(c*p)|| / c = o(c)``

        for ``c --> 0``.
        """
        with FailCounter(
                test_name='Verifying that derivative is a first-order '
                          'approximation',
                err_msg="error = inf_c ||A(x+c*p)-A(x)-A'(x)(c*p)|| / c",
                verbose=self.verbose) as counter:
            for [name_x, x], [name_dx, dx] in samples(self.operator.domain,
                                                      self.operator.domain):
                # Precompute some values
                deriv = self.operator.derivative(x)
                derivdx = deriv(dx)
                opx = self.operator(x)

                c = 1e-4  # initial step
                derivative_ok = False

                minerror = float('inf')
                while c > 1e-14:
                    exact_step = self.operator(x + dx * c) - opx
                    expected_step = c * derivdx
                    err = (exact_step - expected_step).norm() / c

                    # TODO: allow specification of tolerance
                    if err < 1e-4:
                        derivative_ok = True
                        break
                    else:
                        minerror = min(minerror, err)

                    c /= 10.0

                if not derivative_ok:
                    counter.fail('x={:15s} p={:15s}, error={}'
                                 ''.format(name_x, name_dx, minerror))

    def derivative(self):
        """Verify that `Operator.derivative` works appropriately.

        The code verifies if

            ``||A(x+c*p) - A(x) - A'(x)(c*p)|| / c = o(c)``

        for ``c --> 0`` using a selection of elements ``x`` and ``p``.

        References
        ----------
        Wikipedia article on `Derivative
        <https://en.wikipedia.org/wiki/Derivative>`_.
        Wikipedia article on `Frechet derivative
        <https://en.wikipedia.org/wiki/Fr%C3%A9chet_derivative>`_.
        """

        if self.verbose:
            print('\n== Verifying operator derivative  ==')
        try:
            deriv = self.operator.derivative(self.operator.domain.zero())

            if not deriv.is_linear:
                print('Derivative is not a linear operator')
                return
        except NotImplementedError:
            print('Operator has no derivative')
            return

        if self.operator.is_linear and deriv is self.operator:
            if self.verbose:
                print('A is linear and A.derivative is A')
            return

        self._derivative_convergence()

    def _scale_invariance(self):
        """Verify ``A(c*x) = c * A(x)``."""
        with FailCounter(
                test_name='Verifying homogeneity under scalar multiplication',
                err_msg='error = ||A(c*x)-c*A(x)|| / |c| ||A|| ||x||',
                verbose=self.verbose) as counter:
            for [name_x, x], [_, scale] in samples(self.operator.domain,
                                                   self.operator.domain.field):
                opx = self.operator(x)
                scaled_opx = self.operator(scale * x)

                denom = self.operator_norm * scale * x.norm()
                error = (0 if denom == 0
                         else (scaled_opx - opx * scale).norm() / denom)

                # TODO: allow specification of tolerance
                if error > 0.00001:
                    counter.fail('x={:25s} scale={:7.2f} error={:6.5f}'
                                 ''.format(name_x, scale, error))

    def _addition_invariance(self):
        """Verify ``A(x+y) = A(x) + A(y)``."""
        with FailCounter(
                test_name='Verifying distributivity under vector addition',
                err_msg='error = ||A(x+y) - A(x) - A(y)|| / '
                        '||A||(||x|| + ||y||)',
                verbose=self.verbose) as counter:
            for [name_x, x], [name_y, y] in samples(self.operator.domain,
                                                    self.operator.domain):
                opx = self.operator(x)
                opy = self.operator(y)
                opxy = self.operator(x + y)

                denom = self.operator_norm * (x.norm() + y.norm())
                error = (0 if denom == 0
                         else (opxy - opx - opy).norm() / denom)

                # TODO: allow specification of tolerance
                if error > 0.00001:
                    counter.fail('x={:25s} y={:25s} error={:6.5f}'
                                 ''.format(name_x, name_y, error))

    def linear(self):
        """Verify that the operator is actually linear."""
        if not self.operator.is_linear:
            print('Operator is not linear')
            return

        if self.verbose:
            print('\n== Verifying operator linearity ==\n')

        # Test if zero gives zero
        result = self.operator(self.operator.domain.zero())
        result_norm = result.norm()
        if result_norm != 0.0:
            print("||A(0)||={:6.5f}. Should be 0.0000".format(result_norm))

        self._scale_invariance()
        self._addition_invariance()

    def run_tests(self):
        """Run all tests on this operator."""
        if self.verbose:
            print('\n== RUNNING ALL TESTS ==')
            print('Operator = {}'.format(self.operator))

        self.norm()

        if self.operator.is_linear:
            self.linear()
            self.adjoint()
        else:
            self.derivative()

    def __str__(self):
        return '{}({})'.format(self.__class__.__name__, self.operator)

    def __repr__(self):
        return '{}({!r})'.format(self.__class__.__name__, self.operator)

if __name__ == '__main__':
    # pylint: disable=wrong-import-position

    import odl
    space = odl.uniform_discr([0, 0], [1, 1], [3, 3])
    # Linear operator
    I = odl.IdentityOperator(space)
    OperatorTest(I).run_tests()

    # Nonlinear operator op(x) = x**4
    op = odl.PowerOperator(space, 4)
    OperatorTest(op).run_tests()
<commit_msg>ENH: add logging method to OperatorTest<commit_after># Copyright 2014-2016 The ODL development group
#
# This file is part of ODL.
#
# ODL is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# ODL is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with ODL.  If not, see <http://www.gnu.org/licenses/>.

"""Standardized tests for `Operator`'s."""

# Imports for common Python 2/3 codebase
from __future__ import print_function, division, absolute_import
from future import standard_library
standard_library.install_aliases()
from builtins import object

import numpy as np

from odl.diagnostics.examples import samples
from odl.operator import power_method_opnorm
from odl.util.testutils import FailCounter

__all__ = ('OperatorTest',)

class OperatorTest(object):

    """Automated tests for `Operator` implementations.

    This class allows users to automatically test various
    features of an Operator such as linearity, the adjoint definition and
    definition of the derivative.
    """

    def __init__(self, operator, operator_norm=None, verbose=True):
        """Initialize a new instance.

        Parameters
        ----------
        operator : `Operator`
            The operator to run tests on
        operator_norm : `float`
            The norm of the operator, used for error estimates
            can be estimated otherwise.
        """
        self.operator = operator
        self.verbose = False
        if operator_norm is None:
            self.operator_norm = self.norm()
        else:
            self.operator_norm = float(operator_norm)

        self.verbose = bool(verbose)

    def log(self, message):
        """Print message if ``self.verbose == True``."""
        if self.verbose:
            print(message)

    def norm(self):
        """Estimate the operator norm of the operator.

        The norm is estimated by calculating

        ``A(x).norm() / x.norm()``

        for some nonzero ``x``

        Returns
        -------
        norm : `float`
            Estimate of operator norm

        References
        ----------
        Wikipedia article on `Operator norm
        <https://en.wikipedia.org/wiki/Operator_norm>`_.
        """
        self.log('\n== Calculating operator norm ==\n')

        operator_norm = max(power_method_opnorm(self.operator, niter=2,
                                                xstart=x)
                            for name, x in samples(self.operator.domain)
                            if name != 'Zero')

        self.log('Norm is at least: {}'.format(operator_norm))
        self.operator_norm = operator_norm
        return operator_norm

    def self_adjoint(self):
        """Verify ``<Ax, y> == <x, Ay>``."""
        left_inner_vals = []
        right_inner_vals = []

        with FailCounter(
                test_name='Verifying the identity <Ax, y> = <x, Ay>',
                err_msg='error = |<Ax, y> - <x, Ay>| / ||A|| ||x|| ||y||',
                verbose=self.verbose) as counter:

            for [name_x, x], [name_y, y] in samples(self.operator.domain,
                                                    self.operator.range):
                x_norm = x.norm()
                y_norm = y.norm()

                l_inner = self.operator(x).inner(y)
                r_inner = x.inner(self.operator(y))

                denom = self.operator_norm * x_norm * y_norm
                error = 0 if denom == 0 else abs(l_inner - r_inner) / denom

                # TODO: allow specification of tolerance
                if error > 0.00001:
                    counter.fail('x={:25s} y={:25s} : error={:6.5f}'
                                 ''.format(name_x, name_y, error))

                left_inner_vals.append(l_inner)
                right_inner_vals.append(r_inner)

        scale = np.polyfit(left_inner_vals, right_inner_vals, 1)[0]
        self.log('\nThe adjoint seems to be scaled according to:')
        self.log('(x, Ay) / (Ax, y) = {}. Should be 1.0'.format(scale))

    def _adjoint_definition(self):
        """Verify ``<Ax, y> == <x, A^* y>``."""
        left_inner_vals = []
        right_inner_vals = []

        with FailCounter(
                test_name='Verifying the identity <Ax, y> = <x, A^T y>',
                err_msg='error = |<Ax, y< - <x, A^* y>| / ||A|| ||x|| ||y||',
                verbose=self.verbose) as counter:

            for [name_x, x], [name_y, y] in samples(self.operator.domain,
                                                    self.operator.range):
                x_norm = x.norm()
                y_norm = y.norm()

                l_inner = self.operator(x).inner(y)
                r_inner = x.inner(self.operator.adjoint(y))

                denom = self.operator_norm * x_norm * y_norm
                error = 0 if denom == 0 else abs(l_inner - r_inner) / denom

                # TODO: allow specification of tolerance
                if error > 0.00001:
                    counter.fail('x={:25s} y={:25s} : error={:6.5f}'
                                 ''.format(name_x, name_y, error))

                left_inner_vals.append(l_inner)
                right_inner_vals.append(r_inner)

        scale = np.polyfit(left_inner_vals, right_inner_vals, 1)[0]
        self.log('\nThe adjoint seems to be scaled according to:')
        self.log('(x, A^T y) / (Ax, y) = {}. Should be 1.0'.format(scale))

    def _adjoint_of_adjoint(self):
        """Verify ``(A^*)^* == A``"""
        try:
            self.operator.adjoint.adjoint
        except AttributeError:
            print('A^* has no adjoint')
            return

        if self.operator.adjoint.adjoint is self.operator:
            self.log('(A^*)^* == A')
            return

        with FailCounter(
                test_name='\nVerifying the identity Ax = (A^*)^* x',
                err_msg='error = ||Ax - (A^*)^* x|| / ||A|| ||x||',
                verbose=self.verbose) as counter:
            for [name_x, x] in self.operator.domain.examples:
                opx = self.operator(x)
                op_adj_adj_x = self.operator.adjoint.adjoint(x)

                denom = self.operator_norm * x.norm()
                if denom == 0:
                    error = 0
                else:
                    error = (opx - op_adj_adj_x).norm() / denom
                # TODO: allow specification of tolerance
                if error > 0.00001:
                    counter.fail('x={:25s} : error={:6.5f}'
                                 ''.format(name_x, error))

    def adjoint(self):
        """Verify that `Operator.adjoint` works appropriately.

        References
        ----------
        Wikipedia article on `Adjoint
        <https://en.wikipedia.org/wiki/Adjoint>`_.
        """
        try:
            self.operator.adjoint
        except NotImplementedError:
            print('Operator has no adjoint')
            return

        self.log('\n== Verifying operator adjoint ==\n')

        domain_range_ok = True
        if self.operator.domain != self.operator.adjoint.range:
            print('*** ERROR: A.domain != A.adjoint.range ***')
            domain_range_ok = False

        if self.operator.range != self.operator.adjoint.domain:
            print('*** ERROR: A.range != A.adjoint.domain ***')
            domain_range_ok = False

        if domain_range_ok:
            self.log('Domain and range of adjoint are OK.')
        else:
            print('Domain and range of adjoint are not OK, exiting.')
            return

        self._adjoint_definition()
        self._adjoint_of_adjoint()

    def _derivative_convergence(self):
        """Verify that the derivative is a first-order approximation.

        The code verifies if

            ``||A(x+c*p) - A(x) - A'(x)(c*p)|| / c = o(c)``

        for ``c --> 0``.
        """
        with FailCounter(
                test_name='Verifying that derivative is a first-order '
                          'approximation',
                err_msg="error = inf_c ||A(x+c*p)-A(x)-A'(x)(c*p)|| / c",
                verbose=self.verbose) as counter:
            for [name_x, x], [name_dx, dx] in samples(self.operator.domain,
                                                      self.operator.domain):
                # Precompute some values
                deriv = self.operator.derivative(x)
                derivdx = deriv(dx)
                opx = self.operator(x)

                c = 1e-4  # initial step
                derivative_ok = False

                minerror = float('inf')
                while c > 1e-14:
                    exact_step = self.operator(x + dx * c) - opx
                    expected_step = c * derivdx
                    err = (exact_step - expected_step).norm() / c

                    # TODO: allow specification of tolerance
                    if err < 1e-4:
                        derivative_ok = True
                        break
                    else:
                        minerror = min(minerror, err)

                    c /= 10.0

                if not derivative_ok:
                    counter.fail('x={:15s} p={:15s}, error={}'
                                 ''.format(name_x, name_dx, minerror))

    def derivative(self):
        """Verify that `Operator.derivative` works appropriately.

        The code verifies if

            ``||A(x+c*p) - A(x) - A'(x)(c*p)|| / c = o(c)``

        for ``c --> 0`` using a selection of elements ``x`` and ``p``.

        References
        ----------
        Wikipedia article on `Derivative
        <https://en.wikipedia.org/wiki/Derivative>`_.
        Wikipedia article on `Frechet derivative
        <https://en.wikipedia.org/wiki/Fr%C3%A9chet_derivative>`_.
        """
        self.log('\n== Verifying operator derivative  ==')

        try:
            deriv = self.operator.derivative(self.operator.domain.zero())

            if not deriv.is_linear:
                print('Derivative is not a linear operator')
                return
        except NotImplementedError:
            print('Operator has no derivative')
            return

        if self.operator.is_linear and deriv is self.operator:
            self.log('A is linear and A.derivative is A')
            return

        self._derivative_convergence()

    def _scale_invariance(self):
        """Verify ``A(c*x) = c * A(x)``."""
        with FailCounter(
                test_name='Verifying homogeneity under scalar multiplication',
                err_msg='error = ||A(c*x)-c*A(x)|| / |c| ||A|| ||x||',
                verbose=self.verbose) as counter:
            for [name_x, x], [_, scale] in samples(self.operator.domain,
                                                   self.operator.domain.field):
                opx = self.operator(x)
                scaled_opx = self.operator(scale * x)

                denom = self.operator_norm * scale * x.norm()
                error = (0 if denom == 0
                         else (scaled_opx - opx * scale).norm() / denom)

                # TODO: allow specification of tolerance
                if error > 0.00001:
                    counter.fail('x={:25s} scale={:7.2f} error={:6.5f}'
                                 ''.format(name_x, scale, error))

    def _addition_invariance(self):
        """Verify ``A(x+y) = A(x) + A(y)``."""
        with FailCounter(
                test_name='Verifying distributivity under vector addition',
                err_msg='error = ||A(x+y) - A(x) - A(y)|| / '
                        '||A||(||x|| + ||y||)',
                verbose=self.verbose) as counter:
            for [name_x, x], [name_y, y] in samples(self.operator.domain,
                                                    self.operator.domain):
                opx = self.operator(x)
                opy = self.operator(y)
                opxy = self.operator(x + y)

                denom = self.operator_norm * (x.norm() + y.norm())
                error = (0 if denom == 0
                         else (opxy - opx - opy).norm() / denom)

                # TODO: allow specification of tolerance
                if error > 0.00001:
                    counter.fail('x={:25s} y={:25s} error={:6.5f}'
                                 ''.format(name_x, name_y, error))

    def linear(self):
        """Verify that the operator is actually linear."""
        if not self.operator.is_linear:
            print('Operator is not linear')
            return

        self.log('\n== Verifying operator linearity ==\n')

        # Test if zero gives zero
        result = self.operator(self.operator.domain.zero())
        result_norm = result.norm()
        if result_norm != 0.0:
            print("||A(0)||={:6.5f}. Should be 0.0000".format(result_norm))

        self._scale_invariance()
        self._addition_invariance()

    def run_tests(self):
        """Run all tests on this operator."""
        print('\n== RUNNING ALL TESTS ==')
        print('Operator = {}'.format(self.operator))

        self.norm()

        if self.operator.is_linear:
            self.linear()
            self.adjoint()
        else:
            self.derivative()

    def __str__(self):
        return '{}({})'.format(self.__class__.__name__, self.operator)

    def __repr__(self):
        return '{}({!r})'.format(self.__class__.__name__, self.operator)

if __name__ == '__main__':
    # pylint: disable=wrong-import-position

    import odl
    space = odl.uniform_discr([0, 0], [1, 1], [3, 3])
    # Linear operator
    I = odl.IdentityOperator(space)
    OperatorTest(I, verbose=False).run_tests()

    # Nonlinear operator op(x) = x**4
    op = odl.PowerOperator(space, 4)
    OperatorTest(op).run_tests()
<|endoftext|>


<commit_before>from collections import namedtuple

from sparsereg.model.ffx import _path_is_saturated, _path_is_overfit

def test__path_is_saturated():
    model = namedtuple("Model", ["train_score_"])
    models = [model(1) for _ in range(10)]

    assert not _path_is_saturated(models, n_tail=len(models) + 1)
    assert _path_is_saturated(models, n_tail=1)
    assert _path_is_saturated(models, n_tail=5)

# def test__path_is_overfit():
#     model = namedtuple("Model", ["test_score_"])
#     models = [model(-i) for i in range(10)]
#
#     assert not _path_is_overfit(models)
#     assert not _path_is_overfit([models[0]])
<commit_msg>fix broken import<commit_after>from collections import namedtuple

from sparsereg.model.ffx import _path_is_saturated #, _path_is_overfit

def test__path_is_saturated():
    model = namedtuple("Model", ["train_score_"])
    models = [model(1) for _ in range(10)]

    assert not _path_is_saturated(models, n_tail=len(models) + 1)
    assert _path_is_saturated(models, n_tail=1)
    assert _path_is_saturated(models, n_tail=5)

# def test__path_is_overfit():
#     model = namedtuple("Model", ["test_score_"])
#     models = [model(-i) for i in range(10)]
#
#     assert not _path_is_overfit(models)
#     assert not _path_is_overfit([models[0]])
<|endoftext|>


<commit_before>from django.contrib.auth.models import User

from selectable.base import ModelLookup
from selectable.registry import registry

from example.core.models import Fruit, City

class FruitLookup(ModelLookup):
    model = Fruit
    search_field = 'name__icontains'

registry.register(FruitLookup)

class OwnerLookup(ModelLookup):
    model = User
    search_field = 'username__icontains'

registry.register(OwnerLookup)

class CityLookup(ModelLookup):
    model = City
    search_field = 'name__icontains'

    def get_query(self, request, term):
        results = super(CityLookup, self).get_query(request, term)
        state = request.GET.get('state', '')
        if state:
            results = results.filter(state=state)
        return results

    def get_item_label(self, item):
        return u"%s, %s" % (item.name, item.state)

registry.register(CityLookup)
<commit_msg>Change example lookups to new style.<commit_after>from django.contrib.auth.models import User

from selectable.base import ModelLookup
from selectable.registry import registry

from example.core.models import Fruit, City

class FruitLookup(ModelLookup):
    model = Fruit
    search_fields = ('name__icontains', )

registry.register(FruitLookup)

class OwnerLookup(ModelLookup):
    model = User
    search_fields = ('username__icontains', )

registry.register(OwnerLookup)

class CityLookup(ModelLookup):
    model = City
    search_fields = ('name__icontains', )

    def get_query(self, request, term):
        results = super(CityLookup, self).get_query(request, term)
        state = request.GET.get('state', '')
        if state:
            results = results.filter(state=state)
        return results

    def get_item_label(self, item):
        return u"%s, %s" % (item.name, item.state)

registry.register(CityLookup)
<|endoftext|>


<commit_before>import threading
import socket
import select
import logging
import inspect
import errno
from datetime import datetime

from jsonrpc import *
from errors import *

class RpcServer(object):
    """
    Flexible RPC server that runs in a Python thread. Wraps any Python object
    and allows remote machines to call functions and receive returned data.

    :param logger: Logger instance if you wish to override the internal instance
    :type logger: Logging.logger
    :param port: Port to attach socket to
    :type port: int
    :param name: Internal name for the RPC server thread
    :type name: str

    .. note::

        Method calls to functions that begin with an underscore are considered
        protected and will not be invoked
    """

    # Exclusive access times out after 60 seconds
    EXCLUSIVE_TIMEOUT = 60.0

    _identity = 'JSON-RPC/2.0'

    def __init__(self, **kwargs):
        self.logger = kwargs.get('logger', logging)
        self.port = kwargs.get('port', 0)
        self.name = kwargs.get('name', 'RPCServer')

        # RPC State Variables
        self.rpc_objects = []

        self.e_alive = threading.Event()
        self.rpc_lock = threading.Lock()
        self.rpc_startTime = datetime.now()

        # Attempt to bind a socket
        try:
            srv_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            srv_socket.bind(('',self.port))
            srv_socket.listen(5)
            srv_socket.setblocking(0)

            # Update port if randomly assigned
            _, self.port = srv_socket.getsockname()

        except socket.error as e:
            if e.errno == errno.EADDRINUSE:
                # Server is already running
                raise RpcServerPortInUse

            else:
                raise

        self.e_alive.set()

        self.__rpc_thread = threading.Thread(name=self.name, target=self.__thread_run, args=(srv_socket,))
        self.__rpc_thread.start()

    def __thread_run(self, srv_socket):
        _, self.port = srv_socket.getsockname()

        self.logger.debug('[%s] RPC Server started on port %i', self.name, self.port)

        self.rpc_startTime = datetime.now()

        while self.e_alive.isSet():
            # Service Socket
            try:
                ready_to_read,_,_ = select.select([srv_socket], [], [], 1.0)

                if srv_socket in ready_to_read:
                    # Spawn a new thread to service the connection
                    connection, address = srv_socket.accept()

                    # Spawn a new thread to service the connection
                    connThread = RpcConnection(server=self,
                                               socket=connection,
                                               logger=self.logger)

                    connThread.start()

            except:
                self.logger.exception('RPC Server Socket Handler Exception')

            # Clean up old connections
            # TODO

        srv_socket.close()

        self.logger.debug('[%s] RPC Server stopped', self.name)

    #===========================================================================
    # Server Management
    #===========================================================================

    def getName(self):
        return self.name

    def registerObject(self, reg_obj):
        self.rpc_objects.append(reg_obj)

    def unregisterObject(self, reg_obj):
        self.rpc_objects.remove(reg_obj)

    #===========================================================================
    # Methods
    #===========================================================================

    def findMethod(self, method):
        """
        Return a bound method after searching through the server methods and the
        methods of all registered objects
        """
        # Disallow access to methods that start with an underscore
        if method == '' or method.startswith('_'):
            # Invalid - Protected Method
            self.logger.warning('RPC Request Denied (Protected Method): %s', method)
            raise AttributeError

        elif method.startswith('rpc'):
            # Valid - RPC Server method
            return self._getMethod(self, method)

        else:
            # Check registered objects
            for reg_obj in self.rpc_objects:
                try:
                    return self._getMethod(reg_obj, method)
                except AttributeError:
                    pass

            # Unable to find method
            raise RpcMethodNotFound()

    def _getMethod(self, obj, method):
        test_method = getattr(obj, method)
        if inspect.ismethod(test_method):
            return test_method
        else:
            raise AttributeError

    #===========================================================================
    # RPC Functions
    #===========================================================================

    def rpc_getMethods(self, address=None):
        """
        Get a list of valid methods in the registered objects. Protected methods
        that begin with an underscore ('_') are not included.

        :returns: list of strings
        """
        # Catalog methods
        self.validMethods = []

        target_members = inspect.getmembers(self)

        for attr, val in target_members:
            if inspect.ismethod(val) and attr.startswith('rpc'):
                self.validMethods.append(attr)

        for reg_obj in self.rpc_objects:
            target_members = inspect.getmembers(reg_obj)

            for attr, val in target_members:
                if inspect.ismethod(val) and not attr.startswith('_'):
                    self.validMethods.append(attr)

        return self.validMethods

    def rpc_isRunning(self, address=None):
        """
        Check if there is an RpcServer thread running

        :returns: bool - True if running, False if not running
        """
        return self.e_alive.is_set()

    def rpc_stop(self, connection=None):
        """
        Close the server socket and stop the RpcServer thread

        .. note::

            This operation may take a small amount of time for the socket to
            register the stop request.
        """
        self.e_alive.clear()
        self.__rpc_thread.join()

    def rpc_uptime(self, connection=None):
        """
        Get the uptime of the RpcServer

        :returns: int - uptime in seconds
        """
        if hasattr(self, 'rpc_startTime'):
            delta = datetime.now() - self.rpc_startTime
            return delta.total_seconds()
        else:
            return 0

    def rpc_getPort(self, address=None):
        """
        Get the bound port of a running RpcServer thread.

        :returns: int - Port
        """
        return self.port

    def rpc_getHostname(self, address=None):
        """
        Get the hostname of the RpcServer host.

        :returns: str - hostname
        """
        return socket.gethostname()

    def rpc_getConnections(self, address=None):
        """
        Get the number of connections to the RPC server

        :returns: int
        """
        return len(self.rpc_connections)

class RpcConnection(threading.Thread):
    """
    Receives and processes RPC requests from remote connections (or local ones)

    Before any requests can be processed, the thread must acquire a lock to
    synchronize access to RPC Server resources

    Any method that begins with an underscore is considered protected and will
    be invalidated by the RPC server

    :param server: RPC Server object
    :type server: RpcServer
    :param socket: RPC Request Socket
    :type socket: socket.socket
    :param logger: Logger instance if you wish to override the internal instance
    :type logger: Logging.logger
    """
    RPC_MAX_PACKET_SIZE = 1048576 # 1MB

    def __init__(self, server, socket, **kwargs):
        threading.Thread.__init__(self)

        self.server = server
        self.socket = socket
        self.logger = kwargs.get('logger', logging)

        self.address, _ = self.socket.getsockname()

        self.lock = self.server.rpc_lock
        #self.name = self.parent.name

        # Give the thread a meaningful name
        self.name = '%s-%s' % (self.server.getName(), self.address)

    def run(self):

        try:
            ready_to_read,_,_ = select.select([self.socket],[],[], 1.0)
            # TODO: Read until all data is in the buffer

            if self.socket in ready_to_read:

                data = self.socket.recv(self.RPC_MAX_PACKET_SIZE)

                if data:
                    # Process the incoming data as a JSON RPC packet
                    in_packet = JsonRpcPacket(data)
                    errors = in_packet.getErrors()
                    requests = in_packet.getRequests()

                    out_packet = JsonRpcPacket()

                    if len(errors) == 0:
                        # Only process requests if no errors were found during parsing
                        for req in requests:
                            # Process Requests in order
                            id = req.getID()
                            try:
                                result = self.processRequest(req)

                                # Check if the request was a notification
                                if id is not None:
                                    out_packet.addResponse(id, result)

                            # Catch exceptions during method execution
                            # DO NOT ALLOW ANY EXCEPTIONS TO PASS THIS LEVEL
                            except RpcMethodNotFound:
                                out_packet.addError_MethodNotFound(id)

                            except TypeError:
                                # Raised when arguments mismatch, but also other cases
                                # Not a perfect solution, but whatever.
                                out_packet.addError_InvalidParams(id)
                                self.logger.exception("RPC Server Type Error")

                            except Exception as e:
                                # Catch-all for everything else
                                out_packet.addError_ServerException(id, e.message)
                                self.logger.exception("RPC Server Exception")

                    # Encode the outputs of the RPC requests
                    out_str = out_packet.export()
                    self.socket.send(out_str)

        except socket.error as e:
            # Socket closed poorly from client
            if e.errno == errno.ECONNABORTED:
                self.logger.error('[%s] Client socket closed before data could be sent', self.name)
            else:
                self.logger.error('[%s] Socket closed with error: %s', self.name, e.errno)

        except:
            # Log an exception, close the connection
            self.logger.exception('[%s] Unhandled Exception', self.name)

    def processRequest(self, req):
        id = req.getID()
        method = req.getMethod()
        self.logger.debug('[%s, %i] RPC Request: %s', self.name, id, method)

        try:
            test_method = self.server.findMethod(method)

            with self.lock:
                return req.call(test_method)

        # Bubble all exceptions up to the calling function
        except RpcMethodNotFound:
            self.logger.error('RPC Method Not Found')
            raise
        <commit_msg>RPC Server socket receives full request before processing. Possibly fixes a bug that was never reported or root-caused<commit_after>import threading
import socket
import select
import logging
import inspect
import errno
from datetime import datetime

from jsonrpc import *
from errors import *

class RpcServer(object):
    """
    Flexible RPC server that runs in a Python thread. Wraps any Python object
    and allows remote machines to call functions and receive returned data.

    :param logger: Logger instance if you wish to override the internal instance
    :type logger: Logging.logger
    :param port: Port to attach socket to
    :type port: int
    :param name: Internal name for the RPC server thread
    :type name: str

    .. note::

        Method calls to functions that begin with an underscore are considered
        protected and will not be invoked
    """

    # Exclusive access times out after 60 seconds
    EXCLUSIVE_TIMEOUT = 60.0

    _identity = 'JSON-RPC/2.0'

    def __init__(self, **kwargs):
        self.logger = kwargs.get('logger', logging)
        self.port = kwargs.get('port', 0)
        self.name = kwargs.get('name', 'RPCServer')

        # RPC State Variables
        self.rpc_objects = []

        self.e_alive = threading.Event()
        self.rpc_lock = threading.Lock()
        self.rpc_startTime = datetime.now()

        # Attempt to bind a socket
        try:
            srv_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            srv_socket.bind(('',self.port))
            srv_socket.listen(5)
            srv_socket.setblocking(0)

            # Update port if randomly assigned
            _, self.port = srv_socket.getsockname()

        except socket.error as e:
            if e.errno == errno.EADDRINUSE:
                # Server is already running
                raise RpcServerPortInUse

            else:
                raise

        self.e_alive.set()

        self.__rpc_thread = threading.Thread(name=self.name, target=self.__thread_run, args=(srv_socket,))
        self.__rpc_thread.start()

    def __thread_run(self, srv_socket):
        _, self.port = srv_socket.getsockname()

        self.logger.debug('[%s] RPC Server started on port %i', self.name, self.port)

        self.rpc_startTime = datetime.now()

        while self.e_alive.isSet():
            # Service Socket
            try:
                ready_to_read,_,_ = select.select([srv_socket], [], [], 1.0)

                if srv_socket in ready_to_read:
                    # Spawn a new thread to service the connection
                    connection, address = srv_socket.accept()

                    # Spawn a new thread to service the connection
                    connThread = RpcConnection(server=self,
                                               socket=connection,
                                               logger=self.logger)

                    connThread.start()

            except:
                self.logger.exception('RPC Server Socket Handler Exception')

            # Clean up old connections
            # TODO

        srv_socket.close()

        self.logger.debug('[%s] RPC Server stopped', self.name)

    #===========================================================================
    # Server Management
    #===========================================================================

    def getName(self):
        return self.name

    def registerObject(self, reg_obj):
        self.rpc_objects.append(reg_obj)

    def unregisterObject(self, reg_obj):
        self.rpc_objects.remove(reg_obj)

    #===========================================================================
    # Methods
    #===========================================================================

    def findMethod(self, method):
        """
        Return a bound method after searching through the server methods and the
        methods of all registered objects
        """
        # Disallow access to methods that start with an underscore
        if method == '' or method.startswith('_'):
            # Invalid - Protected Method
            self.logger.warning('RPC Request Denied (Protected Method): %s', method)
            raise AttributeError

        elif method.startswith('rpc'):
            # Valid - RPC Server method
            return self._getMethod(self, method)

        else:
            # Check registered objects
            for reg_obj in self.rpc_objects:
                try:
                    return self._getMethod(reg_obj, method)
                except AttributeError:
                    pass

            # Unable to find method
            raise RpcMethodNotFound()

    def _getMethod(self, obj, method):
        test_method = getattr(obj, method)
        if inspect.ismethod(test_method):
            return test_method
        else:
            raise AttributeError

    #===========================================================================
    # RPC Functions
    #===========================================================================

    def rpc_getMethods(self, address=None):
        """
        Get a list of valid methods in the registered objects. Protected methods
        that begin with an underscore ('_') are not included.

        :returns: list of strings
        """
        # Catalog methods
        self.validMethods = []

        target_members = inspect.getmembers(self)

        for attr, val in target_members:
            if inspect.ismethod(val) and attr.startswith('rpc'):
                self.validMethods.append(attr)

        for reg_obj in self.rpc_objects:
            target_members = inspect.getmembers(reg_obj)

            for attr, val in target_members:
                if inspect.ismethod(val) and not attr.startswith('_'):
                    self.validMethods.append(attr)

        return self.validMethods

    def rpc_isRunning(self, address=None):
        """
        Check if there is an RpcServer thread running

        :returns: bool - True if running, False if not running
        """
        return self.e_alive.is_set()

    def rpc_stop(self, connection=None):
        """
        Close the server socket and stop the RpcServer thread

        .. note::

            This operation may take a small amount of time for the socket to
            register the stop request.
        """
        self.e_alive.clear()
        self.__rpc_thread.join()

    def rpc_uptime(self, connection=None):
        """
        Get the uptime of the RpcServer

        :returns: int - uptime in seconds
        """
        if hasattr(self, 'rpc_startTime'):
            delta = datetime.now() - self.rpc_startTime
            return delta.total_seconds()
        else:
            return 0

    def rpc_getPort(self, address=None):
        """
        Get the bound port of a running RpcServer thread.

        :returns: int - Port
        """
        return self.port

    def rpc_getHostname(self, address=None):
        """
        Get the hostname of the RpcServer host.

        :returns: str - hostname
        """
        return socket.gethostname()

    def rpc_getConnections(self, address=None):
        """
        Get the number of connections to the RPC server

        :returns: int
        """
        return len(self.rpc_connections)

class RpcConnection(threading.Thread):
    """
    Receives and processes RPC requests from remote connections (or local ones)

    Before any requests can be processed, the thread must acquire a lock to
    synchronize access to RPC Server resources

    Any method that begins with an underscore is considered protected and will
    be invalidated by the RPC server

    :param server: RPC Server object
    :type server: RpcServer
    :param socket: RPC Request Socket
    :type socket: socket.socket
    :param logger: Logger instance if you wish to override the internal instance
    :type logger: Logging.logger
    """
    RPC_MAX_PACKET_SIZE = 4096

    def __init__(self, server, socket, **kwargs):
        threading.Thread.__init__(self)

        self.server = server
        self.socket = socket
        self.logger = kwargs.get('logger', logging)

        self.address, _ = self.socket.getsockname()

        self.lock = self.server.rpc_lock
        #self.name = self.parent.name

        # Give the thread a meaningful name
        self.name = '%s-%s' % (self.server.getName(), self.address)

    def run(self):

        try:
            ready_to_read,_,_ = select.select([self.socket],[],[], 1.0)
            # TODO: Read until all data is in the buffer

            if self.socket in ready_to_read:

                data = self.socket.recv(self.RPC_MAX_PACKET_SIZE)
                seg = None
                # Receive the full packet
                try:
                    while seg != "":
                        seg = self.socket.recv(self.RPC_MAX_PACKET_SIZE)
                        data += seg
                except socket.error as e:
                    if e.errno == errno.EWOULDBLOCK:
                        # No more data to process
                        pass
                    else:
                        pass

                if data:
                    # Process the incoming data as a JSON RPC packet
                    in_packet = JsonRpcPacket(data)
                    errors = in_packet.getErrors()
                    requests = in_packet.getRequests()

                    out_packet = JsonRpcPacket()

                    if len(errors) == 0:
                        # Only process requests if no errors were found during parsing
                        for req in requests:
                            # Process Requests in order
                            id = req.getID()
                            try:
                                result = self.processRequest(req)

                                # Check if the request was a notification
                                if id is not None:
                                    out_packet.addResponse(id, result)

                            # Catch exceptions during method execution
                            # DO NOT ALLOW ANY EXCEPTIONS TO PASS THIS LEVEL
                            except RpcMethodNotFound:
                                out_packet.addError_MethodNotFound(id)

                            except TypeError:
                                # Raised when arguments mismatch, but also other cases
                                # Not a perfect solution, but whatever.
                                out_packet.addError_InvalidParams(id)
                                self.logger.exception("RPC Server Type Error")

                            except Exception as e:
                                # Catch-all for everything else
                                out_packet.addError_ServerException(id, e.message)
                                self.logger.exception("RPC Server Exception")

                    # Encode the outputs of the RPC requests
                    out_str = out_packet.export()
                    self.socket.send(out_str)

        except socket.error as e:
            # Socket closed poorly from client
            if e.errno == errno.ECONNABORTED:
                self.logger.error('[%s] Client socket closed before data could be sent', self.name)
            else:
                self.logger.error('[%s] Socket closed with error: %s', self.name, e.errno)

        except:
            # Log an exception, close the connection
            self.logger.exception('[%s] Unhandled Exception', self.name)

    def processRequest(self, req):
        id = req.getID()
        method = req.getMethod()
        self.logger.debug('[%s, %i] RPC Request: %s', self.name, id, method)

        try:
            test_method = self.server.findMethod(method)

            with self.lock:
                return req.call(test_method)

        # Bubble all exceptions up to the calling function
        except RpcMethodNotFound:
            self.logger.error('RPC Method Not Found')
            raise
        <|endoftext|>


<commit_before>from __future__ import unicode_literals
from elasticsearch import TransportError

from elasticsearch_dsl import Search, DocType, Date, Text, Keyword, MultiSearch, \
    MetaField, Index, Q

from .test_data import DATA

from pytest import raises

class Repository(DocType):
    created_at = Date()
    description = Text(analyzer='snowball')
    tags = Keyword()

    class Meta:
        index = 'git'
        doc_type = 'repos'

class Commit(DocType):
    class Meta:
        doc_type = 'commits'
        index = 'git'
        parent = MetaField(type='repos')

def test_inner_hits_are_wrapped_in_response(data_client):
    s = Search(index='git', doc_type='commits')[0:1].query('has_parent', type='repos', inner_hits={}, query=Q('match_all'))
    response = s.execute()

    commit = response.hits[0]
    assert isinstance(commit.meta.inner_hits.repos, response.__class__)
    assert repr(commit.meta.inner_hits.repos[0]).startswith("<Result(git/repos/elasticsearch-dsl-py): ")

def test_inner_hits_are_wrapped_in_doc_type(data_client):
    i = Index('git')
    i.doc_type(Repository)
    i.doc_type(Commit)
    s = i.search()[0:1].doc_type(Commit).query('has_parent', type='repos', inner_hits={}, query=Q('match_all'))
    response = s.execute()

    commit = response.hits[0]
    assert isinstance(commit.meta.inner_hits.repos, response.__class__)
    assert isinstance(commit.meta.inner_hits.repos[0], Repository)
    assert "Repository(index=%r, doc_type=%r, id=%r)" % ('git', 'repos', 'elasticsearch-dsl-py') == repr(commit.meta.inner_hits.repos[0])

def test_suggest_can_be_run_separately(data_client):
    s = Search()
    s = s.suggest('simple_suggestion', 'elasticserach', term={'field': 'organization'})
    response = s.execute_suggest()

    assert response.success()
    assert response.simple_suggestion[0].options[0].text == 'elasticsearch'

def test_scan_respects_doc_types(data_client):
    repos = list(Repository.search().scan())

    assert 1 == len(repos)
    assert isinstance(repos[0], Repository)

def test_scan_iterates_through_all_docs(data_client):
    s = Search(index='git').filter('term', _type='commits')

    commits = list(s.scan())

    assert 52 == len(commits)
    assert set(d['_id'] for d in DATA if d['_type'] == 'commits') == set(c.meta.id for c in commits)

def test_response_is_cached(data_client):
    s = Repository.search()
    repos = list(s)

    assert hasattr(s, '_response')
    assert s._response.hits == repos

def test_multi_search(data_client):
    s1 = Repository.search()
    s2 = Search(doc_type='commits')

    ms = MultiSearch(index='git')
    ms = ms.add(s1).add(s2)

    r1, r2 = ms.execute()

    assert 1 == len(r1)
    assert isinstance(r1[0], Repository)
    assert r1.search is s1

    assert 52 == r2.hits.total
    assert r2.search is s2

def test_multi_missing(data_client):
    s1 = Repository.search()
    s2 = Search(doc_type='commits')
    s3 = Search(index='does_not_exist')

    ms = MultiSearch()
    ms = ms.add(s1).add(s2).add(s3)

    with raises(TransportError):
        ms.execute()

    r1, r2, r3 = ms.execute(raise_on_error=False)

    assert 1 == len(r1)
    assert isinstance(r1[0], Repository)
    assert r1.search is s1

    assert 52 == r2.hits.total
    assert r2.search is s2

    assert r3 is None
<commit_msg>Inner hits no longer have index<commit_after>from __future__ import unicode_literals
from elasticsearch import TransportError

from elasticsearch_dsl import Search, DocType, Date, Text, Keyword, MultiSearch, \
    MetaField, Index, Q

from .test_data import DATA

from pytest import raises

class Repository(DocType):
    created_at = Date()
    description = Text(analyzer='snowball')
    tags = Keyword()

    class Meta:
        index = 'git'
        doc_type = 'repos'

class Commit(DocType):
    class Meta:
        doc_type = 'commits'
        index = 'git'
        parent = MetaField(type='repos')

def test_inner_hits_are_wrapped_in_response(data_client):
    s = Search(index='git', doc_type='commits')[0:1].query('has_parent', type='repos', inner_hits={}, query=Q('match_all'))
    response = s.execute()

    commit = response.hits[0]
    assert isinstance(commit.meta.inner_hits.repos, response.__class__)
    r = commit.meta.inner_hits.repos[0]
    assert repr(commit.meta.inner_hits.repos[0]).startswith("<Result(repos/elasticsearch-dsl-py): ")

def test_inner_hits_are_wrapped_in_doc_type(data_client):
    i = Index('git')
    i.doc_type(Repository)
    i.doc_type(Commit)
    s = i.search()[0:1].doc_type(Commit).query('has_parent', type='repos', inner_hits={}, query=Q('match_all'))
    response = s.execute()

    commit = response.hits[0]
    assert isinstance(commit.meta.inner_hits.repos, response.__class__)
    assert isinstance(commit.meta.inner_hits.repos[0], Repository)
    assert "Repository(index=%r, doc_type=%r, id=%r)" % ('git', 'repos', 'elasticsearch-dsl-py') == repr(commit.meta.inner_hits.repos[0])

def test_suggest_can_be_run_separately(data_client):
    s = Search()
    s = s.suggest('simple_suggestion', 'elasticserach', term={'field': 'organization'})
    response = s.execute_suggest()

    assert response.success()
    assert response.simple_suggestion[0].options[0].text == 'elasticsearch'

def test_scan_respects_doc_types(data_client):
    repos = list(Repository.search().scan())

    assert 1 == len(repos)
    assert isinstance(repos[0], Repository)

def test_scan_iterates_through_all_docs(data_client):
    s = Search(index='git').filter('term', _type='commits')

    commits = list(s.scan())

    assert 52 == len(commits)
    assert set(d['_id'] for d in DATA if d['_type'] == 'commits') == set(c.meta.id for c in commits)

def test_response_is_cached(data_client):
    s = Repository.search()
    repos = list(s)

    assert hasattr(s, '_response')
    assert s._response.hits == repos

def test_multi_search(data_client):
    s1 = Repository.search()
    s2 = Search(doc_type='commits')

    ms = MultiSearch(index='git')
    ms = ms.add(s1).add(s2)

    r1, r2 = ms.execute()

    assert 1 == len(r1)
    assert isinstance(r1[0], Repository)
    assert r1.search is s1

    assert 52 == r2.hits.total
    assert r2.search is s2

def test_multi_missing(data_client):
    s1 = Repository.search()
    s2 = Search(doc_type='commits')
    s3 = Search(index='does_not_exist')

    ms = MultiSearch()
    ms = ms.add(s1).add(s2).add(s3)

    with raises(TransportError):
        ms.execute()

    r1, r2, r3 = ms.execute(raise_on_error=False)

    assert 1 == len(r1)
    assert isinstance(r1[0], Repository)
    assert r1.search is s1

    assert 52 == r2.hits.total
    assert r2.search is s2

    assert r3 is None
<|endoftext|>


<commit_before><commit_msg>Removing export_outputs value from TPUEstimator. This is not required with TPUEstimators<commit_after><|endoftext|>


<commit_before>import argparse

from classes.cstatistics import CStatistics
from classes.csv_export import CSVExport
from time import sleep
from classes.yoctoammeter import YoctoDevice

def main():
    r"""
    Main program of LightPowertool.

    LightPowertool is a software to measure automatically the energy\
     consumption of devices, using the Yoctopuce ammeter.
    """
    # Parse arguments
    parser = argparse.ArgumentParser()
    parser.add_argument("-f", "--framerate", type=str, default="50/s",
                        help="Give to the main program the framerate.")
    parser.add_argument("-n", "--network", action="store_true",
                        help="Use the network to send your data\
                        - local server like LightPowertoolServer")
    parser.add_argument("-o", "--output", type=str,
                        help="Give an output file name\
                         to store measured values.")
    parser.add_argument("-s", "--statistics", action="store_true",
                        help="Ask to output basic statistics\
                         (mean, median, pvariance,...) on measured values.")
    parser.add_argument("-t", "--time", type=int, default=60,
                        help="Due time to measure data.")
    args = parser.parse_args()

    # Get the yocto device
    yocto_device = YoctoDevice(args.framerate, network=args.network)
    print(yocto_device)

    yocto_device.run()
    sleep(args.time)
    yocto_device.stopMeasure()

    if args.output:
        CSVExport(args.output).export_data(yocto_device._values)

    if args.statistics:
        measures = [y for x, y in yocto_device._values]
        print(CStatistics(measures))

if __name__ == '__main__':
    main()
<commit_msg>perf: Pass as argument generator instead of a list<commit_after>import argparse

from classes.cstatistics import CStatistics
from classes.csv_export import CSVExport
from time import sleep
from classes.yoctoammeter import YoctoDevice

def main():
    r"""
    Main program of LightPowertool.

    LightPowertool is a software to measure automatically the energy\
     consumption of devices, using the Yoctopuce ammeter.
    """
    # Parse arguments
    parser = argparse.ArgumentParser()
    parser.add_argument("-f", "--framerate", type=str, default="50/s",
                        help="Give to the main program the framerate.")
    parser.add_argument("-n", "--network", action="store_true",
                        help="Use the network to send your data\
                        - local server like LightPowertoolServer")
    parser.add_argument("-o", "--output", type=str,
                        help="Give an output file name\
                         to store measured values.")
    parser.add_argument("-s", "--statistics", action="store_true",
                        help="Ask to output basic statistics\
                         (mean, median, pvariance,...) on measured values.")
    parser.add_argument("-t", "--time", type=int, default=60,
                        help="Due time to measure data.")
    args = parser.parse_args()

    # Get the yocto device
    yocto_device = YoctoDevice(args.framerate, network=args.network)
    print(yocto_device)

    yocto_device.run()
    sleep(args.time)
    yocto_device.stopMeasure()

    if args.output:
        CSVExport(args.output)\
            .export_data((value for value in yocto_device._values))

    if args.statistics:
        measures = [y for x, y in yocto_device._values]
        print(CStatistics(measures))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># modifies the N-armed Bandits problem into a non-stationary one
# that is, where the reward value of the bandit arms shifts randomly
#
# compares the use of a sample-average estimate method (used in previous soutions)
# with a non-stationary one, that weights more recent information / rewards
# more heavily than less recent

# PARAMS (via command-line)
# @ alphas -> array of alpha hyperparameters
#       entered as comma-deliniated values
#       default value -> [0.01, 0.1, 1.0]
#       N.B. values must satisfy 0 < a <= 1

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.ticker as mtick
from util.iter_count import IterCount
from util.cmap import colormap
from .solution_util import softmax

import settings

config = settings.n_armed_bandits

def learn(action_value, alpha, temperature):

    numBandits, numArms, numPlays = (config['numBandits'], config['numArms'], config['numPlays'])

    bandits = np.random.normal(0, 1, (numBandits, numArms))

    if action_value == 'sample-average':
        activated, estimates = [np.zeros(bandits.shape) for i in range(2)]
    else:
        estimates = np.zeros(bandits.shape)

    rewards, isOptimal = [np.zeros((numBandits, numPlays)) for i in range(2)]

    ic = IterCount('Play numer {0} of {1}'.format("{}", numPlays))

    for i in range(numPlays):

        ic.update()

        bandits += np.random.normal(0, 0.2, (bandits.shape))

        best = np.argmax(bandits, axis=1)

        arm = softmax(estimates, temperature)

        isOptimal[:, i][arm == best] = 1
        reward = np.random.normal(0, 1, numBandits) + bandits[range(numBandits), arm]
        rewards[:, i] = reward

        if action_value == 'sample-average':
            activated[range(numBandits), arm] += 1
            estimates[range(numBandits), arm] += (reward - estimates[range(numBandits), arm]) / activated[range(numBandits), arm]

        else:
            estimates[range(numBandits), arm] *= 1 - alpha
            estimates[range(numBandits), arm] += alpha * reward

    ic.exit()

    return rewards, isOptimal

def run(alphas):

    print('Running with settings:')
    print('\tnumBandits: {0}\tnumArms: {1}\tnumPlays: {2}\n'.format(config['numBandits'], config['numArms'], config['numPlays']))

    fig = plt.figure(1, (8,8))
    reward_plot = fig.add_subplot(211)
    optimal_plot = fig.add_subplot(212)

    temperature = 0.1
    cmap = colormap(len(alphas) + 1)

    print('Learning with sample-average action value estimate')
    rewards, isOptimal = learn('sample-average', None, temperature)
    reward_plot.plot(range(config['numPlays']), np.mean(rewards, axis=0), c=cmap(0))
    optimal_plot.plot(range(config['numPlays']), np.mean(isOptimal, axis=0) * 100, c=cmap(0))

    print('Learning with nonstationary action value estimate')
    for i, alpha in enumerate(alphas):
        print('Learning with alpha = {}'.format(alpha))
        rewards, isOptimal = learn('nonstationary', alpha, temperature)
        reward_plot.plot(range(config['numPlays']), np.mean(rewards, axis=0), c=cmap(i+1))
        optimal_plot.plot(range(config['numPlays']), np.mean(isOptimal, axis=0) * 100, c=cmap(i+1))

    reward_plot.legend(['Sample-Average'] + ['Alpha: {}'.format(a) for a in alphas])
    optimal_plot.legend(['Sample-Average'] + ['Alpha: {}'.format(a) for a in alphas])

    yticks = mtick.FormatStrFormatter('%.0f%%')
    optimal_plot.yaxis.set_major_formatter(yticks)
    plt.show()
<commit_msg>fixed typo<commit_after># modifies the N-armed Bandits problem into a non-stationary one
# that is, where the reward value of the bandit arms shifts randomly
#
# compares the use of a sample-average estimate method (used in previous soutions)
# with a non-stationary one, that weights more recent information / rewards
# more heavily than less recent

# PARAMS (via command-line)
# @ alphas -> array of alpha hyperparameters
#       entered as comma-deliniated values
#       default value -> [0.01, 0.1, 1.0]
#       N.B. values must satisfy 0 < a <= 1

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.ticker as mtick
from util.iter_count import IterCount
from util.cmap import colormap
from .solution_util import softmax

import settings

config = settings.n_armed_bandits

def learn(action_value, alpha, temperature):

    numBandits, numArms, numPlays = (config['numBandits'], config['numArms'], config['numPlays'])

    bandits = np.random.normal(0, 1, (numBandits, numArms))

    if action_value == 'sample-average':
        activated, estimates = [np.zeros(bandits.shape) for i in range(2)]
    else:
        estimates = np.zeros(bandits.shape)

    rewards, isOptimal = [np.zeros((numBandits, numPlays)) for i in range(2)]

    ic = IterCount('Play number {0} of {1}'.format("{}", numPlays))

    for i in range(numPlays):

        ic.update()

        bandits += np.random.normal(0, 0.2, (bandits.shape))

        best = np.argmax(bandits, axis=1)

        arm = softmax(estimates, temperature)

        isOptimal[:, i][arm == best] = 1
        reward = np.random.normal(0, 1, numBandits) + bandits[range(numBandits), arm]
        rewards[:, i] = reward

        if action_value == 'sample-average':
            activated[range(numBandits), arm] += 1
            estimates[range(numBandits), arm] += (reward - estimates[range(numBandits), arm]) / activated[range(numBandits), arm]

        else:
            estimates[range(numBandits), arm] *= 1 - alpha
            estimates[range(numBandits), arm] += alpha * reward

    ic.exit()

    return rewards, isOptimal

def run(alphas):

    print('Running with settings:')
    print('\tnumBandits: {0}\tnumArms: {1}\tnumPlays: {2}\n'.format(config['numBandits'], config['numArms'], config['numPlays']))

    fig = plt.figure(1, (8,8))
    reward_plot = fig.add_subplot(211)
    optimal_plot = fig.add_subplot(212)

    temperature = 0.1
    cmap = colormap(len(alphas) + 1)

    print('Learning with sample-average action value estimate')
    rewards, isOptimal = learn('sample-average', None, temperature)
    reward_plot.plot(range(config['numPlays']), np.mean(rewards, axis=0), c=cmap(0))
    optimal_plot.plot(range(config['numPlays']), np.mean(isOptimal, axis=0) * 100, c=cmap(0))

    print('Learning with nonstationary action value estimate')
    for i, alpha in enumerate(alphas):
        print('Learning with alpha = {}'.format(alpha))
        rewards, isOptimal = learn('nonstationary', alpha, temperature)
        reward_plot.plot(range(config['numPlays']), np.mean(rewards, axis=0), c=cmap(i+1))
        optimal_plot.plot(range(config['numPlays']), np.mean(isOptimal, axis=0) * 100, c=cmap(i+1))

    reward_plot.legend(['Sample-Average'] + ['Alpha: {}'.format(a) for a in alphas])
    optimal_plot.legend(['Sample-Average'] + ['Alpha: {}'.format(a) for a in alphas])

    yticks = mtick.FormatStrFormatter('%.0f%%')
    optimal_plot.yaxis.set_major_formatter(yticks)
    plt.show()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################
#
# Swing Entwicklung betrieblicher Informationssysteme GmbH
# (<http://www.swing-system.com>)
# Copyright (C) ChriCar Beteiligungs- und Beratungs- GmbH
# all rights reserved
#    26-APR-2012 (GK) created
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs.
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company.
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/> or
# write to the Free Software Foundation, Inc.,
# 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
###############################################
from osv import fields,osv
from tools.translate import _
import logging

class one2many_sorted(fields.one2many):

    _logger = logging.getLogger(__name__)

    def parse_order(self, order):
        result = []
        for col in order.split(',') :
            c = col.strip()
            if ' ASC' in c.upper() :
                result.append((c[0:c.index(' ')], False))
            elif ' DESC' in c.upper() :
                result.append((c[0:c.index(' ')], True))
            else :
                result.append((c, False))
        return list(reversed(result))
    # end def parse_order

    def __init__(self, obj, fields_id, string='unknown', limit=None, **args) :
        self._order  = []
        self._search = []
        self._set    = {}
        if 'order' in args :
            self._order = self.parse_order(args['order'])
        if 'search' in args :
            self._search = args['search']
        if 'set' in args :
            self._set = args['set']
        (fields.one2many).__init__(self, obj, fields_id, string=string, limit=limit, **args)
    # end def __init__

    def property_value(self, cr, user, obj, name):
        property_obj = obj.pool.get('ir.property')
        prop_id = property_obj.search \
            ( cr, user
            , [ ('name', '=', name)
              , ('type', '=', 'text')
              , ('company_id','=', obj.pool.get('res.company')._company_default_get(cr, user))
              ]
            )
        if prop_id :
            return property_obj.browse(cr, user, prop_id[0]).value_text
        return False
    # end def property_value

    def selected (self, cr, user, obj, ids, context=None) :
        _obj = obj.pool.get(self._obj)
        return _obj.search \
            ( cr, user
            , [(self._fields_id, 'in', ids)] + self._search
            , limit = self._limit
            , context=context
            )
    # end def selected

    def get (self, cr, obj, ids, name, user=None, offset=0, context=None, values={}) :
        _obj = obj.pool.get(self._obj)
        if context and 'one2many_sorted_order' in context :
            prop = self.property_value(cr, user, obj, context['one2many_sorted_order'])
            if prop :
                order = self.parse_order(prop)
        else:
            prop = self.property_value(cr, user, obj, "%s.%s.order" % (self._obj, self._fields_id))
            if prop :
                order = self.parse_order(prop)
            else :
                order = self._order
        ids2 = self.selected(cr, user, obj, ids, context=context)
	sortable = []
        for r in _obj.browse(cr, user, ids2, context=context) :
	    d = {}
            for key in ([('id', False)] + order) :
                o = r
                for m in key[0].split('.'):
                    if "()" in m :
                        o = getattr(o, m.strip("()"))()
                    else :
                        o = getattr(o, m)
		d[key[0]] = o if not isinstance(o, str) else _(o)
	    sortable.append(d)
        for key in order :
            sortable.sort(key=lambda d: d[key[0]], reverse=key[1])
        res = {}
        for id in ids : res[id] = []
        for r in _obj.browse(cr, user, [d['id'] for d in sortable], context=context) :
            res[getattr(r, self._fields_id).id].append(r.id)
        return res
    # end def get

    def set(self, cr, obj, id, field, values, user=None, context=None):
        for act in values :
            if act[0] == 0 : # "create"
                for k, v in self._set.iteritems() :
                    act[2][k] = v
        return (fields.one2many).set(cr, obj, id, field, values, user, context)
    # end def set
# end class one2many_sorted

class many2many_sorted(fields.many2many):

    _logger = logging.getLogger(__name__)

    def parse_order(self, order):
        result = []
        for col in order.split(',') :
            c = col.strip()
            if ' ASC' in c.upper() :
                result.append((c[0:c.index(' ')], False))
            elif ' DESC' in c.upper() :
                result.append((c[0:c.index(' ')], True))
            else :
                result.append((c, False))
        return list(reversed(result))
    # end def parse_order

    def __init__(self, obj, rel=None, id1=None, id2=None, string='unknown', limit=None, **args) :
        self._order  = []
        self._set    = {}
        if 'order' in args :
            self._order = self.parse_order(args['order'])
        if 'set' in args :
            self._set = args['set']
        (fields.many2many).__init__(self, obj, rel, id1=id1, id2=id2, string=string, limit=limit, **args)
    # end def __init__

    def property_value(self, cr, user, obj, name):
        property_obj = obj.pool.get('ir.property')
        prop_id = property_obj.search \
            ( cr, user
            , [ ('name', '=', name)
              , ('type', '=', 'text')
              , ('company_id','=', obj.pool.get('res.company')._company_default_get(cr, user))
              ]
            )
        if prop_id :
            return property_obj.browse(cr, user, prop_id[0]).value_text
        return False
    # end def property_value

    def get (self, cr, obj, ids, name, user=None, offset=0, context=None, values={}) :
        _obj = obj.pool.get(self._obj)
        if context and 'many2many_sorted_order' in context :
            prop = self.property_value(cr, user, obj, context['many2many_sorted_order'])
            if prop :
                order = self.parse_order(prop)
        else:
            prop = self.property_value(cr, user, obj, "%s.%s.%s.%s.order" % (self._obj, self._rel, self._id1, self._id2))
            if prop :
                order = self.parse_order(prop)
            else :
                order = self._order
        res = {}
        for id in ids : res[id] = []
        got = (fields.many2many).get(self, cr, obj, ids, name, user=user, offset=offset, context=context, values=values)
        for k, ids2 in got.iteritems() :
            sortable = []
            for r in _obj.browse(cr, user, ids2, context=context) :
                d = {}
                for key in ([('id', False)] + order) :
                    o = r
                    for m in key[0].split('.'):
                        if "()" in m :
                            o = getattr(o, m.strip("()"))()
                        else :
                            o = getattr(o, m)
                    d[key[0]] = o if not isinstance(o, str) else _(o)
                sortable.append(d)
            for key in order :
                sortable.sort(key=lambda d: d[key[0]], reverse=key[1])
            self._logger.info("many2many order criteria: %s", order) ######
            for d in sortable : self._logger.info("sorted %s", d) #############
            for r in _obj.browse(cr, user, [d['id'] for d in sortable], context=context) :
                res[k].append(r.id)
        return res
    # end def get

    def set(self, cr, model, id, name, values, user=None, context=None):
        for act in values :
            if act[0] == 0 : # "create"
                for k, v in self._set.iteritems() :
                    act[2][k] = v
        return (fields.many2many).set(cr, model, id, name, values, user, context)
    # end def set
# end class many2many_sorted

<commit_msg>[FIX] o2m sorted<commit_after># -*- coding: utf-8 -*-
##############################################
#
# Swing Entwicklung betrieblicher Informationssysteme GmbH
# (<http://www.swing-system.com>)
# Copyright (C) ChriCar Beteiligungs- und Beratungs- GmbH
# all rights reserved
#    26-APR-2012 (GK) created
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs.
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company.
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/> or
# write to the Free Software Foundation, Inc.,
# 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
###############################################
from osv import fields,osv
from tools.translate import _
import logging

class one2many_sorted(fields.one2many):

    _logger = logging.getLogger(__name__)

    def parse_order(self, order):
        result = []
        for col in order.split(',') :
            c = col.strip()
            if ' ASC' in c.upper() :
                result.append((c[0:c.index(' ')], False))
            elif ' DESC' in c.upper() :
                result.append((c[0:c.index(' ')], True))
            else :
                result.append((c, False))
        return list(reversed(result))
    # end def parse_order

    def __init__(self, obj, fields_id, string='unknown', limit=None, **args) :
        self._order  = []
        self._search = []
        self._set    = {}
        if 'order' in args :
            self._order = self.parse_order(args['order'])
        if 'search' in args :
            self._search = args['search']
        if 'set' in args :
            self._set = args['set']
        (fields.one2many).__init__(self, obj, fields_id, string=string, limit=limit, **args)
    # end def __init__

    def property_value(self, cr, user, obj, name):
        property_obj = obj.pool.get('ir.property')
        prop_id = property_obj.search \
            ( cr, user
            , [ ('name', '=', name)
              , ('type', '=', 'text')
              , ('company_id','=', obj.pool.get('res.company')._company_default_get(cr, user))
              ]
            )
        if prop_id :
            return property_obj.browse(cr, user, prop_id[0]).value_text
        return False
    # end def property_value

    def selected (self, cr, user, obj, ids, context=None) :
        _obj = obj.pool.get(self._obj)
        return _obj.search \
            ( cr, user
            , [(self._fields_id, 'in', ids)] + self._search
            , limit = self._limit
            , context=context
            )
    # end def selected

    def get (self, cr, obj, ids, name, user=None, offset=0, context=None, values={}) :
        _obj = obj.pool.get(self._obj)
        if context and 'one2many_sorted_order' in context :
            prop = self.property_value(cr, user, obj, context['one2many_sorted_order'])
            if prop :
                order = self.parse_order(prop)
        else:
            prop = self.property_value(cr, user, obj, "%s.%s.order" % (self._obj, self._fields_id))
            if prop :
                order = self.parse_order(prop)
            else :
                order = self._order
        ids2 = self.selected(cr, user, obj, ids, context=context)
	sortable = []
        for r in _obj.browse(cr, user, ids2, context=context) :
	    d = {}
            for key in ([('id', False)] + order) :
                o = r
                for m in key[0].split('.'):
                    if "()" in m :
                        o = getattr(o, m.strip("()"))()
                    else :
                        o = getattr(o, m)
		d[key[0]] = o if not isinstance(o, str) else _(o)
	    sortable.append(d)
        for key in order :
            sortable.sort(key=lambda d: d[key[0]], reverse=key[1])
        res = {}
        for id in ids : res[id] = []
        for r in _obj.browse(cr, user, [d['id'] for d in sortable], context=context) :
            res[getattr(r, self._fields_id).id].append(r.id)
        return res
    # end def get

    def set(self, cr, obj, id, field, values, user=None, context=None):
        for act in values :
            if act[0] == 0 : # "create"
                for k, v in self._set.iteritems() :
                    act[2][k] = v
        return (fields.one2many).set(self, cr, obj, id, field, values, user, context)
    # end def set
# end class one2many_sorted

class many2many_sorted(fields.many2many):

    _logger = logging.getLogger(__name__)

    def parse_order(self, order):
        result = []
        for col in order.split(',') :
            c = col.strip()
            if ' ASC' in c.upper() :
                result.append((c[0:c.index(' ')], False))
            elif ' DESC' in c.upper() :
                result.append((c[0:c.index(' ')], True))
            else :
                result.append((c, False))
        return list(reversed(result))
    # end def parse_order

    def __init__(self, obj, rel=None, id1=None, id2=None, string='unknown', limit=None, **args) :
        self._order  = []
        self._set    = {}
        if 'order' in args :
            self._order = self.parse_order(args['order'])
        if 'set' in args :
            self._set = args['set']
        (fields.many2many).__init__(self, obj, rel, id1=id1, id2=id2, string=string, limit=limit, **args)
    # end def __init__

    def property_value(self, cr, user, obj, name):
        property_obj = obj.pool.get('ir.property')
        prop_id = property_obj.search \
            ( cr, user
            , [ ('name', '=', name)
              , ('type', '=', 'text')
              , ('company_id','=', obj.pool.get('res.company')._company_default_get(cr, user))
              ]
            )
        if prop_id :
            return property_obj.browse(cr, user, prop_id[0]).value_text
        return False
    # end def property_value

    def get (self, cr, obj, ids, name, user=None, offset=0, context=None, values={}) :
        _obj = obj.pool.get(self._obj)
        if context and 'many2many_sorted_order' in context :
            prop = self.property_value(cr, user, obj, context['many2many_sorted_order'])
            if prop :
                order = self.parse_order(prop)
        else:
            prop = self.property_value(cr, user, obj, "%s.%s.%s.%s.order" % (self._obj, self._rel, self._id1, self._id2))
            if prop :
                order = self.parse_order(prop)
            else :
                order = self._order
        res = {}
        for id in ids : res[id] = []
        got = (fields.many2many).get(self, cr, obj, ids, name, user=user, offset=offset, context=context, values=values)
        for k, ids2 in got.iteritems() :
            sortable = []
            for r in _obj.browse(cr, user, ids2, context=context) :
                d = {}
                for key in ([('id', False)] + order) :
                    o = r
                    for m in key[0].split('.'):
                        if "()" in m :
                            o = getattr(o, m.strip("()"))()
                        else :
                            o = getattr(o, m)
                    d[key[0]] = o if not isinstance(o, str) else _(o)
                sortable.append(d)
            for key in order :
                sortable.sort(key=lambda d: d[key[0]], reverse=key[1])
            self._logger.info("many2many order criteria: %s", order) ######
            for d in sortable : self._logger.info("sorted %s", d) #############
            for r in _obj.browse(cr, user, [d['id'] for d in sortable], context=context) :
                res[k].append(r.id)
        return res
    # end def get

    def set(self, cr, model, id, name, values, user=None, context=None):
        for act in values :
            if act[0] == 0 : # "create"
                for k, v in self._set.iteritems() :
                    act[2][k] = v
        return (fields.many2many).set(self, cr, model, id, name, values, user, context)
    # end def set
# end class many2many_sorted

<|endoftext|>