<commit_before># Utility functions for the pip accelerator.
#
# Author: Peter Odding <peter.odding@paylogic.com>
# Last Change: October 28, 2015
# URL: https://github.com/paylogic/pip-accel

"""
:py:mod:`pip_accel.utils` - Utility functions
=============================================

The :py:mod:`pip_accel.utils` module defines several miscellaneous/utility
functions that are used throughout :py:mod:`pip_accel` but don't really belong
with any single module.
"""

# Standard library modules.
import errno
import logging
import os
import platform
import sys

# Modules included in our package.
from pip_accel.compat import WINDOWS

# External dependencies.
from humanfriendly import parse_path
from pip.commands.uninstall import UninstallCommand
from pkg_resources import WorkingSet

# Initialize a logger for this module.
logger = logging.getLogger(__name__)

def compact(text, **kw):
    """
    Compact whitespace in a string and format any keyword arguments into the
    resulting string. Preserves paragraphs.

    :param text: The text to compact (a string).
    :param kw: Any keyword arguments to apply using :py:func:`str.format()`.
    :returns: The compacted, formatted string.
    """
    return '\n\n'.join(' '.join(p.split()) for p in text.split('\n\n')).format(**kw)

def expand_path(pathname):
    """
    Variant of :py:func:`os.path.expanduser()` that doesn't use ``$HOME`` but
    instead uses the home directory of the effective user id. This is basically
    a workaround for ``sudo -s`` not resetting ``$HOME``.

    :param pathname: A pathname that may start with ``~/``, indicating the path
                     should be interpreted as being relative to the home
                     directory of the current (effective) user.
    :returns: The (modified) pathname.
    """
    logger.debug("Expanding pathname: %s", pathname)
    home_directory = find_home_directory()
    logger.debug("Using home directory: %s", home_directory)
    separators = set([os.sep])
    if os.altsep is not None:
        separators.add(os.altsep)
    if len(pathname) >= 2 and pathname[0] == '~' and pathname[1] in separators:
        pathname = os.path.join(home_directory, pathname[2:])
    logger.debug("Result of expansion #1: %s", pathname)
    pathname = parse_path(pathname)
    logger.debug("Result of expansion #2: %s", pathname)
    return pathname

def find_home_directory():
    """
    Look up the home directory of the effective user id.

    :returns: The pathname of the home directory (a string).

    .. note:: On Windows this uses the ``%APPDATA%`` environment variable (if
              available) and otherwise falls back to ``~/Application Data``.
    """
    if WINDOWS:
        directory = os.environ.get('APPDATA')
        if not directory:
            directory = os.path.expanduser(r'~\Application Data')
    else:
        # This module isn't available on Windows so we have to import it here.
        import pwd
        # Look up the home directory of the effective user id so we can
        # generate pathnames relative to the home directory.
        entry = pwd.getpwuid(os.getuid())
        directory = entry.pw_dir
    return directory

def is_root():
    """Detect whether we're running with super user privileges."""
    return False if WINDOWS else os.getuid() == 0

def get_python_version():
    """
    Get a string identifying the currently running Python version.

    This function generates a string that uniquely identifies the currently
    running Python implementation and version. The Python implementation is
    discovered using :py:func:`platform.python_implementation()` and the major
    and minor version numbers are extracted from :py:data:`sys.version_info`.

    :returns: A string containing the name of the Python implementation
              and the major and minor version numbers.

    Example:

    >>> from pip_accel.utils import get_python_version
    >>> get_python_version()
    'CPython-2.7'
    """
    return '%s-%i.%i' % (platform.python_implementation(),
                         sys.version_info[0],
                         sys.version_info[1])

def makedirs(path, mode=0o777):
    """
    Create a directory if it doesn't already exist (keeping concurrency in mind).

    :param path: The pathname of the directory to create (a string).
    :param mode: The mode to apply to newly created directories (an integer,
                 defaults to the octal number ``0777``).
    :returns: ``True`` when the directory was created, ``False`` if it already
              existed.
    :raises: Any exceptions raised by :py:func:`os.makedirs()` except for
             :py:data:`errno.EEXIST` (this error is swallowed and ``False`` is
             returned instead).
    """
    try:
        os.makedirs(path, mode)
        return True
    except OSError as e:
        if e.errno != errno.EEXIST:
            # We don't want to swallow errors other than EEXIST,
            # because we could be obscuring a real problem.
            raise
        return False

def same_directories(path1, path2):
    """
    Check if two pathnames refer to the same directory.

    :param path1: The first pathname (a string).
    :param path2: The second pathname (a string).
    :returns: :data:`True` if both pathnames refer to the same directory,
              :data:`False` otherwise.
    """
    if all(os.path.isdir(p) for p in (path1, path2)):
        try:
            return os.path.samefile(path1, path2)
        except AttributeError:
            # On Windows and Python 2 os.path.samefile() is unavailable.
            return os.path.realpath(path1) == os.path.realpath(path2)
    else:
        return False

def is_installed(package_name):
    """
    Check whether a package is installed in the current environment.

    :param package_name: The name of the package (a string).
    :returns: ``True`` if the package is installed, ``False`` otherwise.
    """
    return package_name.lower() in (d.key.lower() for d in WorkingSet())

def uninstall(*package_names):
    """
    Uninstall one or more packages using the Python equivalent of ``pip uninstall --yes``.

    The package(s) to uninstall must be installed, otherwise pip will raise an
    ``UninstallationError``. You can check for installed packages using
    :py:func:`is_installed()`.

    :param package_names: The names of one or more Python packages (strings).
    """
    command = UninstallCommand()
    opts, args = command.parse_args(['--yes'] + list(package_names))
    command.run(opts, args)

def find_installed_version(package_name):
    """
    Find the version of an installed package.

    :param package_name: The name of the package (a string).
    :returns: The package's version (a string) or ``None`` if the package can't
              be found.
    """
    package_name = package_name.lower()
    for distribution in WorkingSet():
        if distribution.key.lower() == package_name:
            return distribution.version

def match_option(argument, short_option, long_option):
    """
    Match a command line argument against a short and long option.

    :param argument: The command line argument (a string).
    :param short_option: The short option (a string).
    :param long_option: The long option (a string).
    :returns: ``True`` if the argument matches, ``False`` otherwise.
    """
    return short_option[1] in argument[1:] if is_short_option(argument) else argument == long_option

def is_short_option(argument):
    """
    Check if a command line argument is a short option.

    :param argument: The command line argument (a string).
    :returns: ``True`` if the argument is a short option, ``False`` otherwise.
    """
    return len(argument) >= 2 and argument[0] == '-' and argument[1] != '-'

def match_option_with_value(arguments, option, value):
    """
    Check if a list of command line options contains an option with a value.

    :param arguments: The command line arguments (a list of strings).
    :param option: The long option (a string).
    :param value: The expected value (a string).
    :returns: ``True`` if the command line contains the option/value pair,
              ``False`` otherwise.
    """
    return ('%s=%s' % (option, value) in arguments or
            contains_sublist(arguments, [option, value]))

def contains_sublist(lst, sublst):
    """
    Check if one list contains the items from another list (in the same order).

    :param lst: The main list.
    :param sublist: The sublist to check for.
    :returns: :data:`True` if the main list contains the items from the
              sublist in the same order, :data:`False` otherwise.

    Based on `this StackOverflow answer <http://stackoverflow.com/a/3314913>`_.
    """
    n = len(sublst)
    return any((sublst == lst[i:i + n]) for i in range(len(lst) - n + 1))
<commit_msg>Cleanup expand_path() function after AppVeyor debugging (now done :-)<commit_after># Utility functions for the pip accelerator.
#
# Author: Peter Odding <peter.odding@paylogic.com>
# Last Change: October 28, 2015
# URL: https://github.com/paylogic/pip-accel

"""
:py:mod:`pip_accel.utils` - Utility functions
=============================================

The :py:mod:`pip_accel.utils` module defines several miscellaneous/utility
functions that are used throughout :py:mod:`pip_accel` but don't really belong
with any single module.
"""

# Standard library modules.
import errno
import os
import platform
import sys

# Modules included in our package.
from pip_accel.compat import WINDOWS

# External dependencies.
from humanfriendly import parse_path
from pip.commands.uninstall import UninstallCommand
from pkg_resources import WorkingSet

def compact(text, **kw):
    """
    Compact whitespace in a string and format any keyword arguments into the
    resulting string. Preserves paragraphs.

    :param text: The text to compact (a string).
    :param kw: Any keyword arguments to apply using :py:func:`str.format()`.
    :returns: The compacted, formatted string.
    """
    return '\n\n'.join(' '.join(p.split()) for p in text.split('\n\n')).format(**kw)

def expand_path(pathname):
    """
    Variant of :py:func:`os.path.expanduser()` that doesn't use ``$HOME`` but
    instead uses the home directory of the effective user id. This is basically
    a workaround for ``sudo -s`` not resetting ``$HOME``.

    :param pathname: A pathname that may start with ``~/``, indicating the path
                     should be interpreted as being relative to the home
                     directory of the current (effective) user.
    :returns: The (modified) pathname.
    """
    # The following logic previously used regular expressions but that approach
    # turned out to be very error prone, hence the current contraption based on
    # direct string manipulation :-).
    home_directory = find_home_directory()
    separators = set([os.sep])
    if os.altsep is not None:
        separators.add(os.altsep)
    if len(pathname) >= 2 and pathname[0] == '~' and pathname[1] in separators:
        pathname = os.path.join(home_directory, pathname[2:])
    return parse_path(pathname)

def find_home_directory():
    """
    Look up the home directory of the effective user id.

    :returns: The pathname of the home directory (a string).

    .. note:: On Windows this uses the ``%APPDATA%`` environment variable (if
              available) and otherwise falls back to ``~/Application Data``.
    """
    if WINDOWS:
        directory = os.environ.get('APPDATA')
        if not directory:
            directory = os.path.expanduser(r'~\Application Data')
    else:
        # This module isn't available on Windows so we have to import it here.
        import pwd
        # Look up the home directory of the effective user id so we can
        # generate pathnames relative to the home directory.
        entry = pwd.getpwuid(os.getuid())
        directory = entry.pw_dir
    return directory

def is_root():
    """Detect whether we're running with super user privileges."""
    return False if WINDOWS else os.getuid() == 0

def get_python_version():
    """
    Get a string identifying the currently running Python version.

    This function generates a string that uniquely identifies the currently
    running Python implementation and version. The Python implementation is
    discovered using :py:func:`platform.python_implementation()` and the major
    and minor version numbers are extracted from :py:data:`sys.version_info`.

    :returns: A string containing the name of the Python implementation
              and the major and minor version numbers.

    Example:

    >>> from pip_accel.utils import get_python_version
    >>> get_python_version()
    'CPython-2.7'
    """
    return '%s-%i.%i' % (platform.python_implementation(),
                         sys.version_info[0],
                         sys.version_info[1])

def makedirs(path, mode=0o777):
    """
    Create a directory if it doesn't already exist (keeping concurrency in mind).

    :param path: The pathname of the directory to create (a string).
    :param mode: The mode to apply to newly created directories (an integer,
                 defaults to the octal number ``0777``).
    :returns: ``True`` when the directory was created, ``False`` if it already
              existed.
    :raises: Any exceptions raised by :py:func:`os.makedirs()` except for
             :py:data:`errno.EEXIST` (this error is swallowed and ``False`` is
             returned instead).
    """
    try:
        os.makedirs(path, mode)
        return True
    except OSError as e:
        if e.errno != errno.EEXIST:
            # We don't want to swallow errors other than EEXIST,
            # because we could be obscuring a real problem.
            raise
        return False

def same_directories(path1, path2):
    """
    Check if two pathnames refer to the same directory.

    :param path1: The first pathname (a string).
    :param path2: The second pathname (a string).
    :returns: :data:`True` if both pathnames refer to the same directory,
              :data:`False` otherwise.
    """
    if all(os.path.isdir(p) for p in (path1, path2)):
        try:
            return os.path.samefile(path1, path2)
        except AttributeError:
            # On Windows and Python 2 os.path.samefile() is unavailable.
            return os.path.realpath(path1) == os.path.realpath(path2)
    else:
        return False

def is_installed(package_name):
    """
    Check whether a package is installed in the current environment.

    :param package_name: The name of the package (a string).
    :returns: ``True`` if the package is installed, ``False`` otherwise.
    """
    return package_name.lower() in (d.key.lower() for d in WorkingSet())

def uninstall(*package_names):
    """
    Uninstall one or more packages using the Python equivalent of ``pip uninstall --yes``.

    The package(s) to uninstall must be installed, otherwise pip will raise an
    ``UninstallationError``. You can check for installed packages using
    :py:func:`is_installed()`.

    :param package_names: The names of one or more Python packages (strings).
    """
    command = UninstallCommand()
    opts, args = command.parse_args(['--yes'] + list(package_names))
    command.run(opts, args)

def find_installed_version(package_name):
    """
    Find the version of an installed package.

    :param package_name: The name of the package (a string).
    :returns: The package's version (a string) or ``None`` if the package can't
              be found.
    """
    package_name = package_name.lower()
    for distribution in WorkingSet():
        if distribution.key.lower() == package_name:
            return distribution.version

def match_option(argument, short_option, long_option):
    """
    Match a command line argument against a short and long option.

    :param argument: The command line argument (a string).
    :param short_option: The short option (a string).
    :param long_option: The long option (a string).
    :returns: ``True`` if the argument matches, ``False`` otherwise.
    """
    return short_option[1] in argument[1:] if is_short_option(argument) else argument == long_option

def is_short_option(argument):
    """
    Check if a command line argument is a short option.

    :param argument: The command line argument (a string).
    :returns: ``True`` if the argument is a short option, ``False`` otherwise.
    """
    return len(argument) >= 2 and argument[0] == '-' and argument[1] != '-'

def match_option_with_value(arguments, option, value):
    """
    Check if a list of command line options contains an option with a value.

    :param arguments: The command line arguments (a list of strings).
    :param option: The long option (a string).
    :param value: The expected value (a string).
    :returns: ``True`` if the command line contains the option/value pair,
              ``False`` otherwise.
    """
    return ('%s=%s' % (option, value) in arguments or
            contains_sublist(arguments, [option, value]))

def contains_sublist(lst, sublst):
    """
    Check if one list contains the items from another list (in the same order).

    :param lst: The main list.
    :param sublist: The sublist to check for.
    :returns: :data:`True` if the main list contains the items from the
              sublist in the same order, :data:`False` otherwise.

    Based on `this StackOverflow answer <http://stackoverflow.com/a/3314913>`_.
    """
    n = len(sublst)
    return any((sublst == lst[i:i + n]) for i in range(len(lst) - n + 1))
<|endoftext|>


<commit_before># Python Security Project (PySec) and its related class files.
#
# PySec is a set of tools for secure application development under Linux
#
# Copyright 2014 PySec development team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# -*- coding: ascii -*-
"""Various math utilities"""

def distance(p1, p2):
    """Return the distance between two points.
    The point is a tuple: (x, y).
    """
    x1, y1 = p1
    x2, y2 = p2
    return ((x2 - x1) ** 2 + (y2 - y1) ** 2) ** 0.5

<commit_msg>add centroid, next_multiple, div_ceil<commit_after># Python Security Project (PySec) and its related class files.
#
# PySec is a set of tools for secure application development under Linux
#
# Copyright 2014 PySec development team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# -*- coding: ascii -*-
"""Various math utilities"""
import stats

def distance(p1, p2):
    """Return the distance between two points.
    The point is a tuple: (x, y).
    """
    x1, y1 = p1
    x2, y2 = p2
    return ((x2 - x1) ** 2 + (y2 - y1) ** 2) ** 0.5

def centroid(*points):
    """Calculate the centroid point of a points set in a 2-dimensional space"""
    cx = stats.avg()
    cy = stats.avg()
    cx.next()
    cy.next()
    for x, y in points:
        x = float(x)
        y = float(y)
        cx.send(x)
        cy.send(y)
    return cx.next(), cy.next()

def div_ceil(n, m):
    """Return the smallest integer x such that x * m >= n."""
    q, r = divmod(n, m)
    return q + int(r != 0)

def next_multiple(n, m):
    """Return the smallest multiple of m greater than n"""
    return div_ceil(n, m) * m

<|endoftext|>


<commit_before># coding: utf-8
import json
from collections import namedtuple

from pysuru.base import BaseAPI, ObjectMixin
from pysuru.services import ServiceInstanceAPI

app_attrs = (
    'name',
    'description',
    'ip',
    'owner',
    'teamowner',
    'platform',
    'pool',
)

class App(namedtuple('App', app_attrs), ObjectMixin):
    def services(self):
        api = ServiceInstanceAPI(self._client)
        return api.filter_by_app(self.name)

class AppsAPI(BaseAPI):

    def list(self):
        result = []
        _, response = self.client.get('/apps')
        for data in response:
            result.append(App.create(self.client, **data))
        return result

    def get(self, name):
        status, response = self.client.get('/apps/{}'.format(name))
        if status == 404:
            raise AppDoesNotExists('App {} not found'.format(name))

        if status != 200:
            return False

        data = {
            'name': response['name'],
            'description': response['description'],
            'ip': response['ip'],
            'owner': response['owner'],
            'teamowner': response['teamowner'],
            'platform': response['platform'],
            'pool': response['pool'],
        }
        return App.create(self.client, **data)

    def add(self, data):
        body = json.dumps(data)
        http_response = self.client.urlopen('POST', '/apps', body=body)
        if http_response.status == 200:
            return True
        elif http_response.status == 409:
            raise AppAlreadyExists('App {} already exists'.format(data['name']))
        return False

    def update(self, name, data):
        body = json.dumps(data)
        http_response = self.client.urlopen('POST', '/apps/{}'.format(name),
                                            body=body)
        if http_response.status == 200:
            return True
        else:
            return False

    def bind_service(self, name, service_instance):
        path = ('/services/{}/instances/{}/{}'
                .format(service_instance.type, service_instance.name, name))
        http_response = self.client.urlopen('PUT', path)
        if http_response.status == 200:
            return True
        return False

class AppAlreadyExists(Exception):
    pass

class AppDoesNotExists(Exception):
    pass
<commit_msg>Fix app creation<commit_after># coding: utf-8
import json
from collections import namedtuple
from urllib.parse import urlencode

from pysuru.base import BaseAPI, ObjectMixin
from pysuru.services import ServiceInstanceAPI

app_attrs = (
    'name',
    'description',
    'ip',
    'owner',
    'teamowner',
    'platform',
    'pool',
)

class App(namedtuple('App', app_attrs), ObjectMixin):
    def services(self):
        api = ServiceInstanceAPI(self._client)
        return api.filter_by_app(self.name)

class AppsAPI(BaseAPI):

    def list(self):
        result = []
        _, response = self.client.get('/apps')
        for data in response:
            result.append(App.create(self.client, **data))
        return result

    def get(self, name):
        status, response = self.client.get('/apps/{}'.format(name))
        if status == 404:
            raise AppDoesNotExists('App {} not found'.format(name))

        if status != 200:
            return False

        data = {
            'name': response['name'],
            'description': response['description'],
            'ip': response['ip'],
            'owner': response['owner'],
            'teamowner': response['teamowner'],
            'platform': response['platform'],
            'pool': response['pool'],
        }
        return App.create(self.client, **data)

    def create(self, data):
        body = urlencode(data)
        http_response = self.client.urlopen('POST', '/apps', body=body)
        if http_response.status == 200:
            return True
        elif http_response.status == 409:
            raise AppAlreadyExists('App {} already exists'.format(data['name']))
        return False

    def update(self, name, data):
        body = json.dumps(data)
        http_response = self.client.urlopen('POST', '/apps/{}'.format(name),
                                            body=body)
        if http_response.status == 200:
            return True
        else:
            return False

    def bind_service(self, name, service_instance):
        path = ('/services/{}/instances/{}/{}'
                .format(service_instance.type, service_instance.name, name))
        http_response = self.client.urlopen('PUT', path)
        if http_response.status == 200:
            return True
        return False

class AppAlreadyExists(Exception):
    pass

class AppDoesNotExists(Exception):
    pass
<|endoftext|>


<commit_before>"""Classes for use in unit conversion."""
import numpy
from scipy.interpolate import PchipInterpolator

import pytac
from pytac.exceptions import UnitsException

def unit_function(value):
    """Default value for the pre and post functions used in unit conversion.

    Args:
        value (float): The value to be converted.

    Returns:
        float: The result of the conversion.
    """
    return value

class UnitConv(object):
    """Class to convert between physics and engineering units.

    This class does not do conversion but does return values if the target
    units are the same as the provided units. Subclasses should implement
    _raw_eng_to_phys() and _raw_phys_to_eng() in order to provide complete
    unit conversion.

    The two arguments to this function represent functions that are
    applied to the result of the initial conversion. One happens after
    the conversion, the other happens before the conversion back.

    **Attributes:**

    Attributes:
        name (str): An identifier for the unit conversion object.
        eng_units (str): The unit type of the post conversion engineering
                          value.
        phys_units (str): The unit type of the post conversion physics value.

    .. Private Attributes:
           _post_eng_to_phys (function): Function to be applied after the
                                         initial conversion.
           _pre_phys_to_eng (function): Function to be applied before the
                                         initial conversion.
    """
    def __init__(self, post_eng_to_phys=unit_function,
                 pre_phys_to_eng=unit_function, engineering_units='',
                 physics_units='', name=None):
        """
        Args:
            post_eng_to_phys (function): Function to be applied after the
                                          initial conversion.
            pre_phys_to_eng (function): Function to be applied before the
                                         initial conversion.
            engineering_units (str): The unit type of the post conversion
                                      engineering value.
            physics_units (str): The unit type of the post conversion physics
                                  value.
            name (str): An identifier for the unit conversion object.

        **Methods:**
        """
        self.name = name
        self._post_eng_to_phys = post_eng_to_phys
        self._pre_phys_to_eng = pre_phys_to_eng
        self.eng_units = engineering_units
        self.phys_units = physics_units
        self.lower_limit = None
        self.upper_limit = None

    def __str__(self):
        string_rep = self.__class__.__name__
        if self.name is not None:
            string_rep += " {}".format(self.name)
        return string_rep

    def set_post_eng_to_phys(self, post_eng_to_phys):
        """Set the function to be applied after the initial conversion.
        Args:
            post_eng_to_phys (function): Function to be applied after the
                                          initial conversion.
        """
        self._post_eng_to_phys = post_eng_to_phys

    def set_pre_phys_to_eng(self, pre_phys_to_eng):
        """Set the function to be applied before the initial conversion.
        Args:
            pre_phys_to_eng (function): Function to be applied before the
                                         initial conversion.
        """
        self._pre_phys_to_eng = pre_phys_to_eng

    def _raw_eng_to_phys(self, value):
        """Function to be implemented by child classes.

        Args:
            value (float): The engineering value to be converted to physics
                            units.
        """
        raise NotImplementedError(
            '{0}: No eng-to-phys conversion provided'.format(self)
        )

    def eng_to_phys(self, value):
        """Function that does the unit conversion.

        Conversion from engineering to physics units. An additional function
        may be cast on the initial conversion.

        Args:
            value (float): Value to be converted from engineering to physics
                            units.

        Returns:
            float: The result value.

        Raises:
            UnitsException: If the conversion is invalid; i.e. if there are no
                            solutions, or multiple, within conversion limits.
        """
        if self.lower_limit is not None:
            if value < self.lower_limit:
                raise UnitsException("{0}: Input less than lower "
                                     "conversion limit ({1})."
                                     .format(self, self.lower_limit))
        if self.upper_limit is not None:
            if value > self.upper_limit:
                raise UnitsException("{0}: Input greater than "
                                     "upper conversion limit ({1})."
                                     .format(self, self.upper_limit))
        results = self._raw_eng_to_phys(value)
        valid_results = [self._post_eng_to_phys(result)
                         for result in results]
        if len(valid_results) == 1:
            result = valid_results[0]
        elif len(valid_results) == 0:
            # This will not occur for our existing NullUnitConv,
            # PchipUintConv, and PolyUnitConv classes.
            raise UnitsException("{0}: A corresponding physics value "
                                 "does not exist.".format(self))
        else:
            # This will not occur for our existing NullUnitConv,
            # PchipUintConv, and PolyUnitConv classes.
            raise UnitsException("{0}: There are multiple "
                                 "corresponding physics values ({1})."
                                 .format(self, valid_results))
        return result

    def _raw_phys_to_eng(self, value):
        """Function to be implemented by child classes.

        Args:
            value (float): The physics value to be converted to engineering
                            units.
        """
        raise NotImplementedError(
            '{0}: No phys-to-eng conversion provided'.format(self)
        )

    def phys_to_eng(self, value):
        """Function that does the unit conversion.

        Conversion from physics to engineering units. An additional function
        may be cast on the initial conversion.

        Args:
            value (float): Value to be converted from physics to engineering
                            units.

        Returns:
            float: The result value.

        Raises:
            UnitsException: If the conversion is invalid; i.e. if there are no
                            solutions, or multiple, within conversion limits.
        """
        adjusted_value = self._pre_phys_to_eng(value)
        results = self._raw_phys_to_eng(adjusted_value)

        if self.lower_limit is not None:
            results = [r for r in results if r >= self.lower_limit]
        if self.upper_limit is not None:
            results = [r for r in results if r <= self.upper_limit]
        if len(results) == 1:
            return results[0]
        elif len(results) == 0:
            raise UnitsException("{0}: no conversion result "
                                 "within conversion limits ({1}, "
                                 "{2}).".format(self,
                                                self.lower_limit,
                                                self.upper_limit))
        else:
            raise UnitsException("{0}: There are multiple "
                                 "corresponding engineering values ({1})."
                                 .format(self, results))

    def convert(self, value, origin, target):
        """Convert between two different unit types and chek the validity of
        the result.

        Args:
            value (float): the value to be converted
            origin (str): pytac.ENG or pytac.PHYS
            target (str): pytac.ENG or pytac.PHYS

        Returns:
            float: The resulting value.

        Raises:
            UnitsException: If the conversion is invalid; i.e. if there are no
                             solutions, or multiple, within conversion limits.
        """
        if origin == target:
            return value
        elif origin == pytac.ENG and target == pytac.PHYS:
            return self.eng_to_phys(value)
        elif origin == pytac.PHYS and target == pytac.ENG:
            return self.phys_to_eng(value)
        else:
            raise UnitsException("{0}: Conversion from {1} to {2} "
                                 "not understood.".format(self, origin,
                                                          target))

    def set_conversion_limits(self, lower_limit, upper_limit):
        """Conversion limits to be applied before or after a conversion take
        place. Limits should be set in in engineering units.

        Args:
            lower_limit (float): the lower conversion limit
            upper_limit (float): the upper conversion limit
        """
        self.lower_limit = lower_limit
        self.upper_limit = upper_limit

class PolyUnitConv(UnitConv):
    """Linear interpolation for converting between physics and engineering
    units.

    **Attributes:**

    Attributes:
        p (poly1d): A one-dimensional polynomial of coefficients.
        name (str): An identifier for the unit conversion object.
        eng_units (str): The unit type of the post conversion engineering
                          value.
        phys_units (str): The unit type of the post conversion physics value.

    .. Private Attributes:
           _post_eng_to_phys (function): Function to be applied after the
                                         initial conversion.
           _pre_phys_to_eng (function): Function to be applied before the
                                         initial conversion.
    """
    def __init__(self, coef, post_eng_to_phys=unit_function,
                 pre_phys_to_eng=unit_function, engineering_units='',
                 physics_units='', name=None):
        """
        Args:
            coef (array-like): The polynomial's coefficients, in decreasing
                                powers.
            post_eng_to_phys (float): The value after conversion between ENG
                                       and PHYS.
            pre_eng_to_phys (float): The value before conversion.
            engineering_units (str): The unit type of the post conversion
                                      engineering value.
            physics_units (str): The unit type of the post conversion physics
                                  value.
            name (str): An identifier for the unit conversion object.
        """
        super(self.__class__, self).__init__(post_eng_to_phys,
                                             pre_phys_to_eng,
                                             engineering_units,
                                             physics_units,
                                             name)
        self.p = numpy.poly1d(coef)

    def _raw_eng_to_phys(self, eng_value):
        """Convert between engineering and physics units.

        Args:
            eng_value (float): The engineering value to be converted to physics
                                units.

        Returns:
            list: Containing the converted physics value from the given
                    engineering value.
        """
        return [self.p(eng_value)]

    def _raw_phys_to_eng(self, physics_value):
        """Convert between physics and engineering units.

        Args:
            physics_value (float): The physics value to be converted to
                                    engineering units.

        Returns:
            list: Containing all posible real engineering values converted
                   from the given physics value.
        """
        roots = set((self.p - physics_value).roots)  # remove duplicates
        valid_roots = []
        for root in roots:  # remove imaginary roots
            if not numpy.issubdtype(root.dtype, numpy.complexfloating):
                valid_roots.append(root)
        return valid_roots

class PchipUnitConv(UnitConv):
    """Piecewise Cubic Hermite Interpolating Polynomial unit conversion.

    **Attributes:**

    Attributes:
        x (list): A list of points on the x axis. These must be in increasing
                   order for the interpolation to work. Otherwise, a ValueError
                   is raised.
        y (list): A list of points on the y axis. These must be in increasing
                   or decreasing order. Otherwise, a ValueError is raised.
        pp (PchipInterpolator): A pchip one-dimensional monotonic cubic
                                 interpolation of points on both x and y axes.

        name (str): An identifier for the unit conversion object.
        eng_units (str): The unit type of the post conversion engineering
                          value.
        phys_units (str): The unit type of the post conversion physics value.

    .. Private Attributes:
           _post_eng_to_phys (function): Function to be applied after the
                                         initial conversion.
           _pre_phys_to_eng (function): Function to be applied before the
                                         initial conversion.
    """
    def __init__(self, x, y, post_eng_to_phys=unit_function,
                 pre_phys_to_eng=unit_function, engineering_units='',
                 physics_units='', name=None):
        """
        Args:
            x (list): A list of points on the x axis. These must be in
                       increasing order for the interpolation to work.
                       Otherwise, a ValueError is raised.
            y (list): A list of points on the y axis. These must be in
                       increasing or decreasing order. Otherwise, a ValueError
                       is raised.
            engineering_units (str): The unit type of the post conversion
                                      engineering value.
            physics_units (str): The unit type of the post conversion physics
                                  value.
            name (str): An identifier for the unit conversion object.

        Raises:
            ValueError: if coefficients are not appropriately monotonic.
        """
        super(self.__class__, self).__init__(post_eng_to_phys,
                                             pre_phys_to_eng,
                                             engineering_units,
                                             physics_units,
                                             name)
        self.x = x
        self.y = y
        self.pp = PchipInterpolator(x, y)
        # Set conversion limits to PChip bounds if they are not already set.
        if self.lower_limit is None:
            self.lower_limit = self.x[0]
        if self.upper_limit is None:
            self.upper_limit = self.x[-1]
        # Note that the x coefficients are checked by the PchipInterpolator
        # constructor.
        y_diff = numpy.diff(y)
        if not ((numpy.all(y_diff > 0)) or (numpy.all((y_diff < 0)))):
            raise ValueError("y coefficients must be monotonically "
                             "increasing or decreasing.")

    def _raw_eng_to_phys(self, eng_value):
        """Convert between engineering and physics units.

        Args:
            eng_value (float): The engineering value to be converted to physics
                                units.
        Returns:
            list: Containing the converted physics value from the given
                    engineering value.
        """
        return [self.pp(eng_value)]

    def _raw_phys_to_eng(self, physics_value):
        """Convert between physics and engineering units.

        Args:
            physics_value (float): The physics value to be converted to
                                    engineering units.

        Returns:
            list: Containing all posible real engineering values converted
                   from the given physics value.
        """
        y = [val - physics_value for val in self.y]
        new_pp = PchipInterpolator(self.x, y)
        roots = set(new_pp.roots())  # remove duplicates
        valid_roots = []
        for root in roots:  # remove imaginary roots
            if not numpy.issubdtype(root.dtype, numpy.complexfloating):
                valid_roots.append(root)
        return valid_roots

class NullUnitConv(UnitConv):
    """Returns input value without performing any conversions.

    **Attributes:**

    Attributes:
        name (str): An identifier for the unit conversion object.
        eng_units (str): The unit type of the post conversion engineering
                          value.
        phys_units (str): The unit type of the post conversion physics value.

    .. Private Attributes:
           _post_eng_to_phys (function): Always unit_function as no conversion
                                          is performed.
           _pre_phys_to_eng (function): Always unit_function as no conversion
                                          is performed.
    """
    def __init__(self, engineering_units='', physics_units='', name=None):
        """
        Args:
            engineering_units (str): The unit type of the post conversion
                                      engineering value.
            physics_units (str): The unit type of the post conversion physics
                                  value.
            name (str): An identifier for the unit conversion object.
        """
        super(self.__class__, self).__init__(unit_function, unit_function,
                                             engineering_units, physics_units,
                                             name=None)

    def _raw_eng_to_phys(self, eng_value):
        """Doesn't convert between engineering and physics units.

        Maintains the same syntax as the other UnitConv classes for
        compatibility, but does not perform any conversion.

        Args:
            eng_value (float): The engineering value to be returned unchanged.
        Returns:
            list: Containing the unconverted given engineering value.
        """
        return [eng_value]

    def _raw_phys_to_eng(self, phys_value):
        """Doesn't convert between physics and engineering units.

        Maintains the same syntax as the other UnitConv classes for
        compatibility, but does not perform any conversion.

        Args:
            physics_value (float): The physics value to be returned unchanged.

        Returns:
            list: Containing the unconverted given physics value.
        """
        return [phys_value]
<commit_msg>Restore more informative error message for unitconv failure.<commit_after>"""Classes for use in unit conversion."""
import numpy
from scipy.interpolate import PchipInterpolator

import pytac
from pytac.exceptions import UnitsException

def unit_function(value):
    """Default value for the pre and post functions used in unit conversion.

    Args:
        value (float): The value to be converted.

    Returns:
        float: The result of the conversion.
    """
    return value

class UnitConv(object):
    """Class to convert between physics and engineering units.

    This class does not do conversion but does return values if the target
    units are the same as the provided units. Subclasses should implement
    _raw_eng_to_phys() and _raw_phys_to_eng() in order to provide complete
    unit conversion.

    The two arguments to this function represent functions that are
    applied to the result of the initial conversion. One happens after
    the conversion, the other happens before the conversion back.

    **Attributes:**

    Attributes:
        name (str): An identifier for the unit conversion object.
        eng_units (str): The unit type of the post conversion engineering
                          value.
        phys_units (str): The unit type of the post conversion physics value.

    .. Private Attributes:
           _post_eng_to_phys (function): Function to be applied after the
                                         initial conversion.
           _pre_phys_to_eng (function): Function to be applied before the
                                         initial conversion.
    """
    def __init__(self, post_eng_to_phys=unit_function,
                 pre_phys_to_eng=unit_function, engineering_units='',
                 physics_units='', name=None):
        """
        Args:
            post_eng_to_phys (function): Function to be applied after the
                                          initial conversion.
            pre_phys_to_eng (function): Function to be applied before the
                                         initial conversion.
            engineering_units (str): The unit type of the post conversion
                                      engineering value.
            physics_units (str): The unit type of the post conversion physics
                                  value.
            name (str): An identifier for the unit conversion object.

        **Methods:**
        """
        self.name = name
        self._post_eng_to_phys = post_eng_to_phys
        self._pre_phys_to_eng = pre_phys_to_eng
        self.eng_units = engineering_units
        self.phys_units = physics_units
        self.lower_limit = None
        self.upper_limit = None

    def __str__(self):
        string_rep = self.__class__.__name__
        if self.name is not None:
            string_rep += " {}".format(self.name)
        return string_rep

    def set_post_eng_to_phys(self, post_eng_to_phys):
        """Set the function to be applied after the initial conversion.
        Args:
            post_eng_to_phys (function): Function to be applied after the
                                          initial conversion.
        """
        self._post_eng_to_phys = post_eng_to_phys

    def set_pre_phys_to_eng(self, pre_phys_to_eng):
        """Set the function to be applied before the initial conversion.
        Args:
            pre_phys_to_eng (function): Function to be applied before the
                                         initial conversion.
        """
        self._pre_phys_to_eng = pre_phys_to_eng

    def _raw_eng_to_phys(self, value):
        """Function to be implemented by child classes.

        Args:
            value (float): The engineering value to be converted to physics
                            units.
        """
        raise NotImplementedError(
            '{0}: No eng-to-phys conversion provided'.format(self)
        )

    def eng_to_phys(self, value):
        """Function that does the unit conversion.

        Conversion from engineering to physics units. An additional function
        may be cast on the initial conversion.

        Args:
            value (float): Value to be converted from engineering to physics
                            units.

        Returns:
            float: The result value.

        Raises:
            UnitsException: If the conversion is invalid; i.e. if there are no
                            solutions, or multiple, within conversion limits.
        """
        if self.lower_limit is not None:
            if value < self.lower_limit:
                raise UnitsException("{0}: Input less than lower "
                                     "conversion limit ({1})."
                                     .format(self, self.lower_limit))
        if self.upper_limit is not None:
            if value > self.upper_limit:
                raise UnitsException("{0}: Input greater than "
                                     "upper conversion limit ({1})."
                                     .format(self, self.upper_limit))
        results = self._raw_eng_to_phys(value)
        valid_results = [self._post_eng_to_phys(result)
                         for result in results]
        if len(valid_results) == 1:
            result = valid_results[0]
        elif len(valid_results) == 0:
            # This will not occur for our existing NullUnitConv,
            # PchipUintConv, and PolyUnitConv classes.
            raise UnitsException("{0}: A corresponding physics value "
                                 "does not exist.".format(self))
        else:
            # This will not occur for our existing NullUnitConv,
            # PchipUintConv, and PolyUnitConv classes.
            raise UnitsException("{0}: There are multiple "
                                 "corresponding physics values ({1})."
                                 .format(self, valid_results))
        return result

    def _raw_phys_to_eng(self, value):
        """Function to be implemented by child classes.

        Args:
            value (float): The physics value to be converted to engineering
                            units.
        """
        raise NotImplementedError(
            '{0}: No phys-to-eng conversion provided'.format(self)
        )

    def phys_to_eng(self, value):
        """Function that does the unit conversion.

        Conversion from physics to engineering units. An additional function
        may be cast on the initial conversion.

        Args:
            value (float): Value to be converted from physics to engineering
                            units.

        Returns:
            float: The result value.

        Raises:
            UnitsException: If the conversion is invalid; i.e. if there are no
                            solutions, or multiple, within conversion limits.
        """
        adjusted_value = self._pre_phys_to_eng(value)
        results = self._raw_phys_to_eng(adjusted_value)

        valid_results = results[:]

        if self.lower_limit is not None:
            valid_results = [r for r in valid_results if r >= self.lower_limit]
        if self.upper_limit is not None:
            valid_results = [r for r in valid_results if r <= self.upper_limit]
        if len(valid_results) == 1:
            return valid_results[0]
        elif len(valid_results) == 0:
            raise UnitsException("{0}: none of conversion results {1} "
                                 "within conversion limits ({2}, "
                                 "{3}).".format(self, results,
                                                self.lower_limit,
                                                self.upper_limit))
        else:
            raise UnitsException("{0}: There are multiple "
                                 "corresponding engineering values ({1})."
                                 .format(self, valid_results))

    def convert(self, value, origin, target):
        """Convert between two different unit types and chek the validity of
        the result.

        Args:
            value (float): the value to be converted
            origin (str): pytac.ENG or pytac.PHYS
            target (str): pytac.ENG or pytac.PHYS

        Returns:
            float: The resulting value.

        Raises:
            UnitsException: If the conversion is invalid; i.e. if there are no
                             solutions, or multiple, within conversion limits.
        """
        if origin == target:
            return value
        elif origin == pytac.ENG and target == pytac.PHYS:
            return self.eng_to_phys(value)
        elif origin == pytac.PHYS and target == pytac.ENG:
            return self.phys_to_eng(value)
        else:
            raise UnitsException("{0}: Conversion from {1} to {2} "
                                 "not understood.".format(self, origin,
                                                          target))

    def set_conversion_limits(self, lower_limit, upper_limit):
        """Conversion limits to be applied before or after a conversion take
        place. Limits should be set in in engineering units.

        Args:
            lower_limit (float): the lower conversion limit
            upper_limit (float): the upper conversion limit
        """
        self.lower_limit = lower_limit
        self.upper_limit = upper_limit

class PolyUnitConv(UnitConv):
    """Linear interpolation for converting between physics and engineering
    units.

    **Attributes:**

    Attributes:
        p (poly1d): A one-dimensional polynomial of coefficients.
        name (str): An identifier for the unit conversion object.
        eng_units (str): The unit type of the post conversion engineering
                          value.
        phys_units (str): The unit type of the post conversion physics value.

    .. Private Attributes:
           _post_eng_to_phys (function): Function to be applied after the
                                         initial conversion.
           _pre_phys_to_eng (function): Function to be applied before the
                                         initial conversion.
    """
    def __init__(self, coef, post_eng_to_phys=unit_function,
                 pre_phys_to_eng=unit_function, engineering_units='',
                 physics_units='', name=None):
        """
        Args:
            coef (array-like): The polynomial's coefficients, in decreasing
                                powers.
            post_eng_to_phys (float): The value after conversion between ENG
                                       and PHYS.
            pre_eng_to_phys (float): The value before conversion.
            engineering_units (str): The unit type of the post conversion
                                      engineering value.
            physics_units (str): The unit type of the post conversion physics
                                  value.
            name (str): An identifier for the unit conversion object.
        """
        super(self.__class__, self).__init__(post_eng_to_phys,
                                             pre_phys_to_eng,
                                             engineering_units,
                                             physics_units,
                                             name)
        self.p = numpy.poly1d(coef)

    def _raw_eng_to_phys(self, eng_value):
        """Convert between engineering and physics units.

        Args:
            eng_value (float): The engineering value to be converted to physics
                                units.

        Returns:
            list: Containing the converted physics value from the given
                    engineering value.
        """
        return [self.p(eng_value)]

    def _raw_phys_to_eng(self, physics_value):
        """Convert between physics and engineering units.

        Args:
            physics_value (float): The physics value to be converted to
                                    engineering units.

        Returns:
            list: Containing all posible real engineering values converted
                   from the given physics value.
        """
        roots = set((self.p - physics_value).roots)  # remove duplicates
        valid_roots = []
        for root in roots:  # remove imaginary roots
            if not numpy.issubdtype(root.dtype, numpy.complexfloating):
                valid_roots.append(root)
        return valid_roots

class PchipUnitConv(UnitConv):
    """Piecewise Cubic Hermite Interpolating Polynomial unit conversion.

    **Attributes:**

    Attributes:
        x (list): A list of points on the x axis. These must be in increasing
                   order for the interpolation to work. Otherwise, a ValueError
                   is raised.
        y (list): A list of points on the y axis. These must be in increasing
                   or decreasing order. Otherwise, a ValueError is raised.
        pp (PchipInterpolator): A pchip one-dimensional monotonic cubic
                                 interpolation of points on both x and y axes.

        name (str): An identifier for the unit conversion object.
        eng_units (str): The unit type of the post conversion engineering
                          value.
        phys_units (str): The unit type of the post conversion physics value.

    .. Private Attributes:
           _post_eng_to_phys (function): Function to be applied after the
                                         initial conversion.
           _pre_phys_to_eng (function): Function to be applied before the
                                         initial conversion.
    """
    def __init__(self, x, y, post_eng_to_phys=unit_function,
                 pre_phys_to_eng=unit_function, engineering_units='',
                 physics_units='', name=None):
        """
        Args:
            x (list): A list of points on the x axis. These must be in
                       increasing order for the interpolation to work.
                       Otherwise, a ValueError is raised.
            y (list): A list of points on the y axis. These must be in
                       increasing or decreasing order. Otherwise, a ValueError
                       is raised.
            engineering_units (str): The unit type of the post conversion
                                      engineering value.
            physics_units (str): The unit type of the post conversion physics
                                  value.
            name (str): An identifier for the unit conversion object.

        Raises:
            ValueError: if coefficients are not appropriately monotonic.
        """
        super(self.__class__, self).__init__(post_eng_to_phys,
                                             pre_phys_to_eng,
                                             engineering_units,
                                             physics_units,
                                             name)
        self.x = x
        self.y = y
        self.pp = PchipInterpolator(x, y)
        # Set conversion limits to PChip bounds if they are not already set.
        if self.lower_limit is None:
            self.lower_limit = self.x[0]
        if self.upper_limit is None:
            self.upper_limit = self.x[-1]
        # Note that the x coefficients are checked by the PchipInterpolator
        # constructor.
        y_diff = numpy.diff(y)
        if not ((numpy.all(y_diff > 0)) or (numpy.all((y_diff < 0)))):
            raise ValueError("y coefficients must be monotonically "
                             "increasing or decreasing.")

    def _raw_eng_to_phys(self, eng_value):
        """Convert between engineering and physics units.

        Args:
            eng_value (float): The engineering value to be converted to physics
                                units.
        Returns:
            list: Containing the converted physics value from the given
                    engineering value.
        """
        return [self.pp(eng_value)]

    def _raw_phys_to_eng(self, physics_value):
        """Convert between physics and engineering units.

        Args:
            physics_value (float): The physics value to be converted to
                                    engineering units.

        Returns:
            list: Containing all posible real engineering values converted
                   from the given physics value.
        """
        y = [val - physics_value for val in self.y]
        new_pp = PchipInterpolator(self.x, y)
        roots = set(new_pp.roots())  # remove duplicates
        valid_roots = []
        for root in roots:  # remove imaginary roots
            if not numpy.issubdtype(root.dtype, numpy.complexfloating):
                valid_roots.append(root)
        return valid_roots

class NullUnitConv(UnitConv):
    """Returns input value without performing any conversions.

    **Attributes:**

    Attributes:
        name (str): An identifier for the unit conversion object.
        eng_units (str): The unit type of the post conversion engineering
                          value.
        phys_units (str): The unit type of the post conversion physics value.

    .. Private Attributes:
           _post_eng_to_phys (function): Always unit_function as no conversion
                                          is performed.
           _pre_phys_to_eng (function): Always unit_function as no conversion
                                          is performed.
    """
    def __init__(self, engineering_units='', physics_units='', name=None):
        """
        Args:
            engineering_units (str): The unit type of the post conversion
                                      engineering value.
            physics_units (str): The unit type of the post conversion physics
                                  value.
            name (str): An identifier for the unit conversion object.
        """
        super(self.__class__, self).__init__(unit_function, unit_function,
                                             engineering_units, physics_units,
                                             name=None)

    def _raw_eng_to_phys(self, eng_value):
        """Doesn't convert between engineering and physics units.

        Maintains the same syntax as the other UnitConv classes for
        compatibility, but does not perform any conversion.

        Args:
            eng_value (float): The engineering value to be returned unchanged.
        Returns:
            list: Containing the unconverted given engineering value.
        """
        return [eng_value]

    def _raw_phys_to_eng(self, phys_value):
        """Doesn't convert between physics and engineering units.

        Maintains the same syntax as the other UnitConv classes for
        compatibility, but does not perform any conversion.

        Args:
            physics_value (float): The physics value to be returned unchanged.

        Returns:
            list: Containing the unconverted given physics value.
        """
        return [phys_value]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import argparse
import importlib
import inspect
import re
import sys

import pytest

OBJECT_BLACKLIST = (
    # pytest internals
    '_pytest.runner:exit',
    '_pytest.runner:skip',
    '_pytest.skipping:xfail',

    # unconditional exit
    'posix:_exit',
    '_signal:default_int_handler',
    'atexit.register',

    # low level crashes
    'numpy.fft.fftpack_lite:cffti',
    'numpy.fft.fftpack_lite:rffti',
    'appnope._nope:beginActivityWithOptions',
    'ctypes:string_at',
    'ctypes:wstring_at',
    'gc:_dump_rpy_heap',
    'gc:dump_rpy_heap',
    'matplotlib._image:Image',

    # hangs
    'Tkinter:mainloop',
    'astkit.compat.py3:execfile',
    'click.termui:getchar',
    'click.termui:edit',
    'click.termui:hidden_prompt_func',
    'eventlet.hubs:trampoline',
    'getpass:getpass',
    'getpass:unix_getpass',
    'matplotlib.font_manager:FontManager',
    'pty:_copy',
    'pydoc:serve',
    'pyexpat:ErrorString',
    'skimage:_test',
    'skimage:test',
)

def pytest_addoption(parser):
    group = parser.getgroup('wish')
    group.addoption('--wish-modules', default=(), nargs='+',
                    help="Space separated list of module names.")
    group.addoption('--wish-includes', nargs='+',
                    help="Space separated list of regexs matching full object names to include.")
    group.addoption('--wish-excludes', default=(), nargs='+',
                    help="Space separated list of regexs matching full object names to exclude.")
    group.addoption('--wish-objects', type=argparse.FileType('r'),
                    help="File of full object names to include.")
    group.addoption('--wish-fail', action='store_true', help="Show wish failures.")

def generate_module_objects(module):
    try:
        module_members = inspect.getmembers(module)
    except ImportError:
        raise StopIteration
    for object_name, object_ in module_members:
        if inspect.getmodule(object_) is module:
            yield object_name, object_

def valid_name(name, include_res, exclude_res):
    include_name = any(include_re.match(name) for include_re in include_res)
    exclude_name = any(exclude_re.match(name) for exclude_re in exclude_res)
    return include_name and not exclude_name

def index_modules(modules, include_patterns, exclude_patterns, object_blacklist=OBJECT_BLACKLIST):
    exclude_patterns += tuple(name.strip() + '$' for name in object_blacklist)
    include_res = [re.compile(pattern) for pattern in include_patterns]
    exclude_res = [re.compile(pattern) for pattern in exclude_patterns]
    object_index = {}
    for module_name, module in modules.items():
        for object_name, object_ in generate_module_objects(module):
            full_object_name = '{}:{}'.format(module_name, object_name)
            if valid_name(full_object_name, include_res, exclude_res):
                object_index[full_object_name] = object_
    return object_index

def index_objects(stream):
    module_index = {}
    object_index = {}
    for line in stream:
        full_object_name = line.partition('#')[0].strip()
        if full_object_name:
            module_name, _, object_name = full_object_name.partition(':')
            try:
                module = module_index.setdefault(module_name, importlib.import_module(module_name))
                object_index[full_object_name] = getattr(module, object_name)
            except ImportError:
                pass
            except AttributeError:
                pass
    return object_index

def pytest_generate_tests(metafunc):
    if 'wish' not in metafunc.fixturenames:
        return

    wish_modules = metafunc.config.getoption('wish_modules')
    for module_name in wish_modules:
        importlib.import_module(module_name)

    wish_includes = metafunc.config.getoption('wish_includes') or wish_modules
    wish_excludes = metafunc.config.getoption('wish_excludes')

    # NOTE: 'copy' is needed here because index_modules may unexpectedly trigger a module load
    object_index = index_modules(sys.modules.copy(), wish_includes, wish_excludes)

    wish_objects = metafunc.config.getoption('wish_objects')
    if wish_objects is not None:
        object_index.update(index_objects(wish_objects))

    ids, params = list(zip(*sorted(object_index.items()))) or [(), ()]
    metafunc.parametrize('wish', params, ids=ids, scope='module')

    wish_fail = metafunc.config.getoption('wish_fail')
    if not wish_fail:
        metafunc.function = pytest.mark.xfail(metafunc.function)
<commit_msg>Make except broader and let it be not covered. Waiting for a logging fuction.<commit_after># -*- coding: utf-8 -*-

import argparse
import importlib
import inspect
import re
import sys

import pytest

OBJECT_BLACKLIST = (
    # pytest internals
    '_pytest.runner:exit',
    '_pytest.runner:skip',
    '_pytest.skipping:xfail',

    # unconditional exit
    'posix:_exit',
    '_signal:default_int_handler',
    'atexit.register',

    # low level crashes
    'numpy.fft.fftpack_lite:cffti',
    'numpy.fft.fftpack_lite:rffti',
    'appnope._nope:beginActivityWithOptions',
    'ctypes:string_at',
    'ctypes:wstring_at',
    'gc:_dump_rpy_heap',
    'gc:dump_rpy_heap',
    'matplotlib._image:Image',

    # hangs
    'Tkinter:mainloop',
    'astkit.compat.py3:execfile',
    'click.termui:getchar',
    'click.termui:edit',
    'click.termui:hidden_prompt_func',
    'eventlet.hubs:trampoline',
    'getpass:getpass',
    'getpass:unix_getpass',
    'matplotlib.font_manager:FontManager',
    'pty:_copy',
    'pydoc:serve',
    'pyexpat:ErrorString',
    'skimage:_test',
    'skimage:test',
)

def pytest_addoption(parser):
    group = parser.getgroup('wish')
    group.addoption('--wish-modules', default=(), nargs='+',
                    help="Space separated list of module names.")
    group.addoption('--wish-includes', nargs='+',
                    help="Space separated list of regexs matching full object names to include.")
    group.addoption('--wish-excludes', default=(), nargs='+',
                    help="Space separated list of regexs matching full object names to exclude.")
    group.addoption('--wish-objects', type=argparse.FileType('r'),
                    help="File of full object names to include.")
    group.addoption('--wish-fail', action='store_true', help="Show wish failures.")

def generate_module_objects(module):
    try:
        module_members = inspect.getmembers(module)
    except:  # pragma: no cover
        raise StopIteration
    for object_name, object_ in module_members:
        if inspect.getmodule(object_) is module:
            yield object_name, object_

def valid_name(name, include_res, exclude_res):
    include_name = any(include_re.match(name) for include_re in include_res)
    exclude_name = any(exclude_re.match(name) for exclude_re in exclude_res)
    return include_name and not exclude_name

def index_modules(modules, include_patterns, exclude_patterns, object_blacklist=OBJECT_BLACKLIST):
    exclude_patterns += tuple(name.strip() + '$' for name in object_blacklist)
    include_res = [re.compile(pattern) for pattern in include_patterns]
    exclude_res = [re.compile(pattern) for pattern in exclude_patterns]
    object_index = {}
    for module_name, module in modules.items():
        for object_name, object_ in generate_module_objects(module):
            full_object_name = '{}:{}'.format(module_name, object_name)
            if valid_name(full_object_name, include_res, exclude_res):
                object_index[full_object_name] = object_
    return object_index

def index_objects(stream):
    module_index = {}
    object_index = {}
    for line in stream:
        full_object_name = line.partition('#')[0].strip()
        if full_object_name:
            module_name, _, object_name = full_object_name.partition(':')
            try:
                module = module_index.setdefault(module_name, importlib.import_module(module_name))
                object_index[full_object_name] = getattr(module, object_name)
            except ImportError:
                pass
            except AttributeError:
                pass
    return object_index

def pytest_generate_tests(metafunc):
    if 'wish' not in metafunc.fixturenames:
        return

    wish_modules = metafunc.config.getoption('wish_modules')
    for module_name in wish_modules:
        importlib.import_module(module_name)

    wish_includes = metafunc.config.getoption('wish_includes') or wish_modules
    wish_excludes = metafunc.config.getoption('wish_excludes')

    # NOTE: 'copy' is needed here because index_modules may unexpectedly trigger a module load
    object_index = index_modules(sys.modules.copy(), wish_includes, wish_excludes)

    wish_objects = metafunc.config.getoption('wish_objects')
    if wish_objects is not None:
        object_index.update(index_objects(wish_objects))

    ids, params = list(zip(*sorted(object_index.items()))) or [(), ()]
    metafunc.parametrize('wish', params, ids=ids, scope='module')

    wish_fail = metafunc.config.getoption('wish_fail')
    if not wish_fail:
        metafunc.function = pytest.mark.xfail(metafunc.function)
<|endoftext|>


<commit_before>from __future__ import absolute_import, division, print_function

import logging
import platform
from time import time

import dials.algorithms.integration
from dials.algorithms.integration.processor import job
from dials_algorithms_integration_integrator_ext import ReflectionManagerPerImage
from dials.array_family import flex
from dials.model.data import make_image
from dials.model.data import MultiPanelImageVolume
from dials.model.data import ImageVolume
from dials.util import log
from dials.util.mp import multi_node_parallel_map

logger = logging.getLogger(__name__)

class ProcessorImageBase(object):
    """Processor interface class."""

    def __init__(self, manager):
        """
        Initialise the processor.

        The processor requires a manager class implementing the Manager interface.
        This class executes all the workers in separate threads and accumulates the
        results to expose to the user.

        :param manager: The processing manager
        :param params: The phil parameters
        """
        self.manager = manager

    @property
    def executor(self):
        """
        Get the executor

        :return: The executor
        """
        return self.manager.executor

    @executor.setter
    def executor(self, function):
        """
        Set the executor

        :param function: The executor
        """
        self.manager.executor = function

    def process(self):
        """
        Do all the processing tasks.

        :return: The processing results
        """
        start_time = time()
        self.manager.initialize()
        mp_method = self.manager.params.integration.mp.method
        mp_nproc = min(len(self.manager), self.manager.params.integration.mp.nproc)
        mp_njobs = self.manager.params.integration.mp.njobs
        if (
            mp_nproc > 1 and platform.system() == "Windows"
        ):  # platform.system() forks which is bad for MPI, so don't use it unless nproc > 1
            logger.warning(
                "Multiprocessing is not available on windows. Setting nproc = 1\n"
            )
            mp_nproc = 1
        assert mp_nproc > 0, "Invalid number of processors"
        logger.info(self.manager.summary())
        logger.info(" Using %s with %d parallel job(s)\n" % (mp_method, mp_nproc))
        if mp_nproc > 1:

            def process_output(result):
                for message in result[1]:
                    logger.log(message.levelno, message.msg)
                self.manager.accumulate(result[0])
                result[0].reflections = None
                result[0].data = None

            def execute_task(task):
                log.config_simple_cached()
                result = task()
                handlers = logging.getLogger("dials").handlers
                assert len(handlers) == 1, "Invalid number of logging handlers"
                return result, handlers[0].messages()

            multi_node_parallel_map(
                func=execute_task,
                iterable=list(self.manager.tasks()),
                njobs=mp_njobs,
                nproc=mp_nproc,
                callback=process_output,
                cluster_method=mp_method,
                preserve_order=True,
                preserve_exception_message=True,
            )
        else:
            for task in self.manager.tasks():
                self.manager.accumulate(task())
        self.manager.finalize()
        end_time = time()
        self.manager.time.user_time = end_time - start_time
        result = self.manager.result()
        return result, self.manager.time

class Result(object):
    """
    A class representing a processing result.
    """

    def __init__(self, index, reflections):
        """
        Initialise the data.

        :param index: The processing job index
        :param reflections: The processed reflections
        :param data: Other processed data
        """
        self.index = index
        self.reflections = reflections

class Dataset(object):
    def __init__(self, frames, size):
        self.frames = frames
        nframes = frames[1] - frames[0]
        self.data = []
        self.mask = []
        for sz in size:
            self.data.append(flex.double(flex.grid(nframes, sz[0], sz[1])))
            self.mask.append(flex.bool(flex.grid(nframes, sz[0], sz[1])))

    def set_image(self, index, data, mask):
        for d1, d2 in zip(self.data, data):
            h, w = d2.all()
            d2.reshape(flex.grid(1, h, w))
            d1[index : index + 1, :, :] = d2.as_double()
        for m1, m2 in zip(self.mask, mask):
            h, w = m2.all()
            m2.reshape(flex.grid(1, h, w))
            m1[index : index + 1, :, :] = m2

class Task(object):
    """
    A class to perform a null task.
    """

    def __init__(self, index, frames, reflections, experiments, params, executor):
        """
        Initialise the task

        :param index: The index of the processing job
        :param frames: The frames to process
        :param experiments: The list of experiments
        :param reflections: The list of reflections
        :param params The processing parameters
        :param executor: The executor class
        """
        self.index = index
        self.frames = frames
        self.experiments = experiments
        self.reflections = reflections
        self.params = params
        self.executor = executor

    def __call__(self):
        """
        Do the processing.

        :return: The processed data
        """
        # Set the job index
        job.index = self.index

        # Get the start time
        start_time = time()

        # Check all reflections have same imageset and get it
        exp_id = list(set(self.reflections["id"]))
        imageset = self.experiments[exp_id[0]].imageset
        for i in exp_id[1:]:
            assert (
                self.experiments[i].imageset == imageset
            ), "Task can only handle 1 imageset"

        # Get the sub imageset
        frame00, frame01 = self.frames
        try:
            frame10, frame11 = imageset.get_array_range()
        except Exception:
            frame10, frame11 = (0, len(imageset))
        try:
            assert frame00 < frame01
            assert frame10 < frame11
            assert frame00 >= frame10
            assert frame01 <= frame11
            index0 = frame00 - frame10
            index1 = index0 + (frame01 - frame00)
            assert index0 < index1
            assert index0 >= 0
            assert index1 <= len(imageset)
            imageset = imageset[index0:index1]
        except Exception:
            raise RuntimeError("Programmer Error: bad array range")
        try:
            frame0, frame1 = imageset.get_array_range()
        except Exception:
            frame0, frame1 = (0, len(imageset))

        # Initialise the dataset
        image_volume = MultiPanelImageVolume()
        for panel in self.experiments[0].detector:
            image_volume.add(
                ImageVolume(
                    frame0, frame1, panel.get_image_size()[1], panel.get_image_size()[0]
                )
            )

        # Read all the images into a block of data
        read_time = 0.0
        for i in range(len(imageset)):
            st = time()
            image = imageset.get_corrected_data(i)
            mask = imageset.get_mask(i)
            if self.params.integration.lookup.mask is not None:
                assert len(mask) == len(self.params.lookup.mask), (
                    "Mask/Image are incorrect size %d %d"
                    % (len(mask), len(self.params.integration.lookup.mask))
                )
                mask = tuple(
                    m1 & m2 for m1, m2 in zip(self.params.integration.lookup.mask, mask)
                )
            image_volume.set_image(frame0 + i, make_image(image, mask))
            read_time += time() - st
            del image
            del mask

        # Process the data
        st = time()
        data = self.executor.process(image_volume, self.experiments, self.reflections)
        process_time = time() - st

        # Set the result values
        result = Result(self.index, self.reflections)
        result.read_time = read_time
        result.process_time = process_time
        result.total_time = time() - start_time
        result.data = data
        return result

class ManagerImage(object):
    """
    A class to manage processing book-keeping
    """

    def __init__(self, experiments, reflections, params):
        """
        Initialise the manager.

        :param experiments: The list of experiments
        :param reflections: The list of reflections
        :param params: The phil parameters
        """
        # Initialise the callbacks
        self.executor = None

        # Save some data
        self.experiments = experiments
        self.reflections = reflections

        # Save some parameters
        self.params = params

        # Set the finalized flag to False
        self.finalized = False

        # Initialise the timing information
        self.time = dials.algorithms.integration.TimingInfo()

    def initialize(self):
        """
        Initialise the processing
        """
        # Get the start time
        start_time = time()

        # Ensure the reflections contain bounding boxes
        assert "bbox" in self.reflections, "Reflections have no bbox"

        # Split the reflections into partials
        self._split_reflections()

        # Create the reflection manager
        frames = self.experiments[0].scan.get_array_range()
        self.manager = ReflectionManagerPerImage(frames, self.reflections)

        # Parallel reading of HDF5 from the same handle is not allowed. Python
        # multiprocessing is a bit messed up and used fork on linux so need to
        # close and reopen file.
        for exp in self.experiments:
            if exp.imageset.reader().is_single_file_reader():
                exp.imageset.reader().nullify_format_instance()

        # Set the initialization time
        self.time.initialize = time() - start_time

    def task(self, index):
        """
        Get a task.
        """
        return Task(
            index=index,
            frames=self.manager.frames(index),
            reflections=self.manager.split(index),
            experiments=self.experiments,
            params=self.params,
            executor=self.executor,
        )

    def tasks(self):
        """
        Iterate through the tasks.
        """
        for i in range(len(self)):
            yield self.task(i)

    def accumulate(self, result):
        """
        Accumulate the results.
        """
        self.manager.accumulate(result.index, result.reflections)
        if result.data is not None:
            self.executor.accumulate(result.index, result.data)
        self.time.read += result.read_time
        self.time.process += result.process_time
        self.time.total += result.total_time

    def finalize(self):
        """
        Finalize the processing and finish.
        """
        # Get the start time
        start_time = time()

        # Check manager is finished
        assert self.manager.finished(), "Manager is not finished"

        # Update the time and finalized flag
        self.time.finalize = time() - start_time
        self.finalized = True

    def result(self):
        """
        Return the result.

        :return: The result
        """
        assert self.finalized, "Manager is not finalized"
        return self.reflections

    def finished(self):
        """
        Return if all tasks have finished.

        :return: True/False all tasks have finished
        """
        return self.finalized and self.manager.finished()

    def __len__(self):
        """
        Return the number of tasks.

        :return: the number of tasks
        """
        return len(self.manager)

    def summary(self):
        return ""

    def _split_reflections(self):
        """
        Split the reflections into partials or over job boundaries
        """

        # Optionally split the reflection table into partials, otherwise,
        # split over job boundaries
        num_full = len(self.reflections)
        self.reflections.split_partials()
        num_partial = len(self.reflections)
        assert num_partial >= num_full, "Invalid number of partials"
        if num_partial > num_full:
            logger.info(
                " Split %d reflections into %d partial reflections\n"
                % (num_full, num_partial)
            )

class ProcessorImage(ProcessorImageBase):
    """Top level processor for per image processing."""

    def __init__(self, experiments, reflections, params):
        """Initialise the manager and the processor."""

        # Create the processing manager
        manager = ManagerImage(experiments, reflections, params)

        # Initialise the processor
        super(ProcessorImage, self).__init__(manager)

class InitializerRot(object):
    """
    A pre-processing class for oscillation data.
    """

    def __init__(self, experiments, params):
        """
        Initialise the pre-processor.
        """
        self.experiments = experiments
        self.params = params

    def __call__(self, reflections):
        """
        Do some pre-processing.
        """
        # Compute some reflection properties
        reflections.compute_zeta_multi(self.experiments)
        reflections.compute_d(self.experiments)
        reflections.compute_bbox(self.experiments)

        # Filter the reflections by zeta
        mask = flex.abs(reflections["zeta"]) < self.params.filter.min_zeta
        reflections.set_flags(mask, reflections.flags.dont_integrate)

        # Filter the reflections by powder ring
        if self.params.filter.powder_filter is not None:
            mask = self.params.filter.powder_filter(reflections["d"])
            reflections.set_flags(mask, reflections.flags.in_powder_ring)

class FinalizerRot(object):
    """
    A post-processing class for oscillation data.
    """

    def __init__(self, experiments, params):
        """
        Initialise the post processor.
        """
        self.experiments = experiments
        self.params = params

    def __call__(self, reflections):
        """
        Do some post processing.
        """

        # Compute the corrections
        reflections.compute_corrections(self.experiments)
<commit_msg>Remove dead code<commit_after>from __future__ import absolute_import, division, print_function

import logging
import platform
from time import time

import dials.algorithms.integration
from dials.algorithms.integration.processor import job
from dials_algorithms_integration_integrator_ext import ReflectionManagerPerImage
from dials.model.data import make_image
from dials.model.data import MultiPanelImageVolume
from dials.model.data import ImageVolume
from dials.util import log
from dials.util.mp import multi_node_parallel_map

logger = logging.getLogger(__name__)

class ProcessorImageBase(object):
    """Processor interface class."""

    def __init__(self, manager):
        """
        Initialise the processor.

        The processor requires a manager class implementing the Manager interface.
        This class executes all the workers in separate threads and accumulates the
        results to expose to the user.

        :param manager: The processing manager
        :param params: The phil parameters
        """
        self.manager = manager

    @property
    def executor(self):
        """
        Get the executor

        :return: The executor
        """
        return self.manager.executor

    @executor.setter
    def executor(self, function):
        """
        Set the executor

        :param function: The executor
        """
        self.manager.executor = function

    def process(self):
        """
        Do all the processing tasks.

        :return: The processing results
        """
        start_time = time()
        self.manager.initialize()
        mp_method = self.manager.params.integration.mp.method
        mp_nproc = min(len(self.manager), self.manager.params.integration.mp.nproc)
        mp_njobs = self.manager.params.integration.mp.njobs
        if (
            mp_nproc > 1 and platform.system() == "Windows"
        ):  # platform.system() forks which is bad for MPI, so don't use it unless nproc > 1
            logger.warning(
                "Multiprocessing is not available on windows. Setting nproc = 1\n"
            )
            mp_nproc = 1
        assert mp_nproc > 0, "Invalid number of processors"
        logger.info(self.manager.summary())
        logger.info(" Using %s with %d parallel job(s)\n" % (mp_method, mp_nproc))
        if mp_nproc > 1:

            def process_output(result):
                for message in result[1]:
                    logger.log(message.levelno, message.msg)
                self.manager.accumulate(result[0])
                result[0].reflections = None
                result[0].data = None

            def execute_task(task):
                log.config_simple_cached()
                result = task()
                handlers = logging.getLogger("dials").handlers
                assert len(handlers) == 1, "Invalid number of logging handlers"
                return result, handlers[0].messages()

            multi_node_parallel_map(
                func=execute_task,
                iterable=list(self.manager.tasks()),
                njobs=mp_njobs,
                nproc=mp_nproc,
                callback=process_output,
                cluster_method=mp_method,
                preserve_order=True,
                preserve_exception_message=True,
            )
        else:
            for task in self.manager.tasks():
                self.manager.accumulate(task())
        self.manager.finalize()
        end_time = time()
        self.manager.time.user_time = end_time - start_time
        result = self.manager.result()
        return result, self.manager.time

class Result(object):
    """
    A class representing a processing result.
    """

    def __init__(self, index, reflections):
        """
        Initialise the data.

        :param index: The processing job index
        :param reflections: The processed reflections
        :param data: Other processed data
        """
        self.index = index
        self.reflections = reflections

class Task(object):
    """
    A class to perform a null task.
    """

    def __init__(self, index, frames, reflections, experiments, params, executor):
        """
        Initialise the task

        :param index: The index of the processing job
        :param frames: The frames to process
        :param experiments: The list of experiments
        :param reflections: The list of reflections
        :param params The processing parameters
        :param executor: The executor class
        """
        self.index = index
        self.frames = frames
        self.experiments = experiments
        self.reflections = reflections
        self.params = params
        self.executor = executor

    def __call__(self):
        """
        Do the processing.

        :return: The processed data
        """
        # Set the job index
        job.index = self.index

        # Get the start time
        start_time = time()

        # Check all reflections have same imageset and get it
        exp_id = list(set(self.reflections["id"]))
        imageset = self.experiments[exp_id[0]].imageset
        for i in exp_id[1:]:
            assert (
                self.experiments[i].imageset == imageset
            ), "Task can only handle 1 imageset"

        # Get the sub imageset
        frame00, frame01 = self.frames
        try:
            frame10, frame11 = imageset.get_array_range()
        except Exception:
            frame10, frame11 = (0, len(imageset))
        try:
            assert frame00 < frame01
            assert frame10 < frame11
            assert frame00 >= frame10
            assert frame01 <= frame11
            index0 = frame00 - frame10
            index1 = index0 + (frame01 - frame00)
            assert index0 < index1
            assert index0 >= 0
            assert index1 <= len(imageset)
            imageset = imageset[index0:index1]
        except Exception:
            raise RuntimeError("Programmer Error: bad array range")
        try:
            frame0, frame1 = imageset.get_array_range()
        except Exception:
            frame0, frame1 = (0, len(imageset))

        # Initialise the dataset
        image_volume = MultiPanelImageVolume()
        for panel in self.experiments[0].detector:
            image_volume.add(
                ImageVolume(
                    frame0, frame1, panel.get_image_size()[1], panel.get_image_size()[0]
                )
            )

        # Read all the images into a block of data
        read_time = 0.0
        for i in range(len(imageset)):
            st = time()
            image = imageset.get_corrected_data(i)
            mask = imageset.get_mask(i)
            if self.params.integration.lookup.mask is not None:
                assert len(mask) == len(self.params.lookup.mask), (
                    "Mask/Image are incorrect size %d %d"
                    % (len(mask), len(self.params.integration.lookup.mask))
                )
                mask = tuple(
                    m1 & m2 for m1, m2 in zip(self.params.integration.lookup.mask, mask)
                )
            image_volume.set_image(frame0 + i, make_image(image, mask))
            read_time += time() - st
            del image
            del mask

        # Process the data
        st = time()
        data = self.executor.process(image_volume, self.experiments, self.reflections)
        process_time = time() - st

        # Set the result values
        result = Result(self.index, self.reflections)
        result.read_time = read_time
        result.process_time = process_time
        result.total_time = time() - start_time
        result.data = data
        return result

class ManagerImage(object):
    """
    A class to manage processing book-keeping
    """

    def __init__(self, experiments, reflections, params):
        """
        Initialise the manager.

        :param experiments: The list of experiments
        :param reflections: The list of reflections
        :param params: The phil parameters
        """
        # Initialise the callbacks
        self.executor = None

        # Save some data
        self.experiments = experiments
        self.reflections = reflections

        # Save some parameters
        self.params = params

        # Set the finalized flag to False
        self.finalized = False

        # Initialise the timing information
        self.time = dials.algorithms.integration.TimingInfo()

    def initialize(self):
        """
        Initialise the processing
        """
        # Get the start time
        start_time = time()

        # Ensure the reflections contain bounding boxes
        assert "bbox" in self.reflections, "Reflections have no bbox"

        # Split the reflections into partials
        self._split_reflections()

        # Create the reflection manager
        frames = self.experiments[0].scan.get_array_range()
        self.manager = ReflectionManagerPerImage(frames, self.reflections)

        # Parallel reading of HDF5 from the same handle is not allowed. Python
        # multiprocessing is a bit messed up and used fork on linux so need to
        # close and reopen file.
        for exp in self.experiments:
            if exp.imageset.reader().is_single_file_reader():
                exp.imageset.reader().nullify_format_instance()

        # Set the initialization time
        self.time.initialize = time() - start_time

    def task(self, index):
        """
        Get a task.
        """
        return Task(
            index=index,
            frames=self.manager.frames(index),
            reflections=self.manager.split(index),
            experiments=self.experiments,
            params=self.params,
            executor=self.executor,
        )

    def tasks(self):
        """
        Iterate through the tasks.
        """
        for i in range(len(self)):
            yield self.task(i)

    def accumulate(self, result):
        """
        Accumulate the results.
        """
        self.manager.accumulate(result.index, result.reflections)
        if result.data is not None:
            self.executor.accumulate(result.index, result.data)
        self.time.read += result.read_time
        self.time.process += result.process_time
        self.time.total += result.total_time

    def finalize(self):
        """
        Finalize the processing and finish.
        """
        # Get the start time
        start_time = time()

        # Check manager is finished
        assert self.manager.finished(), "Manager is not finished"

        # Update the time and finalized flag
        self.time.finalize = time() - start_time
        self.finalized = True

    def result(self):
        """
        Return the result.

        :return: The result
        """
        assert self.finalized, "Manager is not finalized"
        return self.reflections

    def finished(self):
        """
        Return if all tasks have finished.

        :return: True/False all tasks have finished
        """
        return self.finalized and self.manager.finished()

    def __len__(self):
        """
        Return the number of tasks.

        :return: the number of tasks
        """
        return len(self.manager)

    def summary(self):
        return ""

    def _split_reflections(self):
        """
        Split the reflections into partials or over job boundaries
        """

        # Optionally split the reflection table into partials, otherwise,
        # split over job boundaries
        num_full = len(self.reflections)
        self.reflections.split_partials()
        num_partial = len(self.reflections)
        assert num_partial >= num_full, "Invalid number of partials"
        if num_partial > num_full:
            logger.info(
                " Split %d reflections into %d partial reflections\n"
                % (num_full, num_partial)
            )

class ProcessorImage(ProcessorImageBase):
    """Top level processor for per image processing."""

    def __init__(self, experiments, reflections, params):
        """Initialise the manager and the processor."""

        # Create the processing manager
        manager = ManagerImage(experiments, reflections, params)

        # Initialise the processor
        super(ProcessorImage, self).__init__(manager)
<|endoftext|>