<commit_before>'''
Installation of Python Packages Using pip
=========================================

These states manage system installed python packages. Note that pip must be
installed for these states to be available, so pip states should include a
requisite to a pkg.installed state for the package which provides pip
(``python-pip`` in most cases). Example:

.. code-block:: yaml

    python-pip:
      pkg.installed

    virtualenvwrapper:
      pip.installed:
        - require:
          - pkg: python-pip
'''

# Import python libs
import logging

# Import salt libs
import salt.utils
from salt.exceptions import CommandExecutionError, CommandNotFoundError

# Import 3rd-party libs
try:
    import pip
    import pip.req
    HAS_PIP = True
except ImportError:
    HAS_PIP = False

logger = logging.getLogger(__name__)

def __virtual__():
    '''
    Only load if the pip module is available in __salt__
    '''
    if HAS_PIP and 'pip.list' in __salt__:
        return 'pip'
    return False

def _find_key(prefix, pip_list):
    '''
    Does a case-insensitive match in the pip_list for the desired package.
    '''
    try:
        match = next(
            iter(x for x in pip_list if x.lower() == prefix.lower())
        )
    except StopIteration:
        return None
    else:
        return match

def _fulfills_version_spec(version, version_spec):
    '''
    Check version number against version specification info and return a
    boolean value based on whether or not the version number meets the
    specified version.
    '''
    for oper, spec in version_spec:
        if oper is None:
            continue
        if not salt.utils.compare_versions(ver1=version, oper=oper, ver2=spec):
            return False
    return True

def installed(name,
              pip_bin=None,
              requirements=None,
              env=None,
              bin_env=None,
              use_wheel=False,
              log=None,
              proxy=None,
              timeout=None,
              repo=None,
              editable=None,
              find_links=None,
              index_url=None,
              extra_index_url=None,
              no_index=False,
              mirrors=None,
              build=None,
              target=None,
              download=None,
              download_cache=None,
              source=None,
              upgrade=False,
              force_reinstall=False,
              ignore_installed=False,
              exists_action=None,
              no_deps=False,
              no_install=False,
              no_download=False,
              install_options=None,
              user=None,
              runas=None,
              no_chown=False,
              cwd=None,
              pre_releases=False,
              __env__='base'):
    '''
    Make sure the package is installed

    name
        The name of the python package to install
    user
        The user under which to run pip
    pip_bin : None
        Deprecated, use bin_env
    use_wheel : False
        Prefer wheel archives (requires pip>=1.4)
    env : None
        Deprecated, use bin_env
    bin_env : None
        the pip executable or virtualenv to use

    .. versionchanged:: 0.17.0
        ``use_wheel`` option added.
    '''
    if pip_bin and not bin_env:
        bin_env = pip_bin
    elif env and not bin_env:
        bin_env = env

    ret = {'name': name, 'result': None, 'comment': '', 'changes': {}}

    if use_wheel:
        min_version = '1.4'
        cur_version = __salt__['pip.version'](bin_env)
        if not salt.utils.compare_versions(ver1=cur_version, oper='>=',
                                           ver2=min_version):
            ret['result'] = False
            ret['comment'] = ('The \'use_wheel\' option is only supported in '
                              'pip {0} and newer. The version of pip detected '
                              'was {1}.').format(min_version, cur_version)
            return ret

    if repo is not None:
        msg = ('The \'repo\' argument to pip.installed is deprecated and will '
               'be removed in 0.18.0. Please use \'name\' instead. The '
               'current value for name, {0!r} will be replaced by the value '
               'of repo, {1!r}'.format(name, repo))
        salt.utils.warn_until((0, 18), msg)
        ret.setdefault('warnings', []).append(msg)
        name = repo

    try:
        try:
            # With pip < 1.2, the __version__ attribute does not exist and
            # vcs+URL urls are not properly parsed.
            # The next line is meant to trigger an AttributeError and handle
            # lower pip versions
            logger.debug('Installed pip version: {0}'.format(pip.__version__))
            install_req = pip.req.InstallRequirement.from_line(name)
        except AttributeError:
            logger.debug('Installed pip version is lower than 1.2')
            supported_vcs = ('git', 'svn', 'hg', 'bzr')
            if name.startswith(supported_vcs):
                for vcs in supported_vcs:
                    if name.startswith(vcs):
                        install_req = pip.req.InstallRequirement.from_line(
                            name.split('{0}+'.format(vcs))[-1]
                        )
                        break
            else:
                install_req = pip.req.InstallRequirement.from_line(name)
    except ValueError as exc:
        ret['result'] = False
        if '=' in name and '==' not in name:
            ret['comment'] = (
                'Invalid version specification in package {0}. \'=\' is not '
                'supported, use \'==\' instead.'.format(name)
            )
            return ret
        ret['comment'] = (
            'pip raised an exception while parsing {0!r}: {1}'.format(
                name, exc
            )
        )
        return ret

    if install_req.req is None:
        # This is most likely an url and there's no way to know what will be
        # installed before actually installing it.
        prefix = ''
        version_spec = []
    else:
        prefix = install_req.req.project_name
        version_spec = install_req.req.specs

    if runas is not None:
        # The user is using a deprecated argument, warn!
        msg = (
            'The \'runas\' argument to pip.installed is deprecated, and will '
            'be removed in 0.18.0. Please use \'user\' instead.'
        )
        salt.utils.warn_until((0, 18), msg)
        ret.setdefault('warnings', []).append(msg)

        # "There can only be one"
        if user:
            raise CommandExecutionError(
                'The \'runas\' and \'user\' arguments are mutually exclusive. '
                'Please use \'user\' as \'runas\' is being deprecated.'
            )
        # Support deprecated 'runas' arg
        else:
            user = runas

    # Replace commas (used for version ranges) with semicolons (which are not
    # supported) in name so it does not treat them as multiple packages.  Comma
    # will be re-added in pip.install call.
    name = name.replace(',', ';')

    # If a requirements file is specified, only install the contents of the
    # requirements file. Similarly, using the --editable flag with pip should
    # also ignore the "name" parameter.
    if requirements or editable:
        name = ''
        comments = []
        if __opts__['test']:
            ret['result'] = None
            if requirements:
                # TODO: Check requirements file against currently-installed
                # packages to provide more accurate state output.
                comments.append('Requirements file {0!r} will be '
                                'processed.'.format(requirements))
            if editable:
                comments.append(
                    'Package will be installed in editable mode (i.e. '
                    'setuptools "develop mode") from {0}.'.format(editable)
                )
            ret['comment'] = ' '.join(comments)
            return ret
    else:
        try:
            pip_list = __salt__['pip.list'](prefix, bin_env=bin_env,
                                            user=user, cwd=cwd)
            prefix_realname = _find_key(prefix, pip_list)
        except (CommandNotFoundError, CommandExecutionError) as err:
            ret['result'] = False
            ret['comment'] = 'Error installing {0!r}: {1}'.format(name, err)
            return ret

        if ignore_installed is False and prefix_realname is not None:
            if force_reinstall is False and not upgrade:
                # Check desired version (if any) against currently-installed
                if (
                        any(version_spec) and
                        _fulfills_version_spec(pip_list[prefix_realname],
                                            version_spec)
                        ) or (not any(version_spec)):
                    ret['result'] = True
                    ret['comment'] = ('Python package {0} already '
                                    'installed'.format(name))
                    return ret

        if __opts__['test']:
            ret['result'] = None
            ret['comment'] = \
                'Python package {0} is set to be installed'.format(name)
            return ret

    pip_install_call = __salt__['pip.install'](
        pkgs='{0}'.format(name) if name else '',
        requirements=requirements,
        bin_env=bin_env,
        use_wheel=use_wheel,
        log=log,
        proxy=proxy,
        timeout=timeout,
        editable=editable,
        find_links=find_links,
        index_url=index_url,
        extra_index_url=extra_index_url,
        no_index=no_index,
        mirrors=mirrors,
        build=build,
        target=target,
        download=download,
        download_cache=download_cache,
        source=source,
        upgrade=upgrade,
        force_reinstall=force_reinstall,
        ignore_installed=ignore_installed,
        exists_action=exists_action,
        no_deps=no_deps,
        no_install=no_install,
        no_download=no_download,
        install_options=install_options,
        user=user,
        no_chown=no_chown,
        cwd=cwd,
        pre_releases=pre_releases,
        __env__=__env__
    )

    if pip_install_call and (pip_install_call['retcode'] == 0):
        ret['result'] = True

        if requirements or editable:
            comments = []
            if requirements:
                comments.append('Successfully processed requirements file '
                                '{0}.'.format(requirements))
                ret['changes']['requirements'] = True
            if editable:
                comments.append('Package successfully installed from VCS '
                                'checkout {0}.'.format(editable))
                ret['changes']['editable'] = True
            ret['comment'] = ' '.join(comments)
        else:
            if not prefix:
                pkg_list = []
            else:
                pkg_list = __salt__['pip.list'](
                    prefix, bin_env, user=user, cwd=cwd
                )
            if not pkg_list:
                ret['comment'] = (
                    'There was no error installing package \'{0}\' although '
                    'it does not show when calling '
                    '\'pip.freeze\'.'.format(name)
                )
                ret['changes']['{0}==???'.format(name)] = 'Installed'
                return ret

            version = list(pkg_list.values())[0]
            pkg_name = next(iter(pkg_list))
            ret['changes']['{0}=={1}'.format(pkg_name, version)] = 'Installed'
            ret['comment'] = 'Package was successfully installed'
    elif pip_install_call:
        ret['result'] = False
        error = 'Error: {0} {1}'.format(pip_install_call['stdout'],
                                        pip_install_call['stderr'])

        if requirements or editable:
            comments = []
            if requirements:
                comments.append('Unable to process requirements file '
                                '{0}.'.format(requirements))
            if editable:
                comments.append('Unable to install from VCS checkout'
                                '{0}.'.format(editable))
            comments.append(error)
            ret['comment'] = ' '.join(comments)
        else:
            ret['comment'] = ('Failed to install package {0}. '
                              '{1}'.format(name, error))
    else:
        ret['result'] = False
        ret['comment'] = 'Could not install package'

    return ret

def removed(name,
            requirements=None,
            bin_env=None,
            log=None,
            proxy=None,
            timeout=None,
            user=None,
            runas=None,
            cwd=None,
            __env__='base'):
    '''
    Make sure that a package is not installed.

    name
        The name of the package to uninstall
    user
        The user under which to run pip
    bin_env : None
        the pip executable or virtualenenv to use
    '''
    ret = {'name': name, 'result': None, 'comment': '', 'changes': {}}

    if runas is not None:
        # The user is using a deprecated argument, warn!
        msg = (
            'The \'runas\' argument to pip.installed is deprecated, and will '
            'be removed in 0.18.0. Please use \'user\' instead.'
        )
        salt.utils.warn_until((0, 18), msg)
        ret.setdefault('warnings', []).append(msg)

    # "There can only be one"
    if runas is not None and user:
        raise CommandExecutionError(
            'The \'runas\' and \'user\' arguments are mutually exclusive. '
            'Please use \'user\' as \'runas\' is being deprecated.'
        )
    # Support deprecated 'runas' arg
    elif runas is not None and not user:
        user = runas

    try:
        pip_list = __salt__['pip.list'](bin_env=bin_env, user=user, cwd=cwd)
    except (CommandExecutionError, CommandNotFoundError) as err:
        ret['result'] = False
        ret['comment'] = 'Error uninstalling \'{0}\': {1}'.format(name, err)
        return ret

    if name not in pip_list:
        ret['result'] = True
        ret['comment'] = 'Package is not installed.'
        return ret

    if __opts__['test']:
        ret['result'] = None
        ret['comment'] = 'Package {0} is set to be removed'.format(name)
        return ret

    if __salt__['pip.uninstall'](pkgs=name,
                                 requirements=requirements,
                                 bin_env=bin_env,
                                 log=log,
                                 proxy=proxy,
                                 timeout=timeout,
                                 user=user,
                                 cwd=cwd,
                                 __env__='base'):
        ret['result'] = True
        ret['changes'][name] = 'Removed'
        ret['comment'] = 'Package was successfully removed.'
    else:
        ret['result'] = False
        ret['comment'] = 'Could not remove package.'
    return ret
<commit_msg>If name is empty it should not be parsed for name and version.<commit_after>'''
Installation of Python Packages Using pip
=========================================

These states manage system installed python packages. Note that pip must be
installed for these states to be available, so pip states should include a
requisite to a pkg.installed state for the package which provides pip
(``python-pip`` in most cases). Example:

.. code-block:: yaml

    python-pip:
      pkg.installed

    virtualenvwrapper:
      pip.installed:
        - require:
          - pkg: python-pip
'''

# Import python libs
import logging

# Import salt libs
import salt.utils
from salt.exceptions import CommandExecutionError, CommandNotFoundError

# Import 3rd-party libs
try:
    import pip
    import pip.req
    HAS_PIP = True
except ImportError:
    HAS_PIP = False

logger = logging.getLogger(__name__)

def __virtual__():
    '''
    Only load if the pip module is available in __salt__
    '''
    if HAS_PIP and 'pip.list' in __salt__:
        return 'pip'
    return False

def _find_key(prefix, pip_list):
    '''
    Does a case-insensitive match in the pip_list for the desired package.
    '''
    try:
        match = next(
            iter(x for x in pip_list if x.lower() == prefix.lower())
        )
    except StopIteration:
        return None
    else:
        return match

def _fulfills_version_spec(version, version_spec):
    '''
    Check version number against version specification info and return a
    boolean value based on whether or not the version number meets the
    specified version.
    '''
    for oper, spec in version_spec:
        if oper is None:
            continue
        if not salt.utils.compare_versions(ver1=version, oper=oper, ver2=spec):
            return False
    return True

def installed(name,
              pip_bin=None,
              requirements=None,
              env=None,
              bin_env=None,
              use_wheel=False,
              log=None,
              proxy=None,
              timeout=None,
              repo=None,
              editable=None,
              find_links=None,
              index_url=None,
              extra_index_url=None,
              no_index=False,
              mirrors=None,
              build=None,
              target=None,
              download=None,
              download_cache=None,
              source=None,
              upgrade=False,
              force_reinstall=False,
              ignore_installed=False,
              exists_action=None,
              no_deps=False,
              no_install=False,
              no_download=False,
              install_options=None,
              user=None,
              runas=None,
              no_chown=False,
              cwd=None,
              pre_releases=False,
              __env__='base'):
    '''
    Make sure the package is installed

    name
        The name of the python package to install
    user
        The user under which to run pip
    pip_bin : None
        Deprecated, use bin_env
    use_wheel : False
        Prefer wheel archives (requires pip>=1.4)
    env : None
        Deprecated, use bin_env
    bin_env : None
        the pip executable or virtualenv to use

    .. versionchanged:: 0.17.0
        ``use_wheel`` option added.
    '''
    if pip_bin and not bin_env:
        bin_env = pip_bin
    elif env and not bin_env:
        bin_env = env

    ret = {'name': name, 'result': None, 'comment': '', 'changes': {}}

    if use_wheel:
        min_version = '1.4'
        cur_version = __salt__['pip.version'](bin_env)
        if not salt.utils.compare_versions(ver1=cur_version, oper='>=',
                                           ver2=min_version):
            ret['result'] = False
            ret['comment'] = ('The \'use_wheel\' option is only supported in '
                              'pip {0} and newer. The version of pip detected '
                              'was {1}.').format(min_version, cur_version)
            return ret

    if repo is not None:
        msg = ('The \'repo\' argument to pip.installed is deprecated and will '
               'be removed in 0.18.0. Please use \'name\' instead. The '
               'current value for name, {0!r} will be replaced by the value '
               'of repo, {1!r}'.format(name, repo))
        salt.utils.warn_until((0, 18), msg)
        ret.setdefault('warnings', []).append(msg)
        name = repo

    if name:
        try:
            try:
                # With pip < 1.2, the __version__ attribute does not exist and
                # vcs+URL urls are not properly parsed.
                # The next line is meant to trigger an AttributeError and handle
                # lower pip versions
                logger.debug('Installed pip version: {0}'.format(pip.__version__))
                install_req = pip.req.InstallRequirement.from_line(name)
            except AttributeError:
                logger.debug('Installed pip version is lower than 1.2')
                supported_vcs = ('git', 'svn', 'hg', 'bzr')
                if name.startswith(supported_vcs):
                    for vcs in supported_vcs:
                        if name.startswith(vcs):
                            install_req = pip.req.InstallRequirement.from_line(
                                name.split('{0}+'.format(vcs))[-1]
                            )
                            break
                else:
                    install_req = pip.req.InstallRequirement.from_line(name)
        except ValueError as exc:
            ret['result'] = False
            if '=' in name and '==' not in name:
                ret['comment'] = (
                    'Invalid version specification in package {0}. \'=\' is not '
                    'supported, use \'==\' instead.'.format(name)
                )
                return ret
            ret['comment'] = (
                'pip raised an exception while parsing {0!r}: {1}'.format(
                    name, exc
                )
            )
            return ret

        if install_req.req is None:
            # This is most likely an url and there's no way to know what will be
            # installed before actually installing it.
            prefix = ''
            version_spec = []
        else:
            prefix = install_req.req.project_name
            version_spec = install_req.req.specs
    else:
        prefix = ''
        version_spec = []

    if runas is not None:
        # The user is using a deprecated argument, warn!
        msg = (
            'The \'runas\' argument to pip.installed is deprecated, and will '
            'be removed in 0.18.0. Please use \'user\' instead.'
        )
        salt.utils.warn_until((0, 18), msg)
        ret.setdefault('warnings', []).append(msg)

        # "There can only be one"
        if user:
            raise CommandExecutionError(
                'The \'runas\' and \'user\' arguments are mutually exclusive. '
                'Please use \'user\' as \'runas\' is being deprecated.'
            )
        # Support deprecated 'runas' arg
        else:
            user = runas

    # Replace commas (used for version ranges) with semicolons (which are not
    # supported) in name so it does not treat them as multiple packages.  Comma
    # will be re-added in pip.install call.
    name = name.replace(',', ';')

    # If a requirements file is specified, only install the contents of the
    # requirements file. Similarly, using the --editable flag with pip should
    # also ignore the "name" parameter.
    if requirements or editable:
        name = ''
        comments = []
        if __opts__['test']:
            ret['result'] = None
            if requirements:
                # TODO: Check requirements file against currently-installed
                # packages to provide more accurate state output.
                comments.append('Requirements file {0!r} will be '
                                'processed.'.format(requirements))
            if editable:
                comments.append(
                    'Package will be installed in editable mode (i.e. '
                    'setuptools "develop mode") from {0}.'.format(editable)
                )
            ret['comment'] = ' '.join(comments)
            return ret
    else:
        try:
            pip_list = __salt__['pip.list'](prefix, bin_env=bin_env,
                                            user=user, cwd=cwd)
            prefix_realname = _find_key(prefix, pip_list)
        except (CommandNotFoundError, CommandExecutionError) as err:
            ret['result'] = False
            ret['comment'] = 'Error installing {0!r}: {1}'.format(name, err)
            return ret

        if ignore_installed is False and prefix_realname is not None:
            if force_reinstall is False and not upgrade:
                # Check desired version (if any) against currently-installed
                if (
                        any(version_spec) and
                        _fulfills_version_spec(pip_list[prefix_realname],
                                            version_spec)
                        ) or (not any(version_spec)):
                    ret['result'] = True
                    ret['comment'] = ('Python package {0} already '
                                    'installed'.format(name))
                    return ret

        if __opts__['test']:
            ret['result'] = None
            ret['comment'] = \
                'Python package {0} is set to be installed'.format(name)
            return ret

    pip_install_call = __salt__['pip.install'](
        pkgs='{0}'.format(name) if name else '',
        requirements=requirements,
        bin_env=bin_env,
        use_wheel=use_wheel,
        log=log,
        proxy=proxy,
        timeout=timeout,
        editable=editable,
        find_links=find_links,
        index_url=index_url,
        extra_index_url=extra_index_url,
        no_index=no_index,
        mirrors=mirrors,
        build=build,
        target=target,
        download=download,
        download_cache=download_cache,
        source=source,
        upgrade=upgrade,
        force_reinstall=force_reinstall,
        ignore_installed=ignore_installed,
        exists_action=exists_action,
        no_deps=no_deps,
        no_install=no_install,
        no_download=no_download,
        install_options=install_options,
        user=user,
        no_chown=no_chown,
        cwd=cwd,
        pre_releases=pre_releases,
        __env__=__env__
    )

    if pip_install_call and (pip_install_call['retcode'] == 0):
        ret['result'] = True

        if requirements or editable:
            comments = []
            if requirements:
                comments.append('Successfully processed requirements file '
                                '{0}.'.format(requirements))
                ret['changes']['requirements'] = True
            if editable:
                comments.append('Package successfully installed from VCS '
                                'checkout {0}.'.format(editable))
                ret['changes']['editable'] = True
            ret['comment'] = ' '.join(comments)
        else:
            if not prefix:
                pkg_list = []
            else:
                pkg_list = __salt__['pip.list'](
                    prefix, bin_env, user=user, cwd=cwd
                )
            if not pkg_list:
                ret['comment'] = (
                    'There was no error installing package \'{0}\' although '
                    'it does not show when calling '
                    '\'pip.freeze\'.'.format(name)
                )
                ret['changes']['{0}==???'.format(name)] = 'Installed'
                return ret

            version = list(pkg_list.values())[0]
            pkg_name = next(iter(pkg_list))
            ret['changes']['{0}=={1}'.format(pkg_name, version)] = 'Installed'
            ret['comment'] = 'Package was successfully installed'
    elif pip_install_call:
        ret['result'] = False
        error = 'Error: {0} {1}'.format(pip_install_call['stdout'],
                                        pip_install_call['stderr'])

        if requirements or editable:
            comments = []
            if requirements:
                comments.append('Unable to process requirements file '
                                '{0}.'.format(requirements))
            if editable:
                comments.append('Unable to install from VCS checkout'
                                '{0}.'.format(editable))
            comments.append(error)
            ret['comment'] = ' '.join(comments)
        else:
            ret['comment'] = ('Failed to install package {0}. '
                              '{1}'.format(name, error))
    else:
        ret['result'] = False
        ret['comment'] = 'Could not install package'

    return ret

def removed(name,
            requirements=None,
            bin_env=None,
            log=None,
            proxy=None,
            timeout=None,
            user=None,
            runas=None,
            cwd=None,
            __env__='base'):
    '''
    Make sure that a package is not installed.

    name
        The name of the package to uninstall
    user
        The user under which to run pip
    bin_env : None
        the pip executable or virtualenenv to use
    '''
    ret = {'name': name, 'result': None, 'comment': '', 'changes': {}}

    if runas is not None:
        # The user is using a deprecated argument, warn!
        msg = (
            'The \'runas\' argument to pip.installed is deprecated, and will '
            'be removed in 0.18.0. Please use \'user\' instead.'
        )
        salt.utils.warn_until((0, 18), msg)
        ret.setdefault('warnings', []).append(msg)

    # "There can only be one"
    if runas is not None and user:
        raise CommandExecutionError(
            'The \'runas\' and \'user\' arguments are mutually exclusive. '
            'Please use \'user\' as \'runas\' is being deprecated.'
        )
    # Support deprecated 'runas' arg
    elif runas is not None and not user:
        user = runas

    try:
        pip_list = __salt__['pip.list'](bin_env=bin_env, user=user, cwd=cwd)
    except (CommandExecutionError, CommandNotFoundError) as err:
        ret['result'] = False
        ret['comment'] = 'Error uninstalling \'{0}\': {1}'.format(name, err)
        return ret

    if name not in pip_list:
        ret['result'] = True
        ret['comment'] = 'Package is not installed.'
        return ret

    if __opts__['test']:
        ret['result'] = None
        ret['comment'] = 'Package {0} is set to be removed'.format(name)
        return ret

    if __salt__['pip.uninstall'](pkgs=name,
                                 requirements=requirements,
                                 bin_env=bin_env,
                                 log=log,
                                 proxy=proxy,
                                 timeout=timeout,
                                 user=user,
                                 cwd=cwd,
                                 __env__='base'):
        ret['result'] = True
        ret['changes'][name] = 'Removed'
        ret['comment'] = 'Package was successfully removed.'
    else:
        ret['result'] = False
        ret['comment'] = 'Could not remove package.'
    return ret
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
    salt.utils.filebuffer
    ~~~~~~~~~~~~~~~~~~~~~

    This utility allows parsing a file in chunks.

    :codeauthor: :email:`Pedro Algarvio (pedro@algarvio.me)`
    :copyright: © 2012 by the SaltStack Team, see AUTHORS for more details.
    :license: Apache 2.0, see LICENSE for more details.
'''

# Import salt libs
import salt.utils
from salt.exceptions import SaltException

class InvalidFileMode(SaltException):
    '''
    An invalid file mode was used to open the file passed to the buffer
    '''

class BufferedReader(object):
    '''
    This object allows iterating through the contents of a file keeping
    X configurable bytes in memory which can be used to, for example,
    do regex search/matching on more than a single line.

    So, **an imaginary, non accurate**, example could be:

        1 - Initiate the BufferedReader filing it to max_in_men:
            br = [1, 2, 3]

        2 - next chunk(pop chunk_size from the left, append chunk_size to the
            right):
                br = [2, 3, 4]

    :type  path: str
    :param path: The file path to be read

    :type  max_in_mem: int
    :param max_in_mem: The maximum bytes kept in memory while iterating through
                       the file. Default 256KB.

    :type  chunk_size: int
    :param chunk_size: The size of each consequent read chunk. Default 32KB.

    :type  mode: str
    :param mode: The mode the file should be opened. **Only read modes**.

    '''
    def __init__(self, path, max_in_mem=256 * 1024, chunk_size=32 * 1024,
                 mode='r'):
        if 'a' in mode or 'w' in mode:
            raise InvalidFileMode("Cannot open file in write or append mode")
        self.__path = path
        self.__file = salt.utils.fopen(self.__path, mode)
        self.__max_in_mem = max_in_mem
        self.__chunk_size = chunk_size
        self.__buffered = None

    # Public attributes
    @property
    def buffered(self):
        return self.__buffered

    # Support iteration
    def __iter__(self):
        return self

    def next(self):
        '''
        Return the next iteration by pop'ing `chunk_size` from the left and
        appending `chunk_size` to the right if there's info on the file left
        to be read.
        '''
        if self.__buffered is None:
            multiplier = self.__max_in_mem / self.__chunk_size
            self.__buffered = ""
        else:
            multiplier = 1
            self.__buffered = self.__buffered[self.__chunk_size:]

        data = self.__file.read(self.__chunk_size * multiplier)

        if not data:
            self.__file.close()
            raise StopIteration

        self.__buffered += data
        return self.__buffered

    # Support with statements
    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        pass

def _main():
    def timeit_string(fpath, max_size, chunk_size):

        breader = BufferedReader(fpath, max_size, chunk_size)
        for chunk in breader:
            chunk
        return

    def sizeof_fmt(num):
        for unit in ['bytes', 'KB', 'MB', 'GB']:
            if num < 1024.0:
                return '{0:3.1f}{1}'.format(num, unit)
            num /= 1024.0
        return '{0:3.1f}{1}'.format(num, 'TB')

    import os, timeit
    fpath = os.path.normpath(os.path.join(
        os.path.dirname(__file__),
        "../../doc/topics/tutorials/starting_states.rst"
    ))

    tpath = "/tmp/starting_states.rst"

    for fmultiplier in (1, 10, 50, 100, 800, 3200):
        ffile = salt.utils.fopen(tpath, "w")
        while fmultiplier > 0:
            ffile.write(salt.utils.fopen(fpath).read())
            fmultiplier -= 1

        ffile.close()

        tnumber = 1000

        print('Running tests against a file with the size of {0}'.format(
            sizeof_fmt(os.stat(tpath).st_size))
        )

        for multiplier in [4, 8, 16, 32, 64, 128, 256]:
            chunk_size = multiplier * 1024
            max_size = chunk_size * 5
            timer = timeit.Timer(
                "timeit_string('{0}', {1:d}, {2:d})".format(
                    tpath, max_size, chunk_size
                ), "from __main__ import timeit_string"
            )
            print("timeit_string ({0: >7} chunks; max: {1: >7}):".format(
                sizeof_fmt(chunk_size), sizeof_fmt(max_size))),
            print(u"{0: >6} \u00B5sec/pass".format(u"{0:0.2f}".format(
                tnumber * timer.timeit(number=tnumber) / tnumber
            )))

        print

if __name__ == '__main__':
    _main()
<commit_msg>Minor doc indentation fix.<commit_after># -*- coding: utf-8 -*-
'''
    salt.utils.filebuffer
    ~~~~~~~~~~~~~~~~~~~~~

    This utility allows parsing a file in chunks.

    :codeauthor: :email:`Pedro Algarvio (pedro@algarvio.me)`
    :copyright: © 2012 by the SaltStack Team, see AUTHORS for more details.
    :license: Apache 2.0, see LICENSE for more details.
'''

# Import salt libs
import salt.utils
from salt.exceptions import SaltException

class InvalidFileMode(SaltException):
    '''
    An invalid file mode was used to open the file passed to the buffer
    '''

class BufferedReader(object):
    '''
    This object allows iterating through the contents of a file keeping
    X configurable bytes in memory which can be used to, for example,
    do regex search/matching on more than a single line.

    So, **an imaginary, non accurate**, example could be:

        1 - Initiate the BufferedReader filing it to max_in_men:
            br = [1, 2, 3]

        2 - next chunk(pop chunk_size from the left, append chunk_size to the
        right):
            br = [2, 3, 4]

    :type  path: str
    :param path: The file path to be read

    :type  max_in_mem: int
    :param max_in_mem: The maximum bytes kept in memory while iterating through
                       the file. Default 256KB.

    :type  chunk_size: int
    :param chunk_size: The size of each consequent read chunk. Default 32KB.

    :type  mode: str
    :param mode: The mode the file should be opened. **Only read modes**.

    '''
    def __init__(self, path, max_in_mem=256 * 1024, chunk_size=32 * 1024,
                 mode='r'):
        if 'a' in mode or 'w' in mode:
            raise InvalidFileMode("Cannot open file in write or append mode")
        self.__path = path
        self.__file = salt.utils.fopen(self.__path, mode)
        self.__max_in_mem = max_in_mem
        self.__chunk_size = chunk_size
        self.__buffered = None

    # Public attributes
    @property
    def buffered(self):
        return self.__buffered

    # Support iteration
    def __iter__(self):
        return self

    def next(self):
        '''
        Return the next iteration by pop'ing `chunk_size` from the left and
        appending `chunk_size` to the right if there's info on the file left
        to be read.
        '''
        if self.__buffered is None:
            multiplier = self.__max_in_mem / self.__chunk_size
            self.__buffered = ""
        else:
            multiplier = 1
            self.__buffered = self.__buffered[self.__chunk_size:]

        data = self.__file.read(self.__chunk_size * multiplier)

        if not data:
            self.__file.close()
            raise StopIteration

        self.__buffered += data
        return self.__buffered

    # Support with statements
    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        pass

def _main():
    def timeit_string(fpath, max_size, chunk_size):

        breader = BufferedReader(fpath, max_size, chunk_size)
        for chunk in breader:
            chunk
        return

    def sizeof_fmt(num):
        for unit in ['bytes', 'KB', 'MB', 'GB']:
            if num < 1024.0:
                return '{0:3.1f}{1}'.format(num, unit)
            num /= 1024.0
        return '{0:3.1f}{1}'.format(num, 'TB')

    import os, timeit
    fpath = os.path.normpath(os.path.join(
        os.path.dirname(__file__),
        "../../doc/topics/tutorials/starting_states.rst"
    ))

    tpath = "/tmp/starting_states.rst"

    for fmultiplier in (1, 10, 50, 100, 800, 3200):
        ffile = salt.utils.fopen(tpath, "w")
        while fmultiplier > 0:
            ffile.write(salt.utils.fopen(fpath).read())
            fmultiplier -= 1

        ffile.close()

        tnumber = 1000

        print('Running tests against a file with the size of {0}'.format(
            sizeof_fmt(os.stat(tpath).st_size))
        )

        for multiplier in [4, 8, 16, 32, 64, 128, 256]:
            chunk_size = multiplier * 1024
            max_size = chunk_size * 5
            timer = timeit.Timer(
                "timeit_string('{0}', {1:d}, {2:d})".format(
                    tpath, max_size, chunk_size
                ), "from __main__ import timeit_string"
            )
            print("timeit_string ({0: >7} chunks; max: {1: >7}):".format(
                sizeof_fmt(chunk_size), sizeof_fmt(max_size))),
            print(u"{0: >6} \u00B5sec/pass".format(u"{0:0.2f}".format(
                tnumber * timer.timeit(number=tnumber) / tnumber
            )))

        print

if __name__ == '__main__':
    _main()
<|endoftext|>


<commit_before>import queue, pickle, os, time, hashlib, re

# wig - Cache
#
# wig uses a cache to store the requests and responses made during a scan.
# This helps limit the amount of requests that it makes, as a request for
# resource is only made once.
# To further limit the amount of requests, wig saves a copy of the cache
# and will reuse it for scans run within 24 hours.
#
# NOTE: The pickle module is used to save and load the cache. This has the
#       side effect, that the extra variables which are added to a an
#       instance of Requests are not kept. Currently 2 variables are added:
#       md5 and md5_404. These values have to be recalculated when loading
#       a pickled cache. Ideally, a custom object should be created for
#       the modified requests, so this is on the TODO.

class Cache(queue.Queue):
	def _init(self, maxsize):
		self.queue = dict()
		self.host = None
		self.cache_dir = './cache/'
		self.now = str(time.time()).split('.')[0]

		# only load cache data that is new than this
		# (currently this is set for 24 hours)
		self.cache_ttl = 60*60*24

		# check if there are caches that are older than ttl
		self._remove_old_caches()

	def __getitem__(self, path):
		return self.queue[path]

	def __setitem__(self, path, response):
		with self.mutex:
			self.queue[path] = response

	def __contains__(self, url):
		with self.mutex:
			return url in self.queue

	def _remove_old_caches(self):
		# remove caches that are too old

		# bail if the directory does not exist
		if not os.path.exists(self.cache_dir): return None

		# iterate over the cache files
		for cache_file in os.listdir(self.cache_dir):

			# skip the file if it's not a cache file
			if not cache_file.endswith('.cache'):
				continue

			# check if the cache is for the host
			hostname, time_ext = cache_file.split('_-_')
			save_time,_ = time_ext.split('.')

			# check the age of the cache, and remove it if older than
			# ttl
			age = int(self.now) - int(save_time)
			if age > self.cache_ttl:
				file_name = os.path.join(self.cache_dir, cache_file)
				os.remove(file_name)

	def _get_name_for_cache_file(self):
		# check if there already is an older version of the cache
		# if there is, return the name of this file, such that,
		# it will be overwritten
		if not os.path.exists(self.cache_dir):
			os.makedirs(self.cache_dir)

		for cache_file in os.listdir(self.cache_dir):
			# skip the file if it's not a cache file
			if not cache_file.endswith('.cache'):
				continue

			# check if the cache is for the host
			hostname,_ = cache_file.split('_-_')
			if hostname == self.cache_name.split('_-_')[0]:
				return os.path.join(self.cache_dir, cache_file)

		# if there aren't any previous cache files, generate a
		# new name for the cache
		return os.path.join(self.cache_dir, self.cache_name)

	def set_host(self, host):
		self.host = host
		self.cache_name = self.host.replace('/','').replace(':','..') + '_-_' + self.now + '.cache'

	def get_num_urls(self):
		return len(set([self.queue[key].id for key in self.queue]))

	def get_responses(self):
		return [self.queue[key] for key in self.queue]

	def save(self):
		# save the queue for later use
		# this will help limit the amount of requests made
		# when scanning the same site multiple times
		with self.mutex:
			file_name = self._get_name_for_cache_file()
			with open(file_name, 'wb') as cache_file:
				try:
					pickle.dump(self.queue, cache_file)
				except:
					print('Error saving cache: ' + file_name)

	def load(self):
		# loads previously saved cache for the host

		# bail if the host is not set
		if self.host is None: return None

		# search the cache dir
		for cache_file in os.listdir(self.cache_dir):

			# skip the file if it's not a cache file
			if not cache_file.endswith('.cache'):
				continue

			# check if the cache is for the host
			hostname, time_ext = cache_file.split('_-_')
			save_time,_ = time_ext.split('.')

			# calc the age of the cache
			age = int(self.now) - int(save_time)

			# overwrite the current queue if it's for the host and the cache is not too old
			if hostname == self.cache_name.split('_-_')[0] and age < self.cache_ttl:
				file_name = os.path.join(self.cache_dir, cache_file)
				try:
					with open(file_name, 'rb') as f:
						data = pickle.load(f)
						for path in data:
							self.__setitem__(path, data[path])
				except:
					print('Error loading cache: ' +  file_name)

<commit_msg>Check if cache dir exists - if not, create it.<commit_after>import queue, pickle, os, time, hashlib, re

# wig - Cache
#
# wig uses a cache to store the requests and responses made during a scan.
# This helps limit the amount of requests that it makes, as a request for
# resource is only made once.
# To further limit the amount of requests, wig saves a copy of the cache
# and will reuse it for scans run within 24 hours.
#
# NOTE: The pickle module is used to save and load the cache. This has the
#       side effect, that the extra variables which are added to a an
#       instance of Requests are not kept. Currently 2 variables are added:
#       md5 and md5_404. These values have to be recalculated when loading
#       a pickled cache. Ideally, a custom object should be created for
#       the modified requests, so this is on the TODO.

class Cache(queue.Queue):
	def _init(self, maxsize):
		self.queue = dict()
		self.host = None
		self.cache_dir = './cache/'
		self.now = str(time.time()).split('.')[0]

		# only load cache data that is new than this
		# (currently this is set for 24 hours)
		self.cache_ttl = 60*60*24

		# check if cache dir exists - create if not
		self._check_or_create_cache()

		# check if there are caches that are older than ttl
		self._remove_old_caches()

	def __getitem__(self, path):
		return self.queue[path]

	def __setitem__(self, path, response):
		with self.mutex:
			self.queue[path] = response

	def __contains__(self, url):
		with self.mutex:
			return url in self.queue

	def _check_or_create_cache(self):
		if not os.path.exists(self.cache_dir):
			os.makedirs(self.cache_dir)

	def _remove_old_caches(self):
		# remove caches that are too old

		# bail if the directory does not exist
		if not os.path.exists(self.cache_dir): return None

		# iterate over the cache files
		for cache_file in os.listdir(self.cache_dir):

			# skip the file if it's not a cache file
			if not cache_file.endswith('.cache'):
				continue

			# check if the cache is for the host
			hostname, time_ext = cache_file.split('_-_')
			save_time,_ = time_ext.split('.')

			# check the age of the cache, and remove it if older than
			# ttl
			age = int(self.now) - int(save_time)
			if age > self.cache_ttl:
				file_name = os.path.join(self.cache_dir, cache_file)
				os.remove(file_name)

	def _get_name_for_cache_file(self):
		# check if there already is an older version of the cache
		# if there is, return the name of this file, such that,
		# it will be overwritten
		if not os.path.exists(self.cache_dir):
			os.makedirs(self.cache_dir)

		for cache_file in os.listdir(self.cache_dir):
			# skip the file if it's not a cache file
			if not cache_file.endswith('.cache'):
				continue

			# check if the cache is for the host
			hostname,_ = cache_file.split('_-_')
			if hostname == self.cache_name.split('_-_')[0]:
				return os.path.join(self.cache_dir, cache_file)

		# if there aren't any previous cache files, generate a
		# new name for the cache
		return os.path.join(self.cache_dir, self.cache_name)

	def set_host(self, host):
		self.host = host
		self.cache_name = self.host.replace('/','').replace(':','..') + '_-_' + self.now + '.cache'

	def get_num_urls(self):
		return len(set([self.queue[key].id for key in self.queue]))

	def get_responses(self):
		return [self.queue[key] for key in self.queue]

	def save(self):
		# save the queue for later use
		# this will help limit the amount of requests made
		# when scanning the same site multiple times
		with self.mutex:
			file_name = self._get_name_for_cache_file()
			with open(file_name, 'wb') as cache_file:
				try:
					pickle.dump(self.queue, cache_file)
				except:
					print('Error saving cache: ' + file_name)

	def load(self):
		# loads previously saved cache for the host

		# bail if the host is not set
		if self.host is None: return None

		# search the cache dir
		for cache_file in os.listdir(self.cache_dir):

			# skip the file if it's not a cache file
			if not cache_file.endswith('.cache'):
				continue

			# check if the cache is for the host
			hostname, time_ext = cache_file.split('_-_')
			save_time,_ = time_ext.split('.')

			# calc the age of the cache
			age = int(self.now) - int(save_time)

			# overwrite the current queue if it's for the host and the cache is not too old
			if hostname == self.cache_name.split('_-_')[0] and age < self.cache_ttl:
				file_name = os.path.join(self.cache_dir, cache_file)
				try:
					with open(file_name, 'rb') as f:
						data = pickle.load(f)
						for path in data:
							self.__setitem__(path, data[path])
				except:
					print('Error loading cache: ' +  file_name)

<|endoftext|>


<commit_before>from __future__ import print_function
import operator

import numpy as np
from sklearn.metrics import euclidean_distances

#==========================================================================================================
# classical multidimensional scaling as described in https://en.wikipedia.org/wiki/Multidimensional_scaling
#==========================================================================================================

# number of dimensions of output
M = 2

#       5m
#  ................
#   .           .
#    .        .
# 3m  .     .  4m
#      .  .
#       .

# Set up the squared proximity matrix
prox_arr = np.array([[0, 5, 3], [5, 0, 4], [3, 4, 0]])
sqrd_prox_arr = prox_arr**2

# Apply double centering
sz = prox_arr.shape[0]
cent_arr = np.eye(sz) - np.ones(sz)/sz
B = -cent_arr.dot(sqrd_prox_arr).dot(cent_arr)/2

# Determine the m largest eigenvalues and corresponding eigenvectors
eig_vals, eig_vecs = np.linalg.eig(B)
eig_vals_vecs = zip(*sorted(zip(eig_vals, eig_vecs.T), key=operator.itemgetter(0), reverse=True)[:M])
eig_vals, eig_vecs = map(np.array, eig_vals_vecs)

# configuration X of n points/coordinates that optimise the cost function
coords = eig_vecs.T.dot((np.eye(M)*eig_vals)**0.5)
print(coords)

# compute euclidean distances
similairities = euclidean_distances(coords)
print(similairities)

# compare the new distance matrix with the original
np.testing.assert_allclose(similairities, prox_arr)
<commit_msg>update comments<commit_after>from __future__ import print_function
import operator

import numpy as np
from sklearn.metrics import euclidean_distances

#==========================================================================================================
# classical multidimensional scaling as described in https://en.wikipedia.org/wiki/Multidimensional_scaling
#==========================================================================================================

# number of dimensions of output
M = 2

#       5m
#  ................
#   .           .
#    .        .
# 3m  .     .  4m
#      .  .
#       .

# Set up the proximity and squared proximity matrices
prox_arr = np.array([[0, 5, 3], [5, 0, 4], [3, 4, 0]])
sqrd_prox_arr = prox_arr**2

# Apply double centering
sz = prox_arr.shape[0]
cent_arr = np.eye(sz) - np.ones(sz)/sz
B = -cent_arr.dot(sqrd_prox_arr).dot(cent_arr)/2

# Determine the M largest eigenvalues and corresponding eigenvectors
eig_vals, eig_vecs = np.linalg.eig(B)
eig_vals_vecs = zip(*sorted(zip(eig_vals, eig_vecs.T), key=operator.itemgetter(0), reverse=True)[:M])
eig_vals, eig_vecs = map(np.array, eig_vals_vecs)

# Configuration X of n points/coordinates that optimise the cost function
coords = eig_vecs.T.dot((np.eye(M)*eig_vals)**0.5)
print(coords)

# Compute euclidean distances
similairities = euclidean_distances(coords)
print(similairities)

# Compare the new proximity matrix with the original
np.testing.assert_allclose(similairities, prox_arr)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import time
from collections import namedtuple

from core import Application
from middleware import Middleware
from render import default_response

# TODO: what are some sane-default intervals?

class StatsMiddleware(Middleware):
    def __init__(self):
        self.hits = []
        self.per_route_resp_counts = {}
        self.per_route_perfs = {}

    def request(self, next, request, _route):
        start_time = time.time()
        pattern = _route.rule
        try:
            rt_counts = self.per_route_resp_counts[pattern]
        except KeyError:
            self.per_route_resp_counts[pattern] = rt_counts = {}
        try:
            resp = next()
            resp_type = getattr(resp, 'status_code', type(resp))
        except Exception as e:
            # see Werkzeug #388
            resp_type = getattr(e, 'code', type(e))
            raise
        finally:
            try:
                rt_counts[repr(resp_type)] += 1
            except KeyError:
                rt_counts[repr(resp_type)] = 1
            end_time = time.time()
            elapsed_time = end_time - start_time
            try:
                self.per_route_perfs[pattern].append(elapsed_time)
            except KeyError:
                self.per_route_perfs[pattern] = [elapsed_time]
        return resp

def _get_stats_dict(_application):
    try:
        stats_mw = [mw for mw in _application.middlewares
                    if isinstance(mw, StatsMiddleware)][0]
    except IndexError:
        return {'error': "StatsMiddleware doesn't seem to be installed"}
    return {'resp_counts': stats_mw.per_route_resp_counts,
            'resp_perfs': stats_mw.per_route_perfs}

def _create_app():
    routes = [('/', _get_stats_dict, default_response)]
    mws = [StatsMiddleware()]
    app = Application(routes, middlewares=mws)
    return app

if __name__ == '__main__':
    sapp = _create_app()
    sapp.serve()
<commit_msg>fix up some basic stats up in this<commit_after># -*- coding: utf-8 -*-

import time
from collections import namedtuple

from core import Application
from middleware import Middleware
from render import default_response

# TODO: what are some sane-default intervals?

Hit = namedtuple('Hit', 'start_time url pattern status_code '
                 ' elapsed_time content_type')

class StatsMiddleware(Middleware):
    def __init__(self):
        self.hits = []
        self.route_hits = {}
        self.url_hits = {}

    def request(self, next, request, _route):
        start_time = time.time()
        try:
            resp = next()
            resp_status = repr(getattr(resp, 'status_code', type(resp)))
            resp_mime_type = resp.content_type.partition(';')[0]
        except Exception as e:
            # see Werkzeug #388
            resp_status = repr(getattr(e, 'code', type(e)))
            resp_mime_type = getattr(e, 'content_type', '').partition(';')[0]
            raise
        finally:
            end_time = time.time()
            elapsed_time = end_time - start_time
            hit = Hit(start_time,
                      request.path,
                      _route.rule,
                      resp_status,
                      elapsed_time,
                      resp_mime_type)
            self.hits.append(hit)
            self.route_hits.setdefault(_route, []).append(hit)
            self.url_hits.setdefault(request.path, []).append(hit)
        return resp

from math import floor, ceil
import itertools

def hits_minutes_ago(hit_list, minutes=None):
    if minutes is None:
        minutes = 0
    start_time = time.time() - (minutes * 60)
    return itertools.dropwhile(lambda h: h.start_time < start_time, hit_list)

def hits_by_status(hit_list):
    ret = {}
    for hit in hit_list:
        try:
            ret[hit.status_code].append(hit)
        except KeyError:
            ret[hit.status_code] = [hit]
    return ret

def percentile(unsorted_data, ptile=50):
    if ptile > 100:
        raise ValueError("it's percentile, not something-else-tile")
    if not unsorted_data:
        return 0.0  # TODO: hrm, lazy
    data = sorted(unsorted_data)
    idx = (float(ptile) / 100) * len(data)
    idx_f, idx_c = int(floor(idx)), min(int(ceil(idx)), len(data) - 1)
    return (data[idx_f] + data[idx_c]) / 2.0

def mean(vals):
    if vals:
        return sum(vals, 0.0) / len(vals)
    else:
        return 0.0

def float_round(n):
    return n - (n % 2 ** -6)

def get_route_stats(rt_hits):
    ret = {}
    hbs = hits_by_status(rt_hits)
    for status, hits in hbs.items():
        ret[status] = cur = {}
        durs = [round(h.elapsed_time * 1000, 2) for h in hits]
        cur['min'] = min(durs)
        cur['max'] = max(durs)
        cur['mean'] = mean(durs)
        cur['count'] = len(durs)
        cur['median'] = percentile(durs, 50)
        cur['ninefive'] = percentile(durs, 95)
    return ret

def _get_stats_dict(_application):
    try:
        stats_mw = [mw for mw in _application.middlewares
                    if isinstance(mw, StatsMiddleware)][0]
    except IndexError:
        return {'error': "StatsMiddleware doesn't seem to be installed"}
    rt_hits = stats_mw.route_hits
    return {'resp_counts': dict([(url, len(rh)) for url, rh
                                 in stats_mw.url_hits.items()]),
            'route_stats': dict([(rt.rule, get_route_stats(rh)) for rt, rh
                                 in rt_hits.items() if rh])}

def _create_app():
    routes = [('/', _get_stats_dict, default_response)]
    mws = [StatsMiddleware()]
    app = Application(routes, middlewares=mws)
    return app

if __name__ == '__main__':
    sapp = _create_app()
    sapp.serve()
<|endoftext|>


<commit_before>"""Functions to help clean & normalize business names.

See http://www.unicode.org/reports/tr15/#Normalization_Forms_Table for details
on Unicode normalization and the NFKD normalization used here.

Basic usage:

>> terms = prepare_terms()
>> basename("Daddy & Sons, Ltd.", terms, prefix=True, middle=True, suffix=True)
Daddy & Sons

"""

import functools
import operator
from collections import OrderedDict
import re
import unicodedata
from .termdata import terms_by_type, terms_by_country
from .non_nfkd_map import NON_NFKD_MAP

tail_removal_rexp = re.compile(r"[^\.\w]+$", flags=re.UNICODE)

def get_unique_terms():
    "retrieve all unique terms from termdata definitions"
    ts = functools.reduce(operator.iconcat, terms_by_type.values(), [])
    cs = functools.reduce(operator.iconcat, terms_by_country.values(), [])
    return set(ts + cs)

def remove_accents(t):
    """based on https://stackoverflow.com/a/51230541"""
    nfkd_form = unicodedata.normalize('NFKD', t.casefold())
    return ''.join(
        NON_NFKD_MAP[c]
            if c in NON_NFKD_MAP
        else c
            for part in nfkd_form for c in part
            if unicodedata.category(part) != 'Mn'
        )

def strip_punct(t):
    return t.replace(".", "").replace(",", "").replace("-", "")

def normalize_terms(terms):
    "normalize terms"
    return (strip_punct(remove_accents(t)) for t in terms)

def strip_tail(name):
    "get rid of all trailing non-letter symbols except the dot"
    match = re.search(tail_removal_rexp, name)
    if match is not None:
        name = name[: match.span()[0]]
    return name

def normalized(text):
    "caseless Unicode normalization"
    return remove_accents(text)

def prepare_terms(custom_terms=None):
    """
    construct an optimized term structure for basename extraction

    :param custom_terms:    list of custom / additional terms

    :return:    list of splited and normalized tuples in the following format:

                input: list of terms        ["GmbH & Co. KG", "C. POR A."]
                output: list of tuples      [(4, ['gmbh', '&', 'co.', 'kg']),
                                             (3, ['c.', 'por', 'a.'])]
       where    4 - is the number of tokens in the term "GmbH & Co. KG"
    """
    terms = get_unique_terms()
    terms |= set(normalize_terms(custom_terms))
    nterms = normalize_terms(terms)
    ntermparts = (t.split() for t in nterms)
    # make sure that the result is deterministic, sort terms descending by number of tokens, ascending by names
    sntermparts = sorted(ntermparts, key=lambda x: (-len(x), x))
    return [(len(tp), tp) for tp in sntermparts]

def basename(name, terms, suffix=True, prefix=False, middle=False, **kwargs):
    "return cleaned base version of the business name"

    name = strip_tail(name)
    nparts = name.split()
    nname = normalized(name)
    nnparts = list(map(strip_punct, nname.split()))
    nnsize = len(nnparts)

    if suffix:
        for termsize, termparts in terms:
            if nnparts[-termsize:] == termparts:
                del nnparts[-termsize:]
                del nparts[-termsize:]

    if prefix:
        for termsize, termparts in terms:
            if nnparts[:termsize] == termparts:
                del nnparts[:termsize]
                del nparts[:termsize]

    if middle:
        for termsize, termparts in terms:
            if termsize > 1:
                sizediff = nnsize - termsize
                if sizediff > 1:
                    for i in range(0, nnsize-termsize+1):
                        if termparts == nnparts[i:i+termsize]:
                            del nnparts[i:i+termsize]
                            del nparts[i:i+termsize]
            else:
                if termparts[0] in nnparts[1:-1]:
                    idx = nnparts[1:-1].index(termparts[0])
                    del nnparts[idx+1]
                    del nparts[idx+1]

    return strip_tail(" ".join(nparts))

<commit_msg>Revert "Merge pull request #59 from maxu777/cleanco_maxu777"<commit_after>"""Functions to help clean & normalize business names.

See http://www.unicode.org/reports/tr15/#Normalization_Forms_Table for details
on Unicode normalization and the NFKD normalization used here.

Basic usage:

>> terms = prepare_terms()
>> basename("Daddy & Sons, Ltd.", terms, prefix=True, middle=True, suffix=True)
Daddy & Sons

"""

import functools
import operator
from collections import OrderedDict
import re
import unicodedata
from .termdata import terms_by_type, terms_by_country
from .non_nfkd_map import NON_NFKD_MAP

tail_removal_rexp = re.compile(r"[^\.\w]+$", flags=re.UNICODE)

def get_unique_terms():
    "retrieve all unique terms from termdata definitions"
    ts = functools.reduce(operator.iconcat, terms_by_type.values(), [])
    cs = functools.reduce(operator.iconcat, terms_by_country.values(), [])
    return set(ts + cs)

def remove_accents(t):
    """based on https://stackoverflow.com/a/51230541"""
    nfkd_form = unicodedata.normalize('NFKD', t.casefold())
    return ''.join(
        NON_NFKD_MAP[c]
            if c in NON_NFKD_MAP
        else c
            for part in nfkd_form for c in part
            if unicodedata.category(part) != 'Mn'
        )

def strip_punct(t):
    return t.replace(".", "").replace(",", "").replace("-", "")

def normalize_terms(terms):
    "normalize terms"
    return (strip_punct(remove_accents(t)) for t in terms)

def strip_tail(name):
    "get rid of all trailing non-letter symbols except the dot"
    match = re.search(tail_removal_rexp, name)
    if match is not None:
        name = name[: match.span()[0]]
    return name

def normalized(text):
    "caseless Unicode normalization"
    return remove_accents(text)

def prepare_terms():
    "construct an optimized term structure for basename extraction"
    terms = get_unique_terms()
    nterms = normalize_terms(terms)
    ntermparts = (t.split() for t in nterms)
    # make sure that the result is deterministic, sort terms descending by number of tokens, ascending by names
    sntermparts = sorted(ntermparts, key=lambda x: (-len(x), x))
    return [(len(tp), tp) for tp in sntermparts]

def basename(name, terms, suffix=True, prefix=False, middle=False, **kwargs):
    "return cleaned base version of the business name"

    name = strip_tail(name)
    nparts = name.split()
    nname = normalized(name)
    nnparts = list(map(strip_punct, nname.split()))
    nnsize = len(nnparts)

    if suffix:
        for termsize, termparts in terms:
            if nnparts[-termsize:] == termparts:
                del nnparts[-termsize:]
                del nparts[-termsize:]

    if prefix:
        for termsize, termparts in terms:
            if nnparts[:termsize] == termparts:
                del nnparts[:termsize]
                del nparts[:termsize]

    if middle:
        for termsize, termparts in terms:
            if termsize > 1:
                sizediff = nnsize - termsize
                if sizediff > 1:
                    for i in range(0, nnsize-termsize+1):
                        if termparts == nnparts[i:i+termsize]:
                            del nnparts[i:i+termsize]
                            del nparts[i:i+termsize]
            else:
                if termparts[0] in nnparts[1:-1]:
                    idx = nnparts[1:-1].index(termparts[0])
                    del nnparts[idx+1]
                    del nparts[idx+1]

    return strip_tail(" ".join(nparts))

<|endoftext|>


<commit_before>import os
import sys
import shutil
import tempfile
import contextlib
import shlex

from ._compat import iteritems, PY2, string_types

# If someone wants to vendor click, we want to ensure the
# correct package is discovered.  Ideally we could use a
# relative import here but unfortunately Python does not
# support that.
clickpkg = sys.modules[__name__.rsplit('.', 1)[0]]

if PY2:
    from cStringIO import StringIO
else:
    import io
    from ._compat import _find_binary_reader

class EchoingStdin(object):

    def __init__(self, input, output):
        self._input = input
        self._output = output

    def __getattr__(self, x):
        return getattr(self._input, x)

    def _echo(self, rv):
        self._output.write(rv)
        return rv

    def read(self, n=-1):
        return self._echo(self._input.read(n))

    def readline(self, n=-1):
        return self._echo(self._input.readline(n))

    def readlines(self):
        return [self._echo(x) for x in self._input.readlines()]

    def __iter__(self):
        return iter(self._echo(x) for x in self._input)

    def __repr__(self):
        return repr(self._input)

def make_input_stream(input, charset):
    # Is already an input stream.
    if hasattr(input, 'read'):
        if PY2:
            return input
        rv = _find_binary_reader(input)
        if rv is not None:
            return rv
        raise TypeError('Could not find binary reader for input stream.')

    if input is None:
        input = b''
    elif not isinstance(input, bytes):
        input = input.encode(charset)
    if PY2:
        return StringIO(input)
    return io.BytesIO(input)

class Result(object):
    """Holds the captured result of an invoked CLI script."""

    def __init__(self, runner, stdout_bytes, stderr_bytes, exit_code,
                 exception, exc_info=None):
        #: The runner that created the result
        self.runner = runner
        #: The standard output as bytes.
        self.stdout_bytes = stdout_bytes
        #: The standard error as bytes, or False(y) if not available
        self.stderr_bytes = stderr_bytes
        #: The exit code as integer.
        self.exit_code = exit_code
        #: The exception that happend if one did.
        self.exception = exception
        #: The traceback
        self.exc_info = exc_info

    @property
    def output(self):
        """The (standard) output as unicode string."""
        return self.stdout

    @property
    def stdout(self):
        """The standard output as unicode string."""
        return self.stdout_bytes.decode(self.runner.charset, 'replace') \
            .replace('\r\n', '\n')

    @property
    def stderr(self):
        """The standard error as unicode string."""
        if not self.stderr_bytes:
            raise ValueError("stderr not separately captured")
        return self.stderr_bytes.decode(self.runner.charset, 'replace') \
            .replace('\r\n', '\n')

    def __repr__(self):
        return '<%s %s>' % (
            type(self).__name__,
            self.exception and repr(self.exception) or 'okay',
        )

class CliRunner(object):
    """The CLI runner provides functionality to invoke a Click command line
    script for unittesting purposes in a isolated environment.  This only
    works in single-threaded systems without any concurrency as it changes the
    global interpreter state.

    :param charset: the character set for the input and output data.  This is
                    UTF-8 by default and should not be changed currently as
                    the reporting to Click only works in Python 2 properly.
    :param env: a dictionary with environment variables for overriding.
    :param echo_stdin: if this is set to `True`, then reading from stdin writes
                       to stdout.  This is useful for showing examples in
                       some circumstances.  Note that regular prompts
                       will automatically echo the input.
    :param mix_stderr: if this is set to `False`, then stdout and stderr are
                       preserved as independent streams.  This is useful for
                       Unix-philosophy apps that have predictable stdout and
                       noisy stderr, such that each may be measured
                       independently
    """

    def __init__(self, charset=None, env=None, echo_stdin=False,
                 mix_stderr=True):
        if charset is None:
            charset = 'utf-8'
        self.charset = charset
        self.env = env or {}
        self.echo_stdin = echo_stdin
        self.mix_stderr = mix_stderr

    def get_default_prog_name(self, cli):
        """Given a command object it will return the default program name
        for it.  The default is the `name` attribute or ``"root"`` if not
        set.
        """
        return cli.name or 'root'

    def make_env(self, overrides=None):
        """Returns the environment overrides for invoking a script."""
        rv = dict(self.env)
        if overrides:
            rv.update(overrides)
        return rv

    @contextlib.contextmanager
    def isolation(self, input=None, env=None, color=False):
        """A context manager that sets up the isolation for invoking of a
        command line tool.  This sets up stdin with the given input data
        and `os.environ` with the overrides from the given dictionary.
        This also rebinds some internals in Click to be mocked (like the
        prompt functionality).

        This is automatically done in the :meth:`invoke` method.

        .. versionadded:: 4.0
           The ``color`` parameter was added.

        :param input: the input stream to put into sys.stdin.
        :param env: the environment overrides as dictionary.
        :param color: whether the output should contain color codes. The
                      application can still override this explicitly.
        """
        input = make_input_stream(input, self.charset)

        old_stdin = sys.stdin
        old_stdout = sys.stdout
        old_stderr = sys.stderr
        old_forced_width = clickpkg.formatting.FORCED_WIDTH
        clickpkg.formatting.FORCED_WIDTH = 80

        env = self.make_env(env)

        if PY2:
            bytes_output = StringIO()
            if self.echo_stdin:
                input = EchoingStdin(input, bytes_output)
            sys.stdout = bytes_output
            if not self.mix_stderr:
                bytes_error = StringIO()
                sys.stderr = bytes_error
        else:
            bytes_output = io.BytesIO()
            if self.echo_stdin:
                input = EchoingStdin(input, bytes_output)
            input = io.TextIOWrapper(input, encoding=self.charset)
            sys.stdout = io.TextIOWrapper(
                bytes_output, encoding=self.charset)
            if not self.mix_stderr:
                bytes_error = io.BytesIO()
                sys.stderr = io.TextIOWrapper(
                    bytes_error, encoding=self.charset)

        if self.mix_stderr:
            sys.stderr = sys.stdout

        sys.stdin = input

        def visible_input(prompt=None):
            sys.stdout.write(prompt or '')
            val = input.readline().rstrip('\r\n')
            sys.stdout.write(val + '\n')
            sys.stdout.flush()
            return val

        def hidden_input(prompt=None):
            sys.stdout.write((prompt or '') + '\n')
            sys.stdout.flush()
            return input.readline().rstrip('\r\n')

        def _getchar(echo):
            char = sys.stdin.read(1)
            if echo:
                sys.stdout.write(char)
                sys.stdout.flush()
            return char

        default_color = color

        def should_strip_ansi(stream=None, color=None):
            if color is None:
                return not default_color
            return not color

        old_visible_prompt_func = clickpkg.termui.visible_prompt_func
        old_hidden_prompt_func = clickpkg.termui.hidden_prompt_func
        old__getchar_func = clickpkg.termui._getchar
        old_should_strip_ansi = clickpkg.utils.should_strip_ansi
        clickpkg.termui.visible_prompt_func = visible_input
        clickpkg.termui.hidden_prompt_func = hidden_input
        clickpkg.termui._getchar = _getchar
        clickpkg.utils.should_strip_ansi = should_strip_ansi

        old_env = {}
        try:
            for key, value in iteritems(env):
                old_env[key] = os.environ.get(key)
                if value is None:
                    try:
                        del os.environ[key]
                    except Exception:
                        pass
                else:
                    os.environ[key] = value
            yield (bytes_output, not self.mix_stderr and bytes_error)
        finally:
            for key, value in iteritems(old_env):
                if value is None:
                    try:
                        del os.environ[key]
                    except Exception:
                        pass
                else:
                    os.environ[key] = value
            sys.stdout = old_stdout
            sys.stderr = old_stderr
            sys.stdin = old_stdin
            clickpkg.termui.visible_prompt_func = old_visible_prompt_func
            clickpkg.termui.hidden_prompt_func = old_hidden_prompt_func
            clickpkg.termui._getchar = old__getchar_func
            clickpkg.utils.should_strip_ansi = old_should_strip_ansi
            clickpkg.formatting.FORCED_WIDTH = old_forced_width

    def invoke(self, cli, args=None, input=None, env=None,
               catch_exceptions=True, color=False, mix_stderr=False, **extra):
        """Invokes a command in an isolated environment.  The arguments are
        forwarded directly to the command line script, the `extra` keyword
        arguments are passed to the :meth:`~clickpkg.Command.main` function of
        the command.

        This returns a :class:`Result` object.

        .. versionadded:: 3.0
           The ``catch_exceptions`` parameter was added.

        .. versionchanged:: 3.0
           The result object now has an `exc_info` attribute with the
           traceback if available.

        .. versionadded:: 4.0
           The ``color`` parameter was added.

        :param cli: the command to invoke
        :param args: the arguments to invoke. It may be given as an iterable
                     or a string. When given as string it will be interpreted
                     as a Unix shell command. More details at
                     :func:`shlex.split`.
        :param input: the input data for `sys.stdin`.
        :param env: the environment overrides.
        :param catch_exceptions: Whether to catch any other exceptions than
                                 ``SystemExit``.
        :param extra: the keyword arguments to pass to :meth:`main`.
        :param color: whether the output should contain color codes. The
                      application can still override this explicitly.
        """
        exc_info = None
        with self.isolation(input=input, env=env, color=color) as outstreams:
            exception = None
            exit_code = 0

            if isinstance(args, string_types):
                args = shlex.split(args)

            try:
                prog_name = extra.pop("prog_name")
            except KeyError:
                prog_name = self.get_default_prog_name(cli)

            try:
                cli.main(args=args or (), prog_name=prog_name, **extra)
            except SystemExit as e:
                exc_info = sys.exc_info()
                exit_code = e.code
                if exit_code is None:
                    exit_code = 0

                if exit_code != 0:
                    exception = e

                if not isinstance(exit_code, int):
                    sys.stdout.write(str(exit_code))
                    sys.stdout.write('\n')
                    exit_code = 1

            except Exception as e:
                if not catch_exceptions:
                    raise
                exception = e
                exit_code = 1
                exc_info = sys.exc_info()
            finally:
                sys.stdout.flush()
                stdout = outstreams[0].getvalue()
                stderr = outstreams[1] and outstreams[1].getvalue()

        return Result(runner=self,
                      stdout_bytes=stdout,
                      stderr_bytes=stderr,
                      exit_code=exit_code,
                      exception=exception,
                      exc_info=exc_info)

    @contextlib.contextmanager
    def isolated_filesystem(self):
        """A context manager that creates a temporary folder and changes
        the current working directory to it for isolated filesystem tests.
        """
        cwd = os.getcwd()
        t = tempfile.mkdtemp()
        os.chdir(t)
        try:
            yield t
        finally:
            os.chdir(cwd)
            try:
                shutil.rmtree(t)
            except (OSError, IOError):
                pass
<commit_msg>fix typo<commit_after>import os
import sys
import shutil
import tempfile
import contextlib
import shlex

from ._compat import iteritems, PY2, string_types

# If someone wants to vendor click, we want to ensure the
# correct package is discovered.  Ideally we could use a
# relative import here but unfortunately Python does not
# support that.
clickpkg = sys.modules[__name__.rsplit('.', 1)[0]]

if PY2:
    from cStringIO import StringIO
else:
    import io
    from ._compat import _find_binary_reader

class EchoingStdin(object):

    def __init__(self, input, output):
        self._input = input
        self._output = output

    def __getattr__(self, x):
        return getattr(self._input, x)

    def _echo(self, rv):
        self._output.write(rv)
        return rv

    def read(self, n=-1):
        return self._echo(self._input.read(n))

    def readline(self, n=-1):
        return self._echo(self._input.readline(n))

    def readlines(self):
        return [self._echo(x) for x in self._input.readlines()]

    def __iter__(self):
        return iter(self._echo(x) for x in self._input)

    def __repr__(self):
        return repr(self._input)

def make_input_stream(input, charset):
    # Is already an input stream.
    if hasattr(input, 'read'):
        if PY2:
            return input
        rv = _find_binary_reader(input)
        if rv is not None:
            return rv
        raise TypeError('Could not find binary reader for input stream.')

    if input is None:
        input = b''
    elif not isinstance(input, bytes):
        input = input.encode(charset)
    if PY2:
        return StringIO(input)
    return io.BytesIO(input)

class Result(object):
    """Holds the captured result of an invoked CLI script."""

    def __init__(self, runner, stdout_bytes, stderr_bytes, exit_code,
                 exception, exc_info=None):
        #: The runner that created the result
        self.runner = runner
        #: The standard output as bytes.
        self.stdout_bytes = stdout_bytes
        #: The standard error as bytes, or False(y) if not available
        self.stderr_bytes = stderr_bytes
        #: The exit code as integer.
        self.exit_code = exit_code
        #: The exception that happened if one did.
        self.exception = exception
        #: The traceback
        self.exc_info = exc_info

    @property
    def output(self):
        """The (standard) output as unicode string."""
        return self.stdout

    @property
    def stdout(self):
        """The standard output as unicode string."""
        return self.stdout_bytes.decode(self.runner.charset, 'replace') \
            .replace('\r\n', '\n')

    @property
    def stderr(self):
        """The standard error as unicode string."""
        if not self.stderr_bytes:
            raise ValueError("stderr not separately captured")
        return self.stderr_bytes.decode(self.runner.charset, 'replace') \
            .replace('\r\n', '\n')

    def __repr__(self):
        return '<%s %s>' % (
            type(self).__name__,
            self.exception and repr(self.exception) or 'okay',
        )

class CliRunner(object):
    """The CLI runner provides functionality to invoke a Click command line
    script for unittesting purposes in a isolated environment.  This only
    works in single-threaded systems without any concurrency as it changes the
    global interpreter state.

    :param charset: the character set for the input and output data.  This is
                    UTF-8 by default and should not be changed currently as
                    the reporting to Click only works in Python 2 properly.
    :param env: a dictionary with environment variables for overriding.
    :param echo_stdin: if this is set to `True`, then reading from stdin writes
                       to stdout.  This is useful for showing examples in
                       some circumstances.  Note that regular prompts
                       will automatically echo the input.
    :param mix_stderr: if this is set to `False`, then stdout and stderr are
                       preserved as independent streams.  This is useful for
                       Unix-philosophy apps that have predictable stdout and
                       noisy stderr, such that each may be measured
                       independently
    """

    def __init__(self, charset=None, env=None, echo_stdin=False,
                 mix_stderr=True):
        if charset is None:
            charset = 'utf-8'
        self.charset = charset
        self.env = env or {}
        self.echo_stdin = echo_stdin
        self.mix_stderr = mix_stderr

    def get_default_prog_name(self, cli):
        """Given a command object it will return the default program name
        for it.  The default is the `name` attribute or ``"root"`` if not
        set.
        """
        return cli.name or 'root'

    def make_env(self, overrides=None):
        """Returns the environment overrides for invoking a script."""
        rv = dict(self.env)
        if overrides:
            rv.update(overrides)
        return rv

    @contextlib.contextmanager
    def isolation(self, input=None, env=None, color=False):
        """A context manager that sets up the isolation for invoking of a
        command line tool.  This sets up stdin with the given input data
        and `os.environ` with the overrides from the given dictionary.
        This also rebinds some internals in Click to be mocked (like the
        prompt functionality).

        This is automatically done in the :meth:`invoke` method.

        .. versionadded:: 4.0
           The ``color`` parameter was added.

        :param input: the input stream to put into sys.stdin.
        :param env: the environment overrides as dictionary.
        :param color: whether the output should contain color codes. The
                      application can still override this explicitly.
        """
        input = make_input_stream(input, self.charset)

        old_stdin = sys.stdin
        old_stdout = sys.stdout
        old_stderr = sys.stderr
        old_forced_width = clickpkg.formatting.FORCED_WIDTH
        clickpkg.formatting.FORCED_WIDTH = 80

        env = self.make_env(env)

        if PY2:
            bytes_output = StringIO()
            if self.echo_stdin:
                input = EchoingStdin(input, bytes_output)
            sys.stdout = bytes_output
            if not self.mix_stderr:
                bytes_error = StringIO()
                sys.stderr = bytes_error
        else:
            bytes_output = io.BytesIO()
            if self.echo_stdin:
                input = EchoingStdin(input, bytes_output)
            input = io.TextIOWrapper(input, encoding=self.charset)
            sys.stdout = io.TextIOWrapper(
                bytes_output, encoding=self.charset)
            if not self.mix_stderr:
                bytes_error = io.BytesIO()
                sys.stderr = io.TextIOWrapper(
                    bytes_error, encoding=self.charset)

        if self.mix_stderr:
            sys.stderr = sys.stdout

        sys.stdin = input

        def visible_input(prompt=None):
            sys.stdout.write(prompt or '')
            val = input.readline().rstrip('\r\n')
            sys.stdout.write(val + '\n')
            sys.stdout.flush()
            return val

        def hidden_input(prompt=None):
            sys.stdout.write((prompt or '') + '\n')
            sys.stdout.flush()
            return input.readline().rstrip('\r\n')

        def _getchar(echo):
            char = sys.stdin.read(1)
            if echo:
                sys.stdout.write(char)
                sys.stdout.flush()
            return char

        default_color = color

        def should_strip_ansi(stream=None, color=None):
            if color is None:
                return not default_color
            return not color

        old_visible_prompt_func = clickpkg.termui.visible_prompt_func
        old_hidden_prompt_func = clickpkg.termui.hidden_prompt_func
        old__getchar_func = clickpkg.termui._getchar
        old_should_strip_ansi = clickpkg.utils.should_strip_ansi
        clickpkg.termui.visible_prompt_func = visible_input
        clickpkg.termui.hidden_prompt_func = hidden_input
        clickpkg.termui._getchar = _getchar
        clickpkg.utils.should_strip_ansi = should_strip_ansi

        old_env = {}
        try:
            for key, value in iteritems(env):
                old_env[key] = os.environ.get(key)
                if value is None:
                    try:
                        del os.environ[key]
                    except Exception:
                        pass
                else:
                    os.environ[key] = value
            yield (bytes_output, not self.mix_stderr and bytes_error)
        finally:
            for key, value in iteritems(old_env):
                if value is None:
                    try:
                        del os.environ[key]
                    except Exception:
                        pass
                else:
                    os.environ[key] = value
            sys.stdout = old_stdout
            sys.stderr = old_stderr
            sys.stdin = old_stdin
            clickpkg.termui.visible_prompt_func = old_visible_prompt_func
            clickpkg.termui.hidden_prompt_func = old_hidden_prompt_func
            clickpkg.termui._getchar = old__getchar_func
            clickpkg.utils.should_strip_ansi = old_should_strip_ansi
            clickpkg.formatting.FORCED_WIDTH = old_forced_width

    def invoke(self, cli, args=None, input=None, env=None,
               catch_exceptions=True, color=False, mix_stderr=False, **extra):
        """Invokes a command in an isolated environment.  The arguments are
        forwarded directly to the command line script, the `extra` keyword
        arguments are passed to the :meth:`~clickpkg.Command.main` function of
        the command.

        This returns a :class:`Result` object.

        .. versionadded:: 3.0
           The ``catch_exceptions`` parameter was added.

        .. versionchanged:: 3.0
           The result object now has an `exc_info` attribute with the
           traceback if available.

        .. versionadded:: 4.0
           The ``color`` parameter was added.

        :param cli: the command to invoke
        :param args: the arguments to invoke. It may be given as an iterable
                     or a string. When given as string it will be interpreted
                     as a Unix shell command. More details at
                     :func:`shlex.split`.
        :param input: the input data for `sys.stdin`.
        :param env: the environment overrides.
        :param catch_exceptions: Whether to catch any other exceptions than
                                 ``SystemExit``.
        :param extra: the keyword arguments to pass to :meth:`main`.
        :param color: whether the output should contain color codes. The
                      application can still override this explicitly.
        """
        exc_info = None
        with self.isolation(input=input, env=env, color=color) as outstreams:
            exception = None
            exit_code = 0

            if isinstance(args, string_types):
                args = shlex.split(args)

            try:
                prog_name = extra.pop("prog_name")
            except KeyError:
                prog_name = self.get_default_prog_name(cli)

            try:
                cli.main(args=args or (), prog_name=prog_name, **extra)
            except SystemExit as e:
                exc_info = sys.exc_info()
                exit_code = e.code
                if exit_code is None:
                    exit_code = 0

                if exit_code != 0:
                    exception = e

                if not isinstance(exit_code, int):
                    sys.stdout.write(str(exit_code))
                    sys.stdout.write('\n')
                    exit_code = 1

            except Exception as e:
                if not catch_exceptions:
                    raise
                exception = e
                exit_code = 1
                exc_info = sys.exc_info()
            finally:
                sys.stdout.flush()
                stdout = outstreams[0].getvalue()
                stderr = outstreams[1] and outstreams[1].getvalue()

        return Result(runner=self,
                      stdout_bytes=stdout,
                      stderr_bytes=stderr,
                      exit_code=exit_code,
                      exception=exception,
                      exc_info=exc_info)

    @contextlib.contextmanager
    def isolated_filesystem(self):
        """A context manager that creates a temporary folder and changes
        the current working directory to it for isolated filesystem tests.
        """
        cwd = os.getcwd()
        t = tempfile.mkdtemp()
        os.chdir(t)
        try:
            yield t
        finally:
            os.chdir(cwd)
            try:
                shutil.rmtree(t)
            except (OSError, IOError):
                pass
<|endoftext|>