<commit_before># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Creates a web page that shows the compatibility between packages as a grid.

For example:

------------------------------------------------------------
             |  absl-py | apache_beam | grpcio | tensorflow |
------------------------------------------------------------
absl-py      |   Good   |             |        |            |
apache_beam  |   Bad    |     Bad     |        |            |
grpcio       |   Good   |     Bad     |  Good  |            |
tensorflow   |   Good   |     Bad     |  Good  |    Good    |
------------------------------------------------------------
"""

import argparse
import datetime
import logging
import os
from typing import Any, Iterable, List, FrozenSet, Mapping
import webbrowser

import jinja2

from compatibility_lib import configs
from compatibility_lib import compatibility_checker
from compatibility_lib import compatibility_store
from compatibility_lib import dependency_highlighter
from compatibility_lib import deprecated_dep_finder
from compatibility_lib import package

logging.basicConfig()
logging.getLogger().setLevel(logging.INFO)

_JINJA2_ENVIRONMENT = jinja2.Environment(
    loader=jinja2.FileSystemLoader('.'), autoescape=jinja2.select_autoescape())

_DEFAULT_INSTALL_NAMES = configs.PKG_LIST

SELF_SUCCESS = {'status': 'SUCCESS', 'self': True}

class _ResultHolder(object):
    def __init__(
        self,
        package_to_results:
            Mapping[package.Package,
                    List[compatibility_store.CompatibilityResult]],
        pairwise_to_results:
            Mapping[FrozenSet[package.Package],
                    List[compatibility_store.CompatibilityResult]],
        package_with_dependency_info=None,
        checker=None,
        store=None
    ):
        self._package_to_results = package_to_results
        self._pairwise_to_results = pairwise_to_results
        self._package_with_dependency_info = package_with_dependency_info
        self.checker = checker
        self.store = store
        self.deprecated_deps = self.get_deprecated_deps()
        self.dependency_to_update = \
            self.get_dependencies_needed_to_update()

    def _is_py_version_incompatible(self, result):
        if result.status == compatibility_store.Status.INSTALL_ERROR:
            for version in [2, 3]:
                for pkg in result.packages:
                    major_version = result.python_major_version
                    name = pkg.install_name
                    unsupported = configs.PKG_PY_VERSION_NOT_SUPPORTED[version]

                    if major_version == version and name in unsupported:
                        return True
        return False

    def has_issues(self, p: package.Package) -> bool:
        """Returns true if the given package has any issues.

        Currently check for:
            1. Self compatibility
            2. Pairwise compatibility
        """
        # Get self result
        for package_2 in self._package_to_results.keys():
            p_and_package_2_result = self.get_result(p, package_2)
            # Don't report the package as having issues if it is purely the
            # result of a self-incompatibility of another package.
            package_2_self_conflict = False
            package_2_self_res = self.get_result(package_2, package_2)
            if SELF_SUCCESS not in package_2_self_res.get(
                    'self_compatibility_check'):
                package_2_self_conflict = True
            pair_res = p_and_package_2_result.get(
                'pairwise_compatibility_check')
            for result in pair_res:
                if not package_2_self_conflict and \
                                result['status'] != 'SUCCESS':
                    return True

        return False

    def get_deprecated_deps(self) -> Mapping[str, List]:
        """
        Returns if there are deprecated dependencies for a
        given package as well as the list of deprecated deps for a package.
        """
        finder = deprecated_dep_finder.DeprecatedDepFinder(
            py_version='3', checker=self.checker, store=self.store)
        deprecated_deps = list(finder.get_deprecated_deps())

        results = {}
        for item in deprecated_deps:
            (pkg_name, deps) = item[0]
            results[pkg_name] = deps

        return results

    def has_deprecated_deps(self, p: package.Package) -> bool:
        return bool(self.deprecated_deps[p.install_name])

    def get_dependencies_needed_to_update(self) -> Mapping[str, List]:
        """
        Returns a dict of package names together with the dependencies that
        they need to update.
        """
        highlighter = dependency_highlighter.DependencyHighlighter(
            py_version='3', checker=self.checker, store=self.store)

        result = highlighter.check_packages(
            packages=configs.PKG_LIST, max_workers=10)
        return result

    def needs_update(self, p: package.Package) -> bool:
        """Returns whether the dependencies for a given package needs to
        update."""
        return bool(self.dependency_to_update[p.install_name])

    def get_statistics(self, packages):
        """Get the total number of packages that has issues."""
        total_packages = len(configs.PKG_LIST)
        total_have_conflicts = 0
        total_have_deprecated_deps = 0
        total_needs_update = 0

        outdated_or_deprecated_pkgs = set()

        for pkg in packages:
            if self.has_issues(pkg):
                total_have_conflicts += 1
            if self.has_deprecated_deps(pkg):
                total_have_deprecated_deps += 1
                outdated_or_deprecated_pkgs.add(pkg)
            if self.needs_update(pkg):
                total_needs_update += 1
                outdated_or_deprecated_pkgs.add(pkg)

        total_success_deps = total_packages - len(outdated_or_deprecated_pkgs)

        return total_packages, total_have_conflicts,\
            total_have_deprecated_deps, total_needs_update,\
            total_success_deps

    def get_package_details(self, p: package.Package):
        """Return the dict of package check summary.

        {
            'self_conflict': True,
            'pairwise_conflict': ['package1', 'package2'],
            'latest_version': '0.1.0',
        }
        """
        # TODO: Will remove after a new build to update the bigquery data.
        pkg_name = p.install_name
        if p.install_name == 'apache-beam[gcp]':
            pkg_name = 'google-cloud-dataflow'
        latest_version = self._package_with_dependency_info[
            pkg_name][pkg_name]['latest_version']
        pairwise_conflict = []

        # Initialize the values
        result = {
            'self_conflict': False,
            'pairwise_conflict': pairwise_conflict,
            'latest_version': latest_version,
        }

        for pair_pkg in configs.PKG_LIST:
            check_result = self.get_result(p, package.Package(pair_pkg))
            # Get self compatibility status
            # TODO: Will remove after a new build to update the bigquery data.
            if pair_pkg == 'apache-beam[gcp]':
                pair_pkg = 'google-cloud-dataflow'
            if pair_pkg == pkg_name:
                result['self_conflict'] = True \
                    if check_result['status_type'] != 'self-success' else False
            # Get pairwise compatibility status
            else:
                if check_result['status_type'] != 'pairwise-success' \
                        and self.has_issues(p):
                    pairwise_conflict.append(pair_pkg)

        result['pairwise_conflict'] = pairwise_conflict

        return result

    def get_result(self,
                   package_1: package.Package,
                   package_2: package.Package) -> Mapping[str, Any]:
        """Returns the installation result of two packages.

        Args:
            package_1: One of the two packages to check installation
                compatibility with.
            package_2: One of the two packages to check installation
                compatibility with.

        Returns:
            The results of installing the two packages together, as a dict:

            {
                'status': <a compatibility_store.Status.value e.g. "SUCCESS">
                'self': <a bool indication whether the result is due to an
                         issue from within a single given package>
                'details': <a str representing the reason for any failure.
                            May be None.>
            }
        """
        self_result = []
        pair_result = []
        status_type = 'self-success'

        if (not self._package_to_results[package_1] or
                not self._package_to_results[package_2]):
            self_result.append(
                {
                    'status': compatibility_store.Status.UNKNOWN.name,
                    'self': True,
                }
            )
            status_type = 'self-unknown'

        package_results = (
                self._package_to_results[package_1] +
                self._package_to_results[package_2])

        for pr in package_results:
            if not self._is_py_version_incompatible(pr) and \
                            pr.status != compatibility_store.Status.SUCCESS:
                self_result.append(
                    {
                        'status': pr.status.value,
                        'self': True,
                        'details': pr.details
                    }
                )
                status_type = 'self-' + pr.status.value.lower()

        if package_1 == package_2:
            if not self_result:
                self_result.append(
                    {
                        'status': compatibility_store.Status.SUCCESS.name,
                        'self': True,
                    }
                )
        else:
            pairwise_results = self._pairwise_to_results[
                frozenset([package_1, package_2])]
            if not pairwise_results:
                pair_result.append(
                    {
                        'status': compatibility_store.Status.UNKNOWN.name,
                        'self': False,
                    }
                )
                status_type = 'pairwise-unknown'
            for pr in pairwise_results:
                if not self._is_py_version_incompatible(pr) and \
                            pr.status != compatibility_store.Status.SUCCESS:
                    pair_result.append(
                        {
                            'status': pr.status.value,
                            'self': False,
                            'details': pr.details
                        }
                    )
                    status_type = 'pairwise-' + pr.status.value.lower()

            if not pair_result:
                pair_result.append(
                    {
                        'status': compatibility_store.Status.SUCCESS.name,
                        'self': False,
                    }
                )
                if status_type is 'self-success':
                    status_type = 'pairwise-success'

        result = {
            'status_type': status_type,
            'self_compatibility_check': self_result,
            'pairwise_compatibility_check': pair_result
        }

        return result

class DashboardBuilder():
    """Build a web page that shows package compatibility status."""

    def __init__(self, packages: Iterable[package.Package],
                 results: _ResultHolder):
        self._packages = packages
        self._results = results

    def build_dashboard(self, template_name) -> str:
        """Returns a web page compatibility grid given a list of packages."""
        current_timestamp = datetime.datetime.now().strftime(
            '%Y-%m-%d %H:%M:%S')
        template = _JINJA2_ENVIRONMENT.get_template(template_name)
        return template.render(
            packages=self._packages,
            results=self._results,
            current_timestamp=current_timestamp)

def main():
    parser = argparse.ArgumentParser(
        description='Display a grid show the dependency compatibility ' +
                    'between Python packages')
    parser.add_argument('--packages', nargs='+',
                        default=_DEFAULT_INSTALL_NAMES,
                        help='the packages to display compatibility ' +
                             'information for')
    parser.add_argument(
        '--browser',
        action='store_true',
        default=False,
        help='display the grid in a browser tab')

    args = parser.parse_args()

    checker = compatibility_checker.CompatibilityChecker()
    store = compatibility_store.CompatibilityStore()

    packages = [
        package.Package(install_name) for install_name in args.packages]
    logging.info("Getting self compatibility results...")
    package_to_results = store.get_self_compatibilities(packages)
    logging.info("Getting pairwise compatibility results...")
    pairwise_to_results = store.get_compatibility_combinations(packages)

    package_with_dependency_info = {}
    for pkg in configs.PKG_LIST:
        # TODO: Will remove after a new build to update the bigquery data.
        if pkg == 'apache-beam[gcp]':
            pkg = 'google-cloud-dataflow'
        dep_info = store.get_dependency_info(pkg)
        package_with_dependency_info[pkg] = dep_info

    results = _ResultHolder(
        package_to_results,
        pairwise_to_results,
        package_with_dependency_info,
        checker,
        store)

    dashboard_builder = DashboardBuilder(packages, results)

    # Build the pairwise grid dashboard
    logging.info('Starting build the grid...')
    grid_html = dashboard_builder.build_dashboard(
        'dashboard/grid-template.html')
    grid_path = os.path.dirname(os.path.abspath(__file__)) + '/grid.html'
    with open(grid_path, 'wt') as f:
        f.write(grid_html)

    # Build the dashboard main page
    logging.info('Starting build the main dashboard...')
    main_html = dashboard_builder.build_dashboard(
        'dashboard/main-template.html')

    main_path = os.path.dirname(os.path.abspath(__file__)) + '/index.html'
    with open(main_path, 'wt') as f:
        f.write(main_html)

    if args.browser:
        webbrowser.open_new_tab('file://' + main_path)

if __name__ == '__main__':
    main()
<commit_msg>Remove the work around for renaming (#156)<commit_after># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Creates a web page that shows the compatibility between packages as a grid.

For example:

------------------------------------------------------------
             |  absl-py | apache_beam | grpcio | tensorflow |
------------------------------------------------------------
absl-py      |   Good   |             |        |            |
apache_beam  |   Bad    |     Bad     |        |            |
grpcio       |   Good   |     Bad     |  Good  |            |
tensorflow   |   Good   |     Bad     |  Good  |    Good    |
------------------------------------------------------------
"""

import argparse
import datetime
import logging
import os
from typing import Any, Iterable, List, FrozenSet, Mapping
import webbrowser

import jinja2

from compatibility_lib import configs
from compatibility_lib import compatibility_checker
from compatibility_lib import compatibility_store
from compatibility_lib import dependency_highlighter
from compatibility_lib import deprecated_dep_finder
from compatibility_lib import package

logging.basicConfig()
logging.getLogger().setLevel(logging.INFO)

_JINJA2_ENVIRONMENT = jinja2.Environment(
    loader=jinja2.FileSystemLoader('.'), autoescape=jinja2.select_autoescape())

_DEFAULT_INSTALL_NAMES = configs.PKG_LIST

SELF_SUCCESS = {'status': 'SUCCESS', 'self': True}

class _ResultHolder(object):
    def __init__(
        self,
        package_to_results:
            Mapping[package.Package,
                    List[compatibility_store.CompatibilityResult]],
        pairwise_to_results:
            Mapping[FrozenSet[package.Package],
                    List[compatibility_store.CompatibilityResult]],
        package_with_dependency_info=None,
        checker=None,
        store=None
    ):
        self._package_to_results = package_to_results
        self._pairwise_to_results = pairwise_to_results
        self._package_with_dependency_info = package_with_dependency_info
        self.checker = checker
        self.store = store
        self.deprecated_deps = self.get_deprecated_deps()
        self.dependency_to_update = \
            self.get_dependencies_needed_to_update()

    def _is_py_version_incompatible(self, result):
        if result.status == compatibility_store.Status.INSTALL_ERROR:
            for version in [2, 3]:
                for pkg in result.packages:
                    major_version = result.python_major_version
                    name = pkg.install_name
                    unsupported = configs.PKG_PY_VERSION_NOT_SUPPORTED[version]

                    if major_version == version and name in unsupported:
                        return True
        return False

    def has_issues(self, p: package.Package) -> bool:
        """Returns true if the given package has any issues.

        Currently check for:
            1. Self compatibility
            2. Pairwise compatibility
        """
        # Get self result
        for package_2 in self._package_to_results.keys():
            p_and_package_2_result = self.get_result(p, package_2)
            # Don't report the package as having issues if it is purely the
            # result of a self-incompatibility of another package.
            package_2_self_conflict = False
            package_2_self_res = self.get_result(package_2, package_2)
            if SELF_SUCCESS not in package_2_self_res.get(
                    'self_compatibility_check'):
                package_2_self_conflict = True
            pair_res = p_and_package_2_result.get(
                'pairwise_compatibility_check')
            for result in pair_res:
                if not package_2_self_conflict and \
                                result['status'] != 'SUCCESS':
                    return True

        return False

    def get_deprecated_deps(self) -> Mapping[str, List]:
        """
        Returns if there are deprecated dependencies for a
        given package as well as the list of deprecated deps for a package.
        """
        finder = deprecated_dep_finder.DeprecatedDepFinder(
            py_version='3', checker=self.checker, store=self.store)
        deprecated_deps = list(finder.get_deprecated_deps())

        results = {}
        for item in deprecated_deps:
            (pkg_name, deps) = item[0]
            results[pkg_name] = deps

        return results

    def has_deprecated_deps(self, p: package.Package) -> bool:
        return bool(self.deprecated_deps[p.install_name])

    def get_dependencies_needed_to_update(self) -> Mapping[str, List]:
        """
        Returns a dict of package names together with the dependencies that
        they need to update.
        """
        highlighter = dependency_highlighter.DependencyHighlighter(
            py_version='3', checker=self.checker, store=self.store)

        result = highlighter.check_packages(
            packages=configs.PKG_LIST, max_workers=10)
        return result

    def needs_update(self, p: package.Package) -> bool:
        """Returns whether the dependencies for a given package needs to
        update."""
        return bool(self.dependency_to_update[p.install_name])

    def get_statistics(self, packages):
        """Get the total number of packages that has issues."""
        total_packages = len(configs.PKG_LIST)
        total_have_conflicts = 0
        total_have_deprecated_deps = 0
        total_needs_update = 0

        outdated_or_deprecated_pkgs = set()

        for pkg in packages:
            if self.has_issues(pkg):
                total_have_conflicts += 1
            if self.has_deprecated_deps(pkg):
                total_have_deprecated_deps += 1
                outdated_or_deprecated_pkgs.add(pkg)
            if self.needs_update(pkg):
                total_needs_update += 1
                outdated_or_deprecated_pkgs.add(pkg)

        total_success_deps = total_packages - len(outdated_or_deprecated_pkgs)

        return total_packages, total_have_conflicts,\
            total_have_deprecated_deps, total_needs_update,\
            total_success_deps

    def get_package_details(self, p: package.Package):
        """Return the dict of package check summary.

        {
            'self_conflict': True,
            'pairwise_conflict': ['package1', 'package2'],
            'latest_version': '0.1.0',
        }
        """
        # The package being checked will appear in the dep list, but for
        # apache-beam[gcp], the package in dep list will just be apache-beam.
        self_dep_name = p.install_name
        if '[' in p.install_name:
            self_dep_name = p.install_name.split('[')[0]

        latest_version = self._package_with_dependency_info[
            p.install_name][self_dep_name]['latest_version']
        pairwise_conflict = []

        # Initialize the values
        result = {
            'self_conflict': False,
            'pairwise_conflict': pairwise_conflict,
            'latest_version': latest_version,
        }

        for pair_pkg in configs.PKG_LIST:
            check_result = self.get_result(p, package.Package(pair_pkg))
            # Get self compatibility status
            if pair_pkg == p.install_name:
                result['self_conflict'] = True \
                    if check_result['status_type'] != 'self-success' else False
            # Get pairwise compatibility status
            else:
                if check_result['status_type'] != 'pairwise-success' \
                        and self.has_issues(p):
                    pairwise_conflict.append(pair_pkg)

        result['pairwise_conflict'] = pairwise_conflict

        return result

    def get_result(self,
                   package_1: package.Package,
                   package_2: package.Package) -> Mapping[str, Any]:
        """Returns the installation result of two packages.

        Args:
            package_1: One of the two packages to check installation
                compatibility with.
            package_2: One of the two packages to check installation
                compatibility with.

        Returns:
            The results of installing the two packages together, as a dict:

            {
                'status': <a compatibility_store.Status.value e.g. "SUCCESS">
                'self': <a bool indication whether the result is due to an
                         issue from within a single given package>
                'details': <a str representing the reason for any failure.
                            May be None.>
            }
        """
        self_result = []
        pair_result = []
        status_type = 'self-success'

        if (not self._package_to_results[package_1] or
                not self._package_to_results[package_2]):
            self_result.append(
                {
                    'status': compatibility_store.Status.UNKNOWN.name,
                    'self': True,
                }
            )
            status_type = 'self-unknown'

        package_results = (
                self._package_to_results[package_1] +
                self._package_to_results[package_2])

        for pr in package_results:
            if not self._is_py_version_incompatible(pr) and \
                            pr.status != compatibility_store.Status.SUCCESS:
                self_result.append(
                    {
                        'status': pr.status.value,
                        'self': True,
                        'details': pr.details
                    }
                )
                status_type = 'self-' + pr.status.value.lower()

        if package_1 == package_2:
            if not self_result:
                self_result.append(
                    {
                        'status': compatibility_store.Status.SUCCESS.name,
                        'self': True,
                    }
                )
        else:
            pairwise_results = self._pairwise_to_results[
                frozenset([package_1, package_2])]
            if not pairwise_results:
                pair_result.append(
                    {
                        'status': compatibility_store.Status.UNKNOWN.name,
                        'self': False,
                    }
                )
                status_type = 'pairwise-unknown'
            for pr in pairwise_results:
                if not self._is_py_version_incompatible(pr) and \
                            pr.status != compatibility_store.Status.SUCCESS:
                    pair_result.append(
                        {
                            'status': pr.status.value,
                            'self': False,
                            'details': pr.details
                        }
                    )
                    status_type = 'pairwise-' + pr.status.value.lower()

            if not pair_result:
                pair_result.append(
                    {
                        'status': compatibility_store.Status.SUCCESS.name,
                        'self': False,
                    }
                )
                if status_type is 'self-success':
                    status_type = 'pairwise-success'

        result = {
            'status_type': status_type,
            'self_compatibility_check': self_result,
            'pairwise_compatibility_check': pair_result
        }

        return result

class DashboardBuilder():
    """Build a web page that shows package compatibility status."""

    def __init__(self, packages: Iterable[package.Package],
                 results: _ResultHolder):
        self._packages = packages
        self._results = results

    def build_dashboard(self, template_name) -> str:
        """Returns a web page compatibility grid given a list of packages."""
        current_timestamp = datetime.datetime.now().strftime(
            '%Y-%m-%d %H:%M:%S')
        template = _JINJA2_ENVIRONMENT.get_template(template_name)
        return template.render(
            packages=self._packages,
            results=self._results,
            current_timestamp=current_timestamp)

def main():
    parser = argparse.ArgumentParser(
        description='Display a grid show the dependency compatibility ' +
                    'between Python packages')
    parser.add_argument('--packages', nargs='+',
                        default=_DEFAULT_INSTALL_NAMES,
                        help='the packages to display compatibility ' +
                             'information for')
    parser.add_argument(
        '--browser',
        action='store_true',
        default=False,
        help='display the grid in a browser tab')

    args = parser.parse_args()

    checker = compatibility_checker.CompatibilityChecker()
    store = compatibility_store.CompatibilityStore()

    packages = [
        package.Package(install_name) for install_name in args.packages]
    logging.info("Getting self compatibility results...")
    package_to_results = store.get_self_compatibilities(packages)
    logging.info("Getting pairwise compatibility results...")
    pairwise_to_results = store.get_compatibility_combinations(packages)

    package_with_dependency_info = {}
    for pkg in configs.PKG_LIST:
        dep_info = store.get_dependency_info(pkg)
        package_with_dependency_info[pkg] = dep_info

    results = _ResultHolder(
        package_to_results,
        pairwise_to_results,
        package_with_dependency_info,
        checker,
        store)

    dashboard_builder = DashboardBuilder(packages, results)

    # Build the pairwise grid dashboard
    logging.info('Starting build the grid...')
    grid_html = dashboard_builder.build_dashboard(
        'dashboard/grid-template.html')
    grid_path = os.path.dirname(os.path.abspath(__file__)) + '/grid.html'
    with open(grid_path, 'wt') as f:
        f.write(grid_html)

    # Build the dashboard main page
    logging.info('Starting build the main dashboard...')
    main_html = dashboard_builder.build_dashboard(
        'dashboard/main-template.html')

    main_path = os.path.dirname(os.path.abspath(__file__)) + '/index.html'
    with open(main_path, 'wt') as f:
        f.write(main_html)

    if args.browser:
        webbrowser.open_new_tab('file://' + main_path)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import contextlib
import functools
import os
import random
import shutil
import tempfile

import numpy as np
import torch
from PIL import Image
from torchvision import io

import __main__  # noqa: 401

IN_CIRCLE_CI = os.getenv("CIRCLECI", False) == "true"
IN_RE_WORKER = os.environ.get("INSIDE_RE_WORKER") is not None
IN_FBCODE = os.environ.get("IN_FBCODE_TORCHVISION") == "1"
CUDA_NOT_AVAILABLE_MSG = "CUDA device not available"
CIRCLECI_GPU_NO_CUDA_MSG = "We're in a CircleCI GPU machine, and this test doesn't need cuda."

@contextlib.contextmanager
def get_tmp_dir(src=None, **kwargs):
    tmp_dir = tempfile.mkdtemp(**kwargs)
    if src is not None:
        os.rmdir(tmp_dir)
        shutil.copytree(src, tmp_dir)
    try:
        yield tmp_dir
    finally:
        shutil.rmtree(tmp_dir)

def set_rng_seed(seed):
    torch.manual_seed(seed)
    random.seed(seed)

class MapNestedTensorObjectImpl:
    def __init__(self, tensor_map_fn):
        self.tensor_map_fn = tensor_map_fn

    def __call__(self, object):
        if isinstance(object, torch.Tensor):
            return self.tensor_map_fn(object)

        elif isinstance(object, dict):
            mapped_dict = {}
            for key, value in object.items():
                mapped_dict[self(key)] = self(value)
            return mapped_dict

        elif isinstance(object, (list, tuple)):
            mapped_iter = []
            for iter in object:
                mapped_iter.append(self(iter))
            return mapped_iter if not isinstance(object, tuple) else tuple(mapped_iter)

        else:
            return object

def map_nested_tensor_object(object, tensor_map_fn):
    impl = MapNestedTensorObjectImpl(tensor_map_fn)
    return impl(object)

def is_iterable(obj):
    try:
        iter(obj)
        return True
    except TypeError:
        return False

@contextlib.contextmanager
def freeze_rng_state():
    rng_state = torch.get_rng_state()
    if torch.cuda.is_available():
        cuda_rng_state = torch.cuda.get_rng_state()
    yield
    if torch.cuda.is_available():
        torch.cuda.set_rng_state(cuda_rng_state)
    torch.set_rng_state(rng_state)

def cycle_over(objs):
    for idx, obj1 in enumerate(objs):
        for obj2 in objs[:idx] + objs[idx + 1 :]:
            yield obj1, obj2

def int_dtypes():
    return (torch.uint8, torch.int8, torch.int16, torch.int32, torch.int64)

def float_dtypes():
    return (torch.float32, torch.float64)

@contextlib.contextmanager
def disable_console_output():
    with contextlib.ExitStack() as stack, open(os.devnull, "w") as devnull:
        stack.enter_context(contextlib.redirect_stdout(devnull))
        stack.enter_context(contextlib.redirect_stderr(devnull))
        yield

def cpu_and_gpu():
    import pytest  # noqa

    return ("cpu", pytest.param("cuda", marks=pytest.mark.needs_cuda))

def needs_cuda(test_func):
    import pytest  # noqa

    return pytest.mark.needs_cuda(test_func)

def _create_data(height=3, width=3, channels=3, device="cpu"):
    # TODO: When all relevant tests are ported to pytest, turn this into a module-level fixture
    tensor = torch.randint(0, 256, (channels, height, width), dtype=torch.uint8, device=device)
    data = tensor.permute(1, 2, 0).contiguous().cpu().numpy()
    mode = "RGB"
    if channels == 1:
        mode = "L"
        data = data[..., 0]
    pil_img = Image.fromarray(data, mode=mode)
    return tensor, pil_img

def _create_data_batch(height=3, width=3, channels=3, num_samples=4, device="cpu"):
    # TODO: When all relevant tests are ported to pytest, turn this into a module-level fixture
    batch_tensor = torch.randint(0, 256, (num_samples, channels, height, width), dtype=torch.uint8, device=device)
    return batch_tensor

assert_equal = functools.partial(torch.testing.assert_close, rtol=0, atol=1e-6)

def get_list_of_videos(tmpdir, num_videos=5, sizes=None, fps=None):
    names = []
    for i in range(num_videos):
        if sizes is None:
            size = 5 * (i + 1)
        else:
            size = sizes[i]
        if fps is None:
            f = 5
        else:
            f = fps[i]
        data = torch.randint(0, 256, (size, 300, 400, 3), dtype=torch.uint8)
        name = os.path.join(tmpdir, f"{i}.mp4")
        names.append(name)
        io.write_video(name, data, fps=f)

    return names

def _assert_equal_tensor_to_pil(tensor, pil_image, msg=None):
    np_pil_image = np.array(pil_image)
    if np_pil_image.ndim == 2:
        np_pil_image = np_pil_image[:, :, None]
    pil_tensor = torch.as_tensor(np_pil_image.transpose((2, 0, 1)))
    if msg is None:
        msg = f"tensor:\n{tensor} \ndid not equal PIL tensor:\n{pil_tensor}"
    assert_equal(tensor.cpu(), pil_tensor, msg=msg)

def _assert_approx_equal_tensor_to_pil(
    tensor, pil_image, tol=1e-5, msg=None, agg_method="mean", allowed_percentage_diff=None
):
    # TODO: we could just merge this into _assert_equal_tensor_to_pil
    np_pil_image = np.array(pil_image)
    if np_pil_image.ndim == 2:
        np_pil_image = np_pil_image[:, :, None]
    pil_tensor = torch.as_tensor(np_pil_image.transpose((2, 0, 1))).to(tensor)

    if allowed_percentage_diff is not None:
        # Assert that less than a given %age of pixels are different
        assert (tensor != pil_tensor).to(torch.float).mean() <= allowed_percentage_diff

    # error value can be mean absolute error, max abs error
    # Convert to float to avoid underflow when computing absolute difference
    tensor = tensor.to(torch.float)
    pil_tensor = pil_tensor.to(torch.float)
    err = getattr(torch, agg_method)(torch.abs(tensor - pil_tensor)).item()
    assert err < tol, f"{err} vs {tol}"

def _test_fn_on_batch(batch_tensors, fn, scripted_fn_atol=1e-8, **fn_kwargs):
    transformed_batch = fn(batch_tensors, **fn_kwargs)
    for i in range(len(batch_tensors)):
        img_tensor = batch_tensors[i, ...]
        transformed_img = fn(img_tensor, **fn_kwargs)
        assert_equal(transformed_img, transformed_batch[i, ...])

    if scripted_fn_atol >= 0:
        scripted_fn = torch.jit.script(fn)
        # scriptable function test
        s_transformed_batch = scripted_fn(batch_tensors, **fn_kwargs)
        torch.testing.assert_close(transformed_batch, s_transformed_batch, rtol=1e-5, atol=scripted_fn_atol)
<commit_msg>revert changes to assert_equal (#5586)<commit_after>import contextlib
import functools
import os
import random
import shutil
import tempfile

import numpy as np
import torch
from PIL import Image
from torchvision import io

import __main__  # noqa: 401

IN_CIRCLE_CI = os.getenv("CIRCLECI", False) == "true"
IN_RE_WORKER = os.environ.get("INSIDE_RE_WORKER") is not None
IN_FBCODE = os.environ.get("IN_FBCODE_TORCHVISION") == "1"
CUDA_NOT_AVAILABLE_MSG = "CUDA device not available"
CIRCLECI_GPU_NO_CUDA_MSG = "We're in a CircleCI GPU machine, and this test doesn't need cuda."

@contextlib.contextmanager
def get_tmp_dir(src=None, **kwargs):
    tmp_dir = tempfile.mkdtemp(**kwargs)
    if src is not None:
        os.rmdir(tmp_dir)
        shutil.copytree(src, tmp_dir)
    try:
        yield tmp_dir
    finally:
        shutil.rmtree(tmp_dir)

def set_rng_seed(seed):
    torch.manual_seed(seed)
    random.seed(seed)

class MapNestedTensorObjectImpl:
    def __init__(self, tensor_map_fn):
        self.tensor_map_fn = tensor_map_fn

    def __call__(self, object):
        if isinstance(object, torch.Tensor):
            return self.tensor_map_fn(object)

        elif isinstance(object, dict):
            mapped_dict = {}
            for key, value in object.items():
                mapped_dict[self(key)] = self(value)
            return mapped_dict

        elif isinstance(object, (list, tuple)):
            mapped_iter = []
            for iter in object:
                mapped_iter.append(self(iter))
            return mapped_iter if not isinstance(object, tuple) else tuple(mapped_iter)

        else:
            return object

def map_nested_tensor_object(object, tensor_map_fn):
    impl = MapNestedTensorObjectImpl(tensor_map_fn)
    return impl(object)

def is_iterable(obj):
    try:
        iter(obj)
        return True
    except TypeError:
        return False

@contextlib.contextmanager
def freeze_rng_state():
    rng_state = torch.get_rng_state()
    if torch.cuda.is_available():
        cuda_rng_state = torch.cuda.get_rng_state()
    yield
    if torch.cuda.is_available():
        torch.cuda.set_rng_state(cuda_rng_state)
    torch.set_rng_state(rng_state)

def cycle_over(objs):
    for idx, obj1 in enumerate(objs):
        for obj2 in objs[:idx] + objs[idx + 1 :]:
            yield obj1, obj2

def int_dtypes():
    return (torch.uint8, torch.int8, torch.int16, torch.int32, torch.int64)

def float_dtypes():
    return (torch.float32, torch.float64)

@contextlib.contextmanager
def disable_console_output():
    with contextlib.ExitStack() as stack, open(os.devnull, "w") as devnull:
        stack.enter_context(contextlib.redirect_stdout(devnull))
        stack.enter_context(contextlib.redirect_stderr(devnull))
        yield

def cpu_and_gpu():
    import pytest  # noqa

    return ("cpu", pytest.param("cuda", marks=pytest.mark.needs_cuda))

def needs_cuda(test_func):
    import pytest  # noqa

    return pytest.mark.needs_cuda(test_func)

def _create_data(height=3, width=3, channels=3, device="cpu"):
    # TODO: When all relevant tests are ported to pytest, turn this into a module-level fixture
    tensor = torch.randint(0, 256, (channels, height, width), dtype=torch.uint8, device=device)
    data = tensor.permute(1, 2, 0).contiguous().cpu().numpy()
    mode = "RGB"
    if channels == 1:
        mode = "L"
        data = data[..., 0]
    pil_img = Image.fromarray(data, mode=mode)
    return tensor, pil_img

def _create_data_batch(height=3, width=3, channels=3, num_samples=4, device="cpu"):
    # TODO: When all relevant tests are ported to pytest, turn this into a module-level fixture
    batch_tensor = torch.randint(0, 256, (num_samples, channels, height, width), dtype=torch.uint8, device=device)
    return batch_tensor

assert_equal = functools.partial(torch.testing.assert_close, rtol=0, atol=0)

def get_list_of_videos(tmpdir, num_videos=5, sizes=None, fps=None):
    names = []
    for i in range(num_videos):
        if sizes is None:
            size = 5 * (i + 1)
        else:
            size = sizes[i]
        if fps is None:
            f = 5
        else:
            f = fps[i]
        data = torch.randint(0, 256, (size, 300, 400, 3), dtype=torch.uint8)
        name = os.path.join(tmpdir, f"{i}.mp4")
        names.append(name)
        io.write_video(name, data, fps=f)

    return names

def _assert_equal_tensor_to_pil(tensor, pil_image, msg=None):
    np_pil_image = np.array(pil_image)
    if np_pil_image.ndim == 2:
        np_pil_image = np_pil_image[:, :, None]
    pil_tensor = torch.as_tensor(np_pil_image.transpose((2, 0, 1)))
    if msg is None:
        msg = f"tensor:\n{tensor} \ndid not equal PIL tensor:\n{pil_tensor}"
    assert_equal(tensor.cpu(), pil_tensor, msg=msg)

def _assert_approx_equal_tensor_to_pil(
    tensor, pil_image, tol=1e-5, msg=None, agg_method="mean", allowed_percentage_diff=None
):
    # TODO: we could just merge this into _assert_equal_tensor_to_pil
    np_pil_image = np.array(pil_image)
    if np_pil_image.ndim == 2:
        np_pil_image = np_pil_image[:, :, None]
    pil_tensor = torch.as_tensor(np_pil_image.transpose((2, 0, 1))).to(tensor)

    if allowed_percentage_diff is not None:
        # Assert that less than a given %age of pixels are different
        assert (tensor != pil_tensor).to(torch.float).mean() <= allowed_percentage_diff

    # error value can be mean absolute error, max abs error
    # Convert to float to avoid underflow when computing absolute difference
    tensor = tensor.to(torch.float)
    pil_tensor = pil_tensor.to(torch.float)
    err = getattr(torch, agg_method)(torch.abs(tensor - pil_tensor)).item()
    assert err < tol, f"{err} vs {tol}"

def _test_fn_on_batch(batch_tensors, fn, scripted_fn_atol=1e-8, **fn_kwargs):
    transformed_batch = fn(batch_tensors, **fn_kwargs)
    for i in range(len(batch_tensors)):
        img_tensor = batch_tensors[i, ...]
        transformed_img = fn(img_tensor, **fn_kwargs)
        torch.testing.assert_close(transformed_img, transformed_batch[i, ...], rtol=0, atol=1e-6)

    if scripted_fn_atol >= 0:
        scripted_fn = torch.jit.script(fn)
        # scriptable function test
        s_transformed_batch = scripted_fn(batch_tensors, **fn_kwargs)
        torch.testing.assert_close(transformed_batch, s_transformed_batch, rtol=1e-5, atol=scripted_fn_atol)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# The MIT License
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
"""
tox-bootstrap
=============

A bootstrap to automatically install tox and dependencies on machines that may
not already have tox installed. This is useful when configuring a number of
Hudson quickly; only Python needs to be installed.

Getting started
---------------

::

    $ cd my_project/
    $ ls
    . .. src/ doc/ setup.py tox.ini
    $ curl http://pytox.googlecode.com/hg/toxbootstrap.py -O

Instead of running "tox", now you can just run "python toxbootstrap.py" which
will take care of installing tox (if not already installed into
``.tox/_toxinstall``)::

    $ python toxbootstrap.py

If you're using Hudson_, you may also do::

    import sys
    sys.path.insert(0, '.') # sometimes necessary :/
    import toxbootstrap
    toxbootstrap.cmdline() # also accepts argv list

.. _Hudson: http://hudson-ci.org/

ToDo
----

1. Detect tox in ``$PATH`` (eg: ``C:\Python26\Scripts`` or
   ``%APPDATA%\Python\Scripts``)

2. Gracefully ignore PyPI xmlrpc downtime errors when checking for new release.

"""

__version__ = '0.9.1.dev1'

import sys
import os
from os import path
import logging

from subprocess import Popen, PIPE, check_call, CalledProcessError

USETOXDEV=os.environ.get('USETOXDEV', False)
TENV='_toxinstall'

PY3 = sys.version_info[0] == 3

if PY3:
    from urllib.request import urlretrieve
    import xmlrpc.client as xmlrpclib
else:
    from urllib import urlretrieve
    import xmlrpclib

logging.basicConfig(level=logging.INFO)

# Last stable: 1.5.1
VIRTUALENVPY_URL = (
    'http://bitbucket.org/ianb/virtualenv/raw/eb94c9ebe0ba/virtualenv.py')

def run(cmd, shell=True):
    """Run the given command in shell"""
    logging.info('Running command: %s', cmd)
    check_call(cmd, shell=shell)

def crun(cmd, shell=True):
    """Run the given command and return its output"""
    logging.info('Running command (for output): %s', cmd)
    p = Popen(cmd, stdout=PIPE, shell=shell)
    stdout, stderr = p.communicate()
    return stdout

def wget(url):
    """Download the given file to current directory"""
    logging.info('Downloading %s', url)
    localpath = path.join(path.abspath(os.getcwd()), path.basename(url))
    urlretrieve(url, localpath)

def has_script(venv, name):
    """Check if the virtualenv has the given script

    Looks for bin/$name (unix) or Scripts/$name.exe (windows) in the virtualenv
    """
    if sys.platform == 'win32':
        return any([path.exists(path.join(venv, 'Scripts', name)),
                    path.exists(path.join(venv, 'Scripts', name + '.exe'))])
    else:
        return path.exists(path.join(venv, 'bin', name))

def activate_path(venv):
    """Return the full path to the script virtualenv directory"""
    if sys.platform == 'win32':
        p = path.abspath(path.join(venv, 'Scripts'))
    else:
        p = path.abspath(path.join(venv, 'bin'))
    assert path.exists(p), p
    os.environ['PATH'] = p + os.pathsep + os.environ['PATH']
    logging.info("added to PATH: %s", p)

def get_script_path(venv, name):
    """Return the full path to the script in virtualenv directory"""
    if sys.platform == 'win32':
        p = path.join(venv, 'Scripts', name)
        if not path.exists(p):
            p = path.join(venv, 'Scripts', name + '.exe')
    else:
        p = path.join(venv, 'bin', name)

    if not path.exists(p):
        raise NameError('cannot find a script named "%s"' % (name,))

    return p

def get_tox_version(venv):
    """Return the installed version of tox"""
    py = get_script_path(venv, 'python')
    s = 'import tox,sys; sys.stdout.write(str(tox.__version__))'
    if sys.version_info[:2] >= (2, 6):
        return crun('%s -s -c "%s"' % (py, s))
    else:
        return crun('%s -c "%s"' % (py, s))

def parse_simple_version(v):
    """A simplified version of pkg_resources.parse_version

    This method can only parse simple versions like the ones with a set of
    numbers separated by dots (eg: 1.2.3)
    """
    return [int(c) for c in v.split('.')]

def pypi_get_latest_version(pkgname):
    """Return the latest version of package from PyPI"""
    pypi = xmlrpclib.ServerProxy('http://pypi.python.org/pypi')
    versions = pypi.package_releases('tox')
    assert versions
    versions.sort(key=parse_simple_version, reverse=True)
    return versions[0]

def ensuredir(p):
    if not path.isdir(p):
        os.makedirs(p)

def cmdline(argv=None):
    currentdir = os.getcwd()
    #os.chdir(path.abspath(path.dirname(__file__)))
    ensuredir('.tox')
    os.chdir('.tox')

    os.environ['PATH'] = os.path.abspath(TENV) + os.path.pathsep + os.environ['PATH']
    # create virtual environment
    if not path.isdir(TENV) or not has_script(TENV, 'python') or \
        not has_script(TENV, 'pip'):
        # get virtualenv.py
        if not path.isfile('virtualenv.py'):
            wget(VIRTUALENVPY_URL)
        assert path.isfile('virtualenv.py')

        # XXX: we use --no-site-packages because: if tox is installed in global
        # site-packages, then pip will not install it locally. ideal fix for
        # this should be to first look for tox in the global scripts/ directory
        run('%s virtualenv.py --no-site-packages --distribute %s' %
                (sys.executable, TENV))

    assert has_script(TENV, 'python'), 'no python script'
    assert has_script(TENV, 'pip'), 'no pip script'
    activate_path(TENV)

    pip = get_script_path(TENV, 'pip')

    # install/upgrade tox itself
    if USETOXDEV:
        if 'PIP_DOWNLOAD_CACHE' in os.environ:
            cache = ""
        else:
            cache = "--download-cache=_download"
            ensuredir('_download')
        run('%s install -q -i http://pypi.testrun.org '
            '--upgrade %s tox' % (pip, cache))
    elif any([
        not has_script(TENV, 'tox'),
        get_tox_version(TENV) != pypi_get_latest_version('tox')]):
        run('%s install --upgrade --download-cache=_download tox' % (pip,))

    toxversion = get_tox_version(TENV)
    assert has_script(TENV, 'tox')
    tox_script = path.abspath(get_script_path(TENV, 'tox'))
    logging.info('tox is installed at %s version %s', tox_script, toxversion)

    virtualenv = get_script_path(TENV, 'virtualenv')
    venv_version = crun('%s --version' % (virtualenv,)).strip()
    logging.info('virtualenv at %s version %s', virtualenv, venv_version)

    # XXX: virtualenv 1.5 is broken; replace it
    if venv_version == '1.5':
        logging.info(
            'Replacing the unstable virtualenv-1.5 with the latest stable')
        run('%s uninstall -y virtualenv' % (pip,))
        run('%s install virtualenv!=1.5' % (pip,))

    # Now run the locally-installed tox
    os.chdir(currentdir)
    try:
        run([tox_script] + (argv or []), shell=False)
    except CalledProcessError:
        _, e, _ = sys.exc_info()
        logging.error('tox exited with error code %d', e.returncode)
        sys.exit(e.returncode)

if __name__ == '__main__':
    cmdline(sys.argv[1:])
<commit_msg>make toxbootstrap print version as well<commit_after>#!/usr/bin/env python
# The MIT License
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
"""
tox-bootstrap
=============

A bootstrap to automatically install tox and dependencies on machines that may
not already have tox installed. This is useful when configuring a number of
Hudson quickly; only Python needs to be installed.

Getting started
---------------

::

    $ cd my_project/
    $ ls
    . .. src/ doc/ setup.py tox.ini
    $ curl http://pytox.googlecode.com/hg/toxbootstrap.py -O

Instead of running "tox", now you can just run "python toxbootstrap.py" which
will take care of installing tox (if not already installed into
``.tox/_toxinstall``)::

    $ python toxbootstrap.py

If you're using Hudson_, you may also do::

    import sys
    sys.path.insert(0, '.') # sometimes necessary :/
    import toxbootstrap
    toxbootstrap.cmdline() # also accepts argv list

.. _Hudson: http://hudson-ci.org/

ToDo
----

1. Detect tox in ``$PATH`` (eg: ``C:\Python26\Scripts`` or
   ``%APPDATA%\Python\Scripts``)

2. Gracefully ignore PyPI xmlrpc downtime errors when checking for new release.

"""

__version__ = '0.9.1.dev1'

import sys
import os
from os import path
import logging

from subprocess import Popen, PIPE, check_call, CalledProcessError

USETOXDEV=os.environ.get('USETOXDEV', False)
TENV='_toxinstall'

PY3 = sys.version_info[0] == 3

if PY3:
    from urllib.request import urlretrieve
    import xmlrpc.client as xmlrpclib
else:
    from urllib import urlretrieve
    import xmlrpclib

logging.basicConfig(level=logging.INFO)

# Last stable: 1.5.1
VIRTUALENVPY_URL = (
    'http://bitbucket.org/ianb/virtualenv/raw/eb94c9ebe0ba/virtualenv.py')

def run(cmd, shell=True):
    """Run the given command in shell"""
    logging.info('Running command: %s', cmd)
    check_call(cmd, shell=shell)

def crun(cmd, shell=True):
    """Run the given command and return its output"""
    logging.info('Running command (for output): %s', cmd)
    p = Popen(cmd, stdout=PIPE, shell=shell)
    stdout, stderr = p.communicate()
    return stdout

def wget(url):
    """Download the given file to current directory"""
    logging.info('Downloading %s', url)
    localpath = path.join(path.abspath(os.getcwd()), path.basename(url))
    urlretrieve(url, localpath)

def has_script(venv, name):
    """Check if the virtualenv has the given script

    Looks for bin/$name (unix) or Scripts/$name.exe (windows) in the virtualenv
    """
    if sys.platform == 'win32':
        return any([path.exists(path.join(venv, 'Scripts', name)),
                    path.exists(path.join(venv, 'Scripts', name + '.exe'))])
    else:
        return path.exists(path.join(venv, 'bin', name))

def activate_path(venv):
    """Return the full path to the script virtualenv directory"""
    if sys.platform == 'win32':
        p = path.abspath(path.join(venv, 'Scripts'))
    else:
        p = path.abspath(path.join(venv, 'bin'))
    assert path.exists(p), p
    os.environ['PATH'] = p + os.pathsep + os.environ['PATH']
    logging.info("added to PATH: %s", p)

def get_script_path(venv, name):
    """Return the full path to the script in virtualenv directory"""
    if sys.platform == 'win32':
        p = path.join(venv, 'Scripts', name)
        if not path.exists(p):
            p = path.join(venv, 'Scripts', name + '.exe')
    else:
        p = path.join(venv, 'bin', name)

    if not path.exists(p):
        raise NameError('cannot find a script named "%s"' % (name,))

    return p

def get_tox_version(venv):
    """Return the installed version of tox"""
    py = get_script_path(venv, 'python')
    s = 'import tox,sys; sys.stdout.write(str(tox.__version__))'
    if sys.version_info[:2] >= (2, 6):
        return crun('%s -s -c "%s"' % (py, s))
    else:
        return crun('%s -c "%s"' % (py, s))

def parse_simple_version(v):
    """A simplified version of pkg_resources.parse_version

    This method can only parse simple versions like the ones with a set of
    numbers separated by dots (eg: 1.2.3)
    """
    return [int(c) for c in v.split('.')]

def pypi_get_latest_version(pkgname):
    """Return the latest version of package from PyPI"""
    pypi = xmlrpclib.ServerProxy('http://pypi.python.org/pypi')
    versions = pypi.package_releases('tox')
    assert versions
    versions.sort(key=parse_simple_version, reverse=True)
    return versions[0]

def ensuredir(p):
    if not path.isdir(p):
        os.makedirs(p)

def cmdline(argv=None):
    logging.info('toxbootstrap version %s', __version__)
    currentdir = os.getcwd()
    #os.chdir(path.abspath(path.dirname(__file__)))
    ensuredir('.tox')
    os.chdir('.tox')

    os.environ['PATH'] = os.path.abspath(TENV) + os.path.pathsep + os.environ['PATH']
    # create virtual environment
    if not path.isdir(TENV) or not has_script(TENV, 'python') or \
        not has_script(TENV, 'pip'):
        # get virtualenv.py
        if not path.isfile('virtualenv.py'):
            wget(VIRTUALENVPY_URL)
        assert path.isfile('virtualenv.py')

        # XXX: we use --no-site-packages because: if tox is installed in global
        # site-packages, then pip will not install it locally. ideal fix for
        # this should be to first look for tox in the global scripts/ directory
        run('%s virtualenv.py --no-site-packages --distribute %s' %
                (sys.executable, TENV))

    assert has_script(TENV, 'python'), 'no python script'
    assert has_script(TENV, 'pip'), 'no pip script'
    activate_path(TENV)

    pip = get_script_path(TENV, 'pip')

    # install/upgrade tox itself
    if USETOXDEV:
        if 'PIP_DOWNLOAD_CACHE' in os.environ:
            cache = ""
        else:
            cache = "--download-cache=_download"
            ensuredir('_download')
        run('%s install -q -i http://pypi.testrun.org '
            '--upgrade %s tox' % (pip, cache))
    elif any([
        not has_script(TENV, 'tox'),
        get_tox_version(TENV) != pypi_get_latest_version('tox')]):
        run('%s install --upgrade --download-cache=_download tox' % (pip,))

    toxversion = get_tox_version(TENV)
    assert has_script(TENV, 'tox')
    tox_script = path.abspath(get_script_path(TENV, 'tox'))
    logging.info('tox is installed at %s version %s', tox_script, toxversion)

    virtualenv = get_script_path(TENV, 'virtualenv')
    venv_version = crun('%s --version' % (virtualenv,)).strip()
    logging.info('virtualenv at %s version %s', virtualenv, venv_version)

    # XXX: virtualenv 1.5 is broken; replace it
    if venv_version == '1.5':
        logging.info(
            'Replacing the unstable virtualenv-1.5 with the latest stable')
        run('%s uninstall -y virtualenv' % (pip,))
        run('%s install virtualenv!=1.5' % (pip,))

    # Now run the locally-installed tox
    os.chdir(currentdir)
    try:
        run([tox_script] + (argv or []), shell=False)
    except CalledProcessError:
        _, e, _ = sys.exc_info()
        logging.error('tox exited with error code %d', e.returncode)
        sys.exit(e.returncode)

if __name__ == '__main__':
    cmdline(sys.argv[1:])
<|endoftext|>


<commit_before># coding: utf-8
from test.util import build_grab, exclude_transport
from test.util import BaseGrabTestCase

from grab.error import GrabInternalError, GrabCouldNotResolveHostError

class GrabRequestTestCase(BaseGrabTestCase):
    def setUp(self):
        self.server.reset()

    def test_get_method(self):
        g = build_grab()
        g.go(self.server.get_url())
        self.assertEquals('GET', self.server.request['method'])

    def test_delete_method(self):
        g = build_grab()
        g.setup(method='delete')
        g.go(self.server.get_url())
        self.assertEquals('DELETE', self.server.request['method'])

    def test_put_method(self):
        g = build_grab()
        g.setup(method='put', post=b'abc')
        g.go(self.server.get_url())
        self.assertEquals('PUT', self.server.request['method'])
        self.assertEquals('3', self.server.request['headers']['Content-Length'])

    def test_head_with_invalid_bytes(self):
        def callback(server):
            server.set_status(200)
            server.add_header('Hello-Bug', b'start\xa0end')
            server.write('')
            server.finish()

        self.server.response['callback'] = callback
        g = build_grab()
        g.go(self.server.get_url())

    @exclude_transport('urllib3')
    def test_redirect_with_invalid_byte(self):
        url = self.server.get_url()
        invalid_url = b'http://\xa0' + url.encode('ascii')

        def callback(server):
            server.set_status(301)
            server.add_header('Location', invalid_url)
            server.write('')
            server.finish()

        self.server.response['callback'] = callback
        g = build_grab()
        self.assertRaises((GrabInternalError, GrabCouldNotResolveHostError),
                          g.go, self.server.get_url())

    def test_options_method(self):
        g = build_grab()
        g.setup(method='options', post=b'abc')
        g.go(self.server.get_url())
        self.assertEquals('OPTIONS', self.server.request['method'])
        self.assertEquals('3', self.server.request['headers']['Content-Length'])

        g = build_grab()
        g.setup(method='options')
        g.go(self.server.get_url())
        self.assertEquals('OPTIONS', self.server.request['method'])
        self.assertTrue('Content-Length' not in self.server.request['headers'])

    def test_request_headers(self):
        g = build_grab(debug=True)
        g.setup(headers={'Foo': 'Bar'})
        g.go(self.server.get_url())
        self.assertEqual('Bar', g.request_headers['foo'])
<commit_msg>Fix bug in test.grab_request test<commit_after># coding: utf-8
from test.util import build_grab, exclude_transport
from test.util import BaseGrabTestCase

from grab.error import (GrabInternalError, GrabCouldNotResolveHostError,
                        GrabTimeoutError)

class GrabRequestTestCase(BaseGrabTestCase):
    def setUp(self):
        self.server.reset()

    def test_get_method(self):
        g = build_grab()
        g.go(self.server.get_url())
        self.assertEquals('GET', self.server.request['method'])

    def test_delete_method(self):
        g = build_grab()
        g.setup(method='delete')
        g.go(self.server.get_url())
        self.assertEquals('DELETE', self.server.request['method'])

    def test_put_method(self):
        g = build_grab()
        g.setup(method='put', post=b'abc')
        g.go(self.server.get_url())
        self.assertEquals('PUT', self.server.request['method'])
        self.assertEquals('3', self.server.request['headers']['Content-Length'])

    def test_head_with_invalid_bytes(self):
        def callback(server):
            server.set_status(200)
            server.add_header('Hello-Bug', b'start\xa0end')
            server.write('')
            server.finish()

        self.server.response['callback'] = callback
        g = build_grab()
        g.go(self.server.get_url())

    @exclude_transport('urllib3')
    def test_redirect_with_invalid_byte(self):
        url = self.server.get_url()
        invalid_url = b'http://\xa0' + url.encode('ascii')

        def callback(server):
            server.set_status(301)
            server.add_header('Location', invalid_url)
            server.write('')
            server.finish()

        self.server.response['callback'] = callback
        g = build_grab()
        # GrabTimeoutError raised when tests are being runned on computer
        # without access to the internet (no DNS service available)
        self.assertRaises((GrabInternalError, GrabCouldNotResolveHostError,
                           GrabTimeoutError),
                          g.go, self.server.get_url())

    def test_options_method(self):
        g = build_grab()
        g.setup(method='options', post=b'abc')
        g.go(self.server.get_url())
        self.assertEquals('OPTIONS', self.server.request['method'])
        self.assertEquals('3', self.server.request['headers']['Content-Length'])

        g = build_grab()
        g.setup(method='options')
        g.go(self.server.get_url())
        self.assertEquals('OPTIONS', self.server.request['method'])
        self.assertTrue('Content-Length' not in self.server.request['headers'])

    def test_request_headers(self):
        g = build_grab(debug=True)
        g.setup(headers={'Foo': 'Bar'})
        g.go(self.server.get_url())
        self.assertEqual('Bar', g.request_headers['foo'])
<|endoftext|>


<commit_before>class TimetableGenerator:

	def __init__(self):
		self.generated = []
		self.generated2 = []
		self.total_combinations = 0

	def store_timetable(self, tt):
		self.total_combinations += 1
		tt.heuristic = tt.total_time()
		tt.heuristic2 = tt.total_time2()

		if len(self.generated) <= 99:
			self.generated.append(tt)
			self.generated2.append(tt)
		else:
			if tt.heuristic < self.generated[99].heuristic:  # tt is better
				self.generated.pop(99)
				self.generated.append(tt)
				self.generated.sort(key=lambda tt: tt.heuristic)

			if tt.heuristic2 < self.generated2[99].heuristic2:  # tt is better
				self.generated2.pop(99)
				self.generated2.append(tt)
				self.generated2.sort(key=lambda tt: tt.heuristic2)

	def generate_timetables(self, lesson_blocks):
		self.generate(Timetable(), lesson_blocks)

		for i in xrange(1, len(self.generated)+1):
			self.generated[len(self.generated) - i].score += i

		for i in xrange(1, len(self.generated2)+1):
			self.generated2[len(self.generated2) - i].score += i

		self.generated.sort(key=lambda tt: tt.score)

	def generate(self, timetable, lesson_blocks):
		if not lesson_blocks:
			self.store_timetable(timetable)
		else:
			next_lesson_block = lesson_blocks[0]
			for shift in next_lesson_block.shifts:
				#if timetable.supports(shift):
				self.generate(timetable.append_shift(shift), lesson_blocks[1:])

class Timetable:

	def __init__(self):
		self.lessons = []
		self.score = 0

	def append_shift(self, shift):
		new_timetable = Timetable()
		new_timetable.lessons.extend(self.lessons)
		new_timetable.lessons.extend(shift.slots)
		return new_timetable

	def supports(self, shift):
		for slot in shift.slots:
			for existing in self.lessons:
				if (slot.overlaps_with(existing)):
					return False
		return True

	#def is_feasible(self):
	#	for slot in self.lessons:
	#		for other in self.lessons:
	#			if (slot is not other) and (slot.overlaps_with(other)):
	#				return False
	#	return True

	# heuristics for selecting timetables
	def total_time2(self):
		result = 0
		for weekday in range(Weekday.MONDAY, Weekday.SUNDAY):
			daily_lessons = [slot for slot in self.lessons if slot.day == weekday]
			if daily_lessons:
				earliest_start = min([slot.start.minutes for slot in daily_lessons])
				latest_end = max([slot.end.minutes for slot in daily_lessons])
				interval = latest_end - earliest_start
		return result

	def total_time(self):
		result = 0
		for weekday in range(Weekday.MONDAY, Weekday.SUNDAY):
			daily_lessons = [slot for slot in self.lessons if slot.day == weekday]
			if daily_lessons:
				if len(daily_lessons) > 1:
					daily_lessons.sort(key=lambda x: x.start.minutes)
					prev = daily_lessons[0].end.minutes
					for slot in daily_lessons[1:]:
						if slot.start.minutes > prev:
							result = result + (slot.start.minutes - prev)
						else:
							result = result + 120

						prev = slot.end.minutes

		return result

class Course(object):

	def __init__(self, name):
		self.name = name
		self.lesson_blocks = []

	def add_lesson_block(self, lesson_block):
		self.lesson_blocks.append(lesson_block)
		lesson_block.parent_course = self

	def get_block_by_category(self, category):
		for block in self.lesson_blocks:
			if block.category == category:
				return block

class LessonBlock:

	def __init__(self, category):
		self.category = category
		self.shifts = []

	def add_shift(self, shift):
		self.shifts.append(shift)
		shift.parent_lesson_block = self

class Shift:

	def __init__(self, name):
		self.name = name
		self.slots = []

	def add_lesson_slot(self, lesson_slot):
		self.slots.append(lesson_slot)
		lesson_slot.parent_shift = self

class LessonSlot:

	def __init__(self, day, start, end, room):
		self.day = day
		self.start = start
		self.end = end
		self.room = room

	def course_name(self):
		return self.parent_shift.parent_lesson_block.parent_course.name

	def lesson_category(self):
		return self.parent_shift.parent_lesson_block.category

	def overlaps_with_group(self, group):
		for ss in group:
			for other in ss:
				if self != other:
					if self.overlaps_with(other) == True:
						return True
		return False

	def overlaps_with(self, other):
		return self.day == other.day and (self.start.is_before(other.end) and self.end.is_after(other.start))

class Weekday:
	MONDAY = 0
	TUESDAY = 1
	WEDNESDAY = 2
	THURSDAY = 3
	FRIDAY = 4
	SATURDAY = 5
	SUNDAY = 6

class Time:

	def __init__(self, hour, minute):
		self.minutes = hour * 60 + minute

	def is_before(self, other):
		return self.minutes < other.minutes

	def is_after(self, other):
		return self.minutes > other.minutes

	def __str__(self):
		return "%d:%02d" % (self.minutes/60, self.minutes%60)
<commit_msg>corrected total_time2 function<commit_after>class TimetableGenerator:

	def __init__(self):
		self.generated = []
		self.generated2 = []
		self.total_combinations = 0

	def store_timetable(self, tt):
		self.total_combinations += 1
		tt.heuristic = tt.total_time()
		tt.heuristic2 = tt.total_time2()

		if len(self.generated) <= 99:
			self.generated.append(tt)
			self.generated2.append(tt)
		else:
			if tt.heuristic < self.generated[99].heuristic:  # tt is better
				self.generated.pop(99)
				self.generated.append(tt)
				self.generated.sort(key=lambda tt: tt.heuristic)

			if tt.heuristic2 < self.generated2[99].heuristic2:  # tt is better
				self.generated2.pop(99)
				self.generated2.append(tt)
				self.generated2.sort(key=lambda tt: tt.heuristic2)

	def generate_timetables(self, lesson_blocks):
		self.generate(Timetable(), lesson_blocks)

		for i in xrange(1, len(self.generated)+1):
			self.generated[len(self.generated) - i].score += i

		for i in xrange(1, len(self.generated2)+1):
			self.generated2[len(self.generated2) - i].score += i

		self.generated.sort(key=lambda tt: tt.score)

	def generate(self, timetable, lesson_blocks):
		if not lesson_blocks:
			self.store_timetable(timetable)
		else:
			next_lesson_block = lesson_blocks[0]
			for shift in next_lesson_block.shifts:
				#if timetable.supports(shift):
				self.generate(timetable.append_shift(shift), lesson_blocks[1:])

class Timetable:

	def __init__(self):
		self.lessons = []
		self.score = 0

	def append_shift(self, shift):
		new_timetable = Timetable()
		new_timetable.lessons.extend(self.lessons)
		new_timetable.lessons.extend(shift.slots)
		return new_timetable

	def supports(self, shift):
		for slot in shift.slots:
			for existing in self.lessons:
				if (slot.overlaps_with(existing)):
					return False
		return True

	#def is_feasible(self):
	#	for slot in self.lessons:
	#		for other in self.lessons:
	#			if (slot is not other) and (slot.overlaps_with(other)):
	#				return False
	#	return True

	# heuristics for selecting timetables
	def total_time2(self):
		result = 0
		for weekday in range(Weekday.MONDAY, Weekday.SUNDAY):
			daily_lessons = [slot for slot in self.lessons if slot.day == weekday]
			if daily_lessons:
				earliest_start = min([slot.start.minutes for slot in daily_lessons])
				latest_end = max([slot.end.minutes for slot in daily_lessons])
				interval = latest_end - earliest_start
				result += intervalo
		return result

	def total_time(self):
		result = 0
		for weekday in range(Weekday.MONDAY, Weekday.SUNDAY):
			daily_lessons = [slot for slot in self.lessons if slot.day == weekday]
			if daily_lessons:
				if len(daily_lessons) > 1:
					daily_lessons.sort(key=lambda x: x.start.minutes)
					prev = daily_lessons[0].end.minutes
					for slot in daily_lessons[1:]:
						if slot.start.minutes > prev:
							result = result + (slot.start.minutes - prev)
						else:
							result = result + 120

						prev = slot.end.minutes

		return result

class Course(object):

	def __init__(self, name):
		self.name = name
		self.lesson_blocks = []

	def add_lesson_block(self, lesson_block):
		self.lesson_blocks.append(lesson_block)
		lesson_block.parent_course = self

	def get_block_by_category(self, category):
		for block in self.lesson_blocks:
			if block.category == category:
				return block

class LessonBlock:

	def __init__(self, category):
		self.category = category
		self.shifts = []

	def add_shift(self, shift):
		self.shifts.append(shift)
		shift.parent_lesson_block = self

class Shift:

	def __init__(self, name):
		self.name = name
		self.slots = []

	def add_lesson_slot(self, lesson_slot):
		self.slots.append(lesson_slot)
		lesson_slot.parent_shift = self

class LessonSlot:

	def __init__(self, day, start, end, room):
		self.day = day
		self.start = start
		self.end = end
		self.room = room

	def course_name(self):
		return self.parent_shift.parent_lesson_block.parent_course.name

	def lesson_category(self):
		return self.parent_shift.parent_lesson_block.category

	def overlaps_with_group(self, group):
		for ss in group:
			for other in ss:
				if self != other:
					if self.overlaps_with(other) == True:
						return True
		return False

	def overlaps_with(self, other):
		return self.day == other.day and (self.start.is_before(other.end) and self.end.is_after(other.start))

class Weekday:
	MONDAY = 0
	TUESDAY = 1
	WEDNESDAY = 2
	THURSDAY = 3
	FRIDAY = 4
	SATURDAY = 5
	SUNDAY = 6

class Time:

	def __init__(self, hour, minute):
		self.minutes = hour * 60 + minute

	def is_before(self, other):
		return self.minutes < other.minutes

	def is_after(self, other):
		return self.minutes > other.minutes

	def __str__(self):
		return "%d:%02d" % (self.minutes/60, self.minutes%60)
<|endoftext|>


<commit_before>import asyncio
import asynqp
from unittest import mock
from asynqp import methods

class OutgoingMethodContext:
    def establish_the_connection(self):
        self.transport = mock.Mock(spec=asyncio.Transport)
        self.protocol = asynqp.AMQP(mock.Mock())
        self.protocol.connection_made(self.transport)

# todo: refactor tests like this to target the protocol
class WhenDeserialisingConnectionStart:
    def given_a_connection_start_method_I_copied_from_the_rabbitmq_server(self):
        self.raw = b"\x00\x0A\x00\x0A\x00\t\x00\x00\x01%\x0ccapabilitiesF\x00\x00\x00X\x12publisher_confirmst\x01\x1aexchange_exchange_bindingst\x01\nbasic.nackt\x01\x16consumer_cancel_notifyt\x01\tcopyrightS\x00\x00\x00'Copyright (C) 2007-2013 GoPivotal, Inc.\x0binformationS\x00\x00\x005Licensed under the MPL.  See http://www.rabbitmq.com/\x08platformS\x00\x00\x00\nErlang/OTP\x07productS\x00\x00\x00\x08RabbitMQ\x07versionS\x00\x00\x00\x053.1.5\x00\x00\x00\x0eAMQPLAIN PLAIN\x00\x00\x00\x0Ben_US en_GB"

    def when_I_deserialise_the_method(self):
        self.result = methods.deserialise_method(self.raw)

    def it_should_have_the_correct_major_version(self):
        assert self.result.version_major == 0

    def it_should_have_the_correct_minor_version(self):
        assert self.result.version_minor == 9

    def it_should_have_the_expected_server_properties(self):
        assert self.result.server_properties == {
            'capabilities': {'publisher_confirms': True,
                             'exchange_exchange_bindings': True,
                             'basic.nack': True,
                             'consumer_cancel_notify': True},
            'copyright': 'Copyright (C) 2007-2013 GoPivotal, Inc.',
            'information': 'Licensed under the MPL.  See http://www.rabbitmq.com/',
            'platform': 'Erlang/OTP',
            'product': 'RabbitMQ',
            'version': '3.1.5'
        }

    def it_should_have_the_security_mechanisms(self):
        assert self.result.mechanisms == 'AMQPLAIN PLAIN'

    def it_should_have_the_correct_locales(self):
        assert self.result.locales == 'en_US en_GB'

class WhenSendingConnectionStartOK(OutgoingMethodContext):
    def given_a_method_to_send(self):
        method = methods.ConnectionStartOK({'somecrap': 'aboutme'}, 'AMQPLAIN', {'auth':'info'}, 'en_US')
        self.frame = asynqp.Frame(asynqp.FrameType.method, 0, method)

    def when_we_send_the_method(self):
        self.protocol.send_frame(self.frame)

    def it_should_send_the_correct_bytestring(self):
        self.transport.write.assert_called_once_with(b'\x01\x00\x00\x00\x00\x00>\x00\n\x00\x0b\x00\x00\x00\x15\x08somecrapS\x00\x00\x00\x07aboutme\x08AMQPLAIN\x00\x00\x00\x0e\x04authS\x00\x00\x00\x04info\x05en_US\xce')

class WhenDeserialisingConnectionTune:
    def given_a_connection_tune_method_I_copied_from_the_rabbitmq_server(self):
        self.raw = b'\x00\x0A\x00\x1E\x00\x00\x00\x02\x00\x00\x02\x58'

    def when_I_deserialise_the_method(self):
        self.result = methods.deserialise_method(self.raw)

    def it_should_have_the_correct_max_channel(self):
        assert self.result.channel_max == 0

    def it_should_have_the_correct_max_frame_length(self):
        assert self.result.frame_max == 131072

    def it_shoud_have_the_correct_heartbeat(self):
        assert self.result.heartbeat == 600

class WhenSendingConnectionTuneOK(OutgoingMethodContext):
    def given_a_method_to_send(self):
        method = methods.ConnectionTuneOK(1024, 131072, 10)
        self.frame = asynqp.Frame(asynqp.FrameType.method, 0, method)

    def when_I_serialise_the_method(self):
        self.protocol.send_frame(self.frame)

    def it_should_return_the_correct_bytestring(self):
        self.transport.write.assert_called_once_with(b'\x01\x00\x00\x00\x00\x00\x0C\x00\n\x00\x1F\x04\x00\x00\x02\x00\x00\x00\x0A\xCE')

class WhenSendingConnectionOpen(OutgoingMethodContext):
    def given_a_method_to_send(self):
        method = methods.ConnectionOpen('/', '', False)
        self.frame = asynqp.Frame(asynqp.FrameType.method, 0, method)

    def when_I_serialise_the_method(self):
        self.protocol.send_frame(self.frame)

    def it_should_return_the_correct_bytestring(self):
        self.transport.write.assert_called_once_with(b'\x01\x00\x00\x00\x00\x00\x08\x00\x0A\x00\x28\x01\x2F\x00\x00\xCE')

class WhenDeserialisingConnectionOpenOK:
    def given_a_connection_open_ok_method_I_copied_from_the_rabbitmq_server(self):
        self.raw = b'\x00\x0A\x00\x29\x00'

    def when_I_deserialise_the_method(self):
        self.result = methods.deserialise_method(self.raw)

    def it_should_be_a_ConnectionOpenOK(self):
        assert isinstance(self.result, methods.ConnectionOpenOK)
<commit_msg>deleted some tests<commit_after>import asyncio
import asynqp
from unittest import mock
from asynqp import methods

class OutgoingMethodContext:
    def establish_the_connection(self):
        self.transport = mock.Mock(spec=asyncio.Transport)
        self.dispatcher = mock.Mock(spec=asynqp.Dispatcher)
        self.protocol = asynqp.AMQP(self.dispatcher)
        self.protocol.connection_made(self.transport)

class WhenDeserialisingConnectionStart:
    def given_a_connection_start_method_I_copied_from_the_rabbitmq_server(self):
        self.raw = b"\x00\x0A\x00\x0A\x00\t\x00\x00\x01%\x0ccapabilitiesF\x00\x00\x00X\x12publisher_confirmst\x01\x1aexchange_exchange_bindingst\x01\nbasic.nackt\x01\x16consumer_cancel_notifyt\x01\tcopyrightS\x00\x00\x00'Copyright (C) 2007-2013 GoPivotal, Inc.\x0binformationS\x00\x00\x005Licensed under the MPL.  See http://www.rabbitmq.com/\x08platformS\x00\x00\x00\nErlang/OTP\x07productS\x00\x00\x00\x08RabbitMQ\x07versionS\x00\x00\x00\x053.1.5\x00\x00\x00\x0eAMQPLAIN PLAIN\x00\x00\x00\x0Ben_US en_GB"

    def when_I_deserialise_the_method(self):
        self.result = methods.deserialise_method(self.raw)

    def it_should_have_the_correct_major_version(self):
        assert self.result.version_major == 0

    def it_should_have_the_correct_minor_version(self):
        assert self.result.version_minor == 9

    def it_should_have_the_expected_server_properties(self):
        assert self.result.server_properties == {
            'capabilities': {'publisher_confirms': True,
                             'exchange_exchange_bindings': True,
                             'basic.nack': True,
                             'consumer_cancel_notify': True},
            'copyright': 'Copyright (C) 2007-2013 GoPivotal, Inc.',
            'information': 'Licensed under the MPL.  See http://www.rabbitmq.com/',
            'platform': 'Erlang/OTP',
            'product': 'RabbitMQ',
            'version': '3.1.5'
        }

    def it_should_have_the_security_mechanisms(self):
        assert self.result.mechanisms == 'AMQPLAIN PLAIN'

    def it_should_have_the_correct_locales(self):
        assert self.result.locales == 'en_US en_GB'

class WhenSendingConnectionStartOK(OutgoingMethodContext):
    def given_a_method_to_send(self):
        method = methods.ConnectionStartOK({'somecrap': 'aboutme'}, 'AMQPLAIN', {'auth':'info'}, 'en_US')
        self.frame = asynqp.Frame(asynqp.FrameType.method, 0, method)

    def when_we_send_the_method(self):
        self.protocol.send_frame(self.frame)

    def it_should_send_the_correct_bytestring(self):
        self.transport.write.assert_called_once_with(b'\x01\x00\x00\x00\x00\x00>\x00\n\x00\x0b\x00\x00\x00\x15\x08somecrapS\x00\x00\x00\x07aboutme\x08AMQPLAIN\x00\x00\x00\x0e\x04authS\x00\x00\x00\x04info\x05en_US\xce')

class WhenDeserialisingConnectionTune:
    def given_a_connection_tune_method_I_copied_from_the_rabbitmq_server(self):
        self.raw = b'\x00\x0A\x00\x1E\x00\x00\x00\x02\x00\x00\x02\x58'

    def when_I_deserialise_the_method(self):
        self.result = methods.deserialise_method(self.raw)

    def it_should_have_the_correct_max_channel(self):
        assert self.result.channel_max == 0

    def it_should_have_the_correct_max_frame_length(self):
        assert self.result.frame_max == 131072

    def it_shoud_have_the_correct_heartbeat(self):
        assert self.result.heartbeat == 600

class WhenSendingConnectionTuneOK(OutgoingMethodContext):
    def given_a_method_to_send(self):
        method = methods.ConnectionTuneOK(1024, 131072, 10)
        self.frame = asynqp.Frame(asynqp.FrameType.method, 0, method)

    def when_I_serialise_the_method(self):
        self.protocol.send_frame(self.frame)

    def it_should_write_the_correct_bytestring(self):
        self.transport.write.assert_called_once_with(b'\x01\x00\x00\x00\x00\x00\x0C\x00\n\x00\x1F\x04\x00\x00\x02\x00\x00\x00\x0A\xCE')
<|endoftext|>


<commit_before>import testenv; testenv.configure_for_tests()

from testlib import sa, testing
from testlib.sa import Table, Column, Integer, String, ForeignKey
from testlib.sa.orm import mapper, relation, create_session
from orm import _base
from testlib.testing import eq_

class TriggerDefaultsTest(_base.MappedTest):
    __requires__ = ('row_triggers',)

    def define_tables(self, metadata):
        dt = Table('dt', metadata,
                   Column('id', Integer, primary_key=True),
                   Column('col1', String(20)),
                   Column('col2', String(20),
                          server_default=sa.schema.FetchedValue()),
                   Column('col3', String(20),
                          sa.schema.FetchedValue(for_update=True)),
                   Column('col4', String(20),
                          sa.schema.FetchedValue(),
                          sa.schema.FetchedValue(for_update=True)))
        for ins in (
            sa.DDL("CREATE TRIGGER dt_ins AFTER INSERT ON dt "
                   "FOR EACH ROW BEGIN "
                   "UPDATE dt SET col2='ins', col4='ins' "
                   "WHERE dt.id = NEW.id; END",
                   on='sqlite'),
            ):
            if testing.against(ins.on):
                break
        else:
            ins = sa.DDL("CREATE TRIGGER dt_ins BEFORE INSERT ON dt "
                         "FOR EACH ROW BEGIN "
                         "SET NEW.col2='ins'; SET NEW.col4='ins'; END")
        ins.execute_at('after-create', dt)
        sa.DDL("DROP TRIGGER dt_ins").execute_at('before-drop', dt)

        for up in (
            sa.DDL("CREATE TRIGGER dt_up AFTER UPDATE ON dt "
                   "FOR EACH ROW BEGIN "
                   "UPDATE dt SET col3='up', col4='up' "
                   "WHERE dt.id = OLD.id; END",
                   on='sqlite'),
            ):
            if testing.against(up.on):
                break
        else:
            up = sa.DDL("CREATE TRIGGER dt_up BEFORE UPDATE ON dt "
                        "FOR EACH ROW BEGIN "
                        "SET NEW.col3='up'; SET NEW.col4='up'; END")
        up.execute_at('after-create', dt)
        sa.DDL("DROP TRIGGER dt_up").execute_at('before-drop', dt)

    def setup_classes(self):
        class Default(_base.BasicEntity):
            pass

    @testing.resolve_artifact_names
    def setup_mappers(self):
        mapper(Default, dt)

    @testing.resolve_artifact_names
    def test_insert(self):

        d1 = Default(id=1)

        eq_(d1.col1, None)
        eq_(d1.col2, None)
        eq_(d1.col3, None)
        eq_(d1.col4, None)

        session = create_session()
        session.add(d1)
        session.flush()

        eq_(d1.col1, None)
        eq_(d1.col2, 'ins')
        eq_(d1.col3, None)
        # don't care which trigger fired
        assert d1.col4 in ('ins', 'up')

    @testing.resolve_artifact_names
    def test_update(self):
        d1 = Default(id=1)

        session = create_session()
        session.add(d1)
        session.flush()
        d1.col1 = 'set'
        session.flush()

        eq_(d1.col1, 'set')
        eq_(d1.col2, 'ins')
        eq_(d1.col3, 'up')
        eq_(d1.col4, 'up')

class ExcludedDefaultsTest(_base.MappedTest):
    def define_tables(self, metadata):
        dt = Table('dt', metadata,
                   Column('id', Integer, primary_key=True),
                   Column('col1', String(20), default="hello"),
        )

    @testing.resolve_artifact_names
    def test_exclude(self):
        class Foo(_base.ComparableEntity):
            pass
        mapper(Foo, dt, exclude_properties=('col1',))

        f1 = Foo()
        sess = create_session()
        sess.add(f1)
        sess.flush()
        eq_(dt.select().execute().fetchall(), [(1, "hello")])

if __name__ == "__main__":
    testenv.main()
<commit_msg>Specialized trigger tests to accomodate mssql syntax.<commit_after>import testenv; testenv.configure_for_tests()

from testlib import sa, testing
from testlib.sa import Table, Column, Integer, String, ForeignKey
from testlib.sa.orm import mapper, relation, create_session
from orm import _base
from testlib.testing import eq_

class TriggerDefaultsTest(_base.MappedTest):
    __requires__ = ('row_triggers',)

    def define_tables(self, metadata):
        dt = Table('dt', metadata,
                   Column('id', Integer, primary_key=True),
                   Column('col1', String(20)),
                   Column('col2', String(20),
                          server_default=sa.schema.FetchedValue()),
                   Column('col3', String(20),
                          sa.schema.FetchedValue(for_update=True)),
                   Column('col4', String(20),
                          sa.schema.FetchedValue(),
                          sa.schema.FetchedValue(for_update=True)))
        for ins in (
            sa.DDL("CREATE TRIGGER dt_ins AFTER INSERT ON dt "
                   "FOR EACH ROW BEGIN "
                   "UPDATE dt SET col2='ins', col4='ins' "
                   "WHERE dt.id = NEW.id; END",
                   on='sqlite'),
            sa.DDL("CREATE TRIGGER dt_ins ON dt AFTER INSERT AS "
                   "UPDATE dt SET col2='ins', col4='ins' "
                   "WHERE dt.id IN (SELECT id FROM inserted);",
                   on='mssql'),
            ):
            if testing.against(ins.on):
                break
        else:
            ins = sa.DDL("CREATE TRIGGER dt_ins BEFORE INSERT ON dt "
                         "FOR EACH ROW BEGIN "
                         "SET NEW.col2='ins'; SET NEW.col4='ins'; END")
        ins.execute_at('after-create', dt)
        sa.DDL("DROP TRIGGER dt_ins").execute_at('before-drop', dt)

        for up in (
            sa.DDL("CREATE TRIGGER dt_up AFTER UPDATE ON dt "
                   "FOR EACH ROW BEGIN "
                   "UPDATE dt SET col3='up', col4='up' "
                   "WHERE dt.id = OLD.id; END",
                   on='sqlite'),
            sa.DDL("CREATE TRIGGER dt_up ON dt AFTER UPDATE AS "
                   "UPDATE dt SET col3='up', col4='up' "
                   "WHERE dt.id IN (SELECT id FROM deleted);",
                   on='mssql'),
            ):
            if testing.against(up.on):
                break
        else:
            up = sa.DDL("CREATE TRIGGER dt_up BEFORE UPDATE ON dt "
                        "FOR EACH ROW BEGIN "
                        "SET NEW.col3='up'; SET NEW.col4='up'; END")
        up.execute_at('after-create', dt)
        sa.DDL("DROP TRIGGER dt_up").execute_at('before-drop', dt)

    def setup_classes(self):
        class Default(_base.BasicEntity):
            pass

    @testing.resolve_artifact_names
    def setup_mappers(self):
        mapper(Default, dt)

    @testing.resolve_artifact_names
    def test_insert(self):

        d1 = Default(id=1)

        eq_(d1.col1, None)
        eq_(d1.col2, None)
        eq_(d1.col3, None)
        eq_(d1.col4, None)

        session = create_session()
        session.add(d1)
        session.flush()

        eq_(d1.col1, None)
        eq_(d1.col2, 'ins')
        eq_(d1.col3, None)
        # don't care which trigger fired
        assert d1.col4 in ('ins', 'up')

    @testing.resolve_artifact_names
    def test_update(self):
        d1 = Default(id=1)

        session = create_session()
        session.add(d1)
        session.flush()
        d1.col1 = 'set'
        session.flush()

        eq_(d1.col1, 'set')
        eq_(d1.col2, 'ins')
        eq_(d1.col3, 'up')
        eq_(d1.col4, 'up')

class ExcludedDefaultsTest(_base.MappedTest):
    def define_tables(self, metadata):
        dt = Table('dt', metadata,
                   Column('id', Integer, primary_key=True),
                   Column('col1', String(20), default="hello"),
        )

    @testing.resolve_artifact_names
    def test_exclude(self):
        class Foo(_base.ComparableEntity):
            pass
        mapper(Foo, dt, exclude_properties=('col1',))

        f1 = Foo()
        sess = create_session()
        sess.add(f1)
        sess.flush()
        eq_(dt.select().execute().fetchall(), [(1, "hello")])

if __name__ == "__main__":
    testenv.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import json
import sys

PREFIXES = [
    ["FAIL", "PASS"],
    ["EXPECTED FAIL", "UNEXPECTED PASS"],
]

def parse_expected_failures():
    expected_failures = set()
    with open("expected-failures.txt", "r") as fp:
        for line in fp:
            line = line.strip()
            if not line:
                continue
            if line.startswith("#"):
                continue
            expected_failures.add(line)
    return expected_failures

def main(filename):
    expected_failures = parse_expected_failures()
    with open(filename, "r") as fp:
        results = json.load(fp)

    unexpected_results = []
    for test in results:
        expected_failure = test["file"] in expected_failures
        actual_result = test["result"]["pass"]
        print("{} {}".format(PREFIXES[expected_failure][actual_result], test["file"]))
        if actual_result == expected_failure:
            if not actual_result:
                print(test["rawResult"]["stderr"])
                print(test["rawResult"]["stdout"])
                print(test["result"]["message"])
            unexpected_results.append(test)

    if unexpected_results:
        print("{} unexpected results:".format(len(unexpected_results)))
        for unexpected in unexpected_results:
            print("- {}".format(unexpected["file"]))
        return False

    print("All results as expected.")
    return True

if __name__ == "__main__":
    sys.exit(0 if main(sys.argv[1]) else 1)
<commit_msg>Print the scenario when running 262-style tests.<commit_after>#!/usr/bin/env python3

import json
import sys

PREFIXES = [
    ["FAIL", "PASS"],
    ["EXPECTED FAIL", "UNEXPECTED PASS"],
]

def parse_expected_failures():
    expected_failures = set()
    with open("expected-failures.txt", "r") as fp:
        for line in fp:
            line = line.strip()
            if not line:
                continue
            if line.startswith("#"):
                continue
            expected_failures.add(line)
    return expected_failures

def main(filename):
    expected_failures = parse_expected_failures()
    with open(filename, "r") as fp:
        results = json.load(fp)

    unexpected_results = []
    for test in results:
        expected_failure = test["file"] in expected_failures
        actual_result = test["result"]["pass"]
        print("{} {} ({})".format(PREFIXES[expected_failure][actual_result], test["file"], test["scenario"]))
        if actual_result == expected_failure:
            if not actual_result:
                print(test["rawResult"]["stderr"])
                print(test["rawResult"]["stdout"])
                print(test["result"]["message"])
            unexpected_results.append(test)

    if unexpected_results:
        print("{} unexpected results:".format(len(unexpected_results)))
        for unexpected in unexpected_results:
            print("- {}".format(unexpected["file"]))
        return False

    print("All results as expected.")
    return True

if __name__ == "__main__":
    sys.exit(0 if main(sys.argv[1]) else 1)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# vim: fdm=indent
'''
author:     Fabio Zanini
date:       07/08/17
content:    Tests for the library.
'''
# Modules
import os
import subprocess as sp

# Controlled sp environment
def run(script, where=None, **kwargs):
    import platform

    if where == 'local' and platform.node() != 'X260':
        return
    if where == 'remote' and platform.node() == 'X260':
        return

    env = os.environ.copy()
    env['SINGLET_CONFIG_FILENAME'] = 'example_data/config_example.yml'

    # Include local tests
    if platform.node() == 'X260':
        singlet_path = os.path.dirname(os.path.dirname(__file__))
        env['PYTHONPATH'] = singlet_path+':'+env['PYTHONPATH']
        print(singlet_path)

    return sp.check_call(
        script,
        env=env,
        shell=True,
        **kwargs)

# Script
if __name__ == '__main__':

    ## Init
    #run('test/dataset/initialize.py')

    ## Queries
    #run('test/dataset/query.py')

    ## Correlations
    #run('test/dataset/correlation.py')

    # Dimensionality reduction
    run('test/dataset/dimensionality.py')

    # Dimensionality reduction
    run('test/dataset/cluster.py')

    # Plot
    # TODO: one could set up PNG comparisons like Matplotlib itself does
    run('test/dataset/plot.py', where='local')
<commit_msg>Uncomment all tests<commit_after>#!/usr/bin/env python
# vim: fdm=indent
'''
author:     Fabio Zanini
date:       07/08/17
content:    Tests for the library.
'''
# Modules
import os
import subprocess as sp

# Controlled sp environment
def run(script, where=None, **kwargs):
    import platform

    if where == 'local' and platform.node() != 'X260':
        return
    if where == 'remote' and platform.node() == 'X260':
        return

    env = os.environ.copy()
    env['SINGLET_CONFIG_FILENAME'] = 'example_data/config_example.yml'

    # Include local tests
    if platform.node() == 'X260':
        singlet_path = os.path.dirname(os.path.dirname(__file__))
        env['PYTHONPATH'] = singlet_path+':'+env['PYTHONPATH']
        print(singlet_path)

    return sp.check_call(
        script,
        env=env,
        shell=True,
        **kwargs)

# Script
if __name__ == '__main__':

    # Init
    run('test/dataset/initialize.py')

    # Queries
    run('test/dataset/query.py')

    # Correlations
    run('test/dataset/correlation.py')

    # Dimensionality reduction
    run('test/dataset/dimensionality.py')

    # Dimensionality reduction
    run('test/dataset/cluster.py')

    # Plot
    # TODO: one could set up PNG comparisons like Matplotlib itself does
    run('test/dataset/plot.py', where='local')
<|endoftext|>