<commit_before># Copyright 2017 The Forseti Security Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

""" Forseti Installer."""

from __future__ import print_function
from abc import ABCMeta
from abc import abstractmethod

import sys

from configs.config import Config
from util import constants
from util import files
from util import gcloud
from util import utils

class ForsetiInstaller(object):
    """Forseti installer base class (abstract)"""
    __metaclass__ = ABCMeta

    # Class variables initialization
    branch = None
    project_id = None
    organization_id = None
    gcp_service_account = None
    user_can_grant_roles = True
    config = Config()

    @abstractmethod
    def __init__(self):
        """Initialize."""
        pass

    def run_setup(self):
        """Run the setup steps."""
        utils.print_banner('Installing Forseti {} v{}'.format(
            self.config.installer_type, utils.get_forseti_version()))

        # Preflight checks.
        self.preflight_checks()

        # Create/Reuse service account(s).
        self.create_or_reuse_service_accts()

        # Create configuration file and deployment template.
        (conf_file_path,
         deployment_tpl_path) = self.create_conf_file_dpl_template()

        # Deployment.
        bucket_name = self.generate_bucket_name()
        deploy_success, deployment_name = self.deploy(deployment_tpl_path,
                                                      conf_file_path,
                                                      bucket_name)

        # After deployment.
        self.post_install_instructions(deploy_success,
                                       deployment_name,
                                       deployment_tpl_path,
                                       conf_file_path,
                                       bucket_name)

    def create_conf_file_dpl_template(self):
        """Create configuration file and deployment template."""
        utils.print_banner('Creating configuration file and'
                           ' deployment template')
        conf_file_path = self.generate_forseti_conf()
        deployment_tpl_path = self.generate_deployment_templates()

        return conf_file_path, deployment_tpl_path

    def preflight_checks(self):
        """Pre-flight checks"""
        self.check_run_properties()
        self.branch = utils.infer_version(self.config.advanced_mode)
        self.project_id, authed_user, is_cloudshell = gcloud.get_gcloud_info()
        gcloud.verify_gcloud_information(self.project_id,
                                         authed_user,
                                         self.config.force_no_cloudshell,
                                         is_cloudshell)
        self.organization_id = gcloud.lookup_organization(self.project_id)
        gcloud.check_billing_enabled(self.project_id, self.organization_id)

    def create_or_reuse_service_accts(self):
        """Create or reuse service accounts."""
        utils.print_banner('Create/Reuse service account(s)')
        self.format_gcp_service_acct_id()
        self.gcp_service_account = gcloud.create_or_reuse_service_acct(
            'gcp_service_account',
            self.gcp_service_account,
            self.config.advanced_mode,
            self.config.dry_run)

    def deploy(self, deployment_tpl_path, conf_file_path, bucket_name):
        """Deploy Forseti using the deployment template

        Args:
            deployment_tpl_path (str): Deployment template path
            conf_file_path (str): Configuration file path
            bucket_name (str): Name of the GCS bucket

        Returns:
            bool: Whether or not the deployment was successful
            str: Deployment name
        """
        deployment_name, return_code = gcloud.create_deployment(
            self.project_id,
            self.organization_id,
            deployment_tpl_path,
            self.config.installer_type,
            self.config.datetimestamp,
            self.config.dry_run)

        deployment_completed = not return_code

        if deployment_completed:
            # If deployed successfully, make sure the VM has been initialized,
            # copy configuration file, deployment template file and
            # rule files to the GCS bucket

            if self.config.dry_run:
                print('This is a dry run, will not copy any files.')

            utils.print_banner('Copying configuration file/deployment'
                               ' template to GCS')

            conf_output_path = constants.FORSETI_CONF_PATH.format(
                bucket_name=bucket_name,
                installer_type=self.config.installer_type)

            print('Copying {} to {}'.format(conf_file_path, conf_output_path))

            files.copy_file_to_destination(
                conf_file_path, conf_output_path,
                is_directory=False, dry_run=self.config.dry_run)

            deployment_tpl_output_path = (
                constants.DEPLOYMENT_TEMPLATE_OUTPUT_PATH.format(bucket_name))

            print('Copying {} to {}'.format(deployment_tpl_path,
                                            deployment_tpl_output_path))

            files.copy_file_to_destination(
                deployment_tpl_path, deployment_tpl_output_path,
                is_directory=False, dry_run=self.config.dry_run)

        return deployment_completed, deployment_name

    def wait_until_vm_initialized(self, vm_name):
        """Check vm init status.

        Args:
            vm_name (str): Name of the VM instance.
        """
        utils.print_banner('Forseti {} VM initialization'.format(
            self.config.installer_type))
        _, zone, name = gcloud.get_vm_instance_info(vm_name)

        # VT100 control codes, use to remove the last line
        erase_line = '\x1b[2K'

        for i in range(0, constants.MAXIMUM_LOOP_COUNT):
            dots = '.' * (i % 10)
            sys.stdout.write('\r{}This may take a few minutes. '
                             'Waiting for Forseti server to start{}'
                             .format(erase_line, dots))
            sys.stdout.flush()
            if gcloud.check_vm_init_status(name, zone):
                break
        # print new line
        print ('Done.\n')

    def check_run_properties(self):
        """Check script run properties."""
        print('Dry run? %s' % self.config.dry_run)
        print('Advanced mode? %s' % self.config.advanced_mode)

    def format_gcp_service_acct_id(self):
        """Format the service account ids."""
        self.gcp_service_account = utils.format_service_acct_id(
            'gcp',
            'reader',
            self.config.timestamp,
            self.project_id)

    def generate_bucket_name(self):
        """Generate GCS bucket name.

        Returns:
            str: Name of the GCS bucket
        """
        return constants.DEFAULT_BUCKET_FMT_V2.format(
            self.project_id,
            self.config.installer_type,
            self.config.timestamp)

    @abstractmethod
    def get_deployment_values(self):
        """Get deployment values

        Returns:
            dict: A dictionary of values needed to generate
                the forseti deployment template
        """
        return {}

    @abstractmethod
    def get_configuration_values(self):
        """Get configuration values

        Returns:
            dict: A dictionary of values needed to generate
                the forseti configuration file
        """
        return {}

    def generate_deployment_templates(self):
        """Generate deployment templates.

        Returns:
            str: Deployment template path
        """
        print('Generate Deployment Manager templates...')

        deploy_values = self.get_deployment_values()

        deployment_tpl_path = files.generate_deployment_templates(
            self.config.installer_type,
            deploy_values,
            self.config.datetimestamp)

        print('\nCreated a deployment template:\n    %s\n' %
              deployment_tpl_path)
        return deployment_tpl_path

    def generate_forseti_conf(self):
        """Generate Forseti conf file.

        Returns:
            str: Forseti configuration file path
        """
        # Create a forseti_conf_{INSTALLER_TYPE}_$TIMESTAMP.yaml config file
        # with values filled in.
        print('\nGenerate forseti_conf_{}_{}.yaml...'
              .format(self.config.installer_type, self.config.datetimestamp))

        conf_values = self.get_configuration_values()

        forseti_conf_path = files.generate_forseti_conf(
            self.config.installer_type,
            conf_values,
            self.config.datetimestamp)

        print('\nCreated forseti_conf_{}_{}.yaml config file:\n    {}\n'.
              format(self.config.installer_type,
                     self.config.datetimestamp,
                     forseti_conf_path))
        return forseti_conf_path

    def post_install_instructions(self, deploy_success, deployment_name,
                                  deployment_tpl_path, forseti_conf_path,
                                  bucket_name):
        """Show post-install instructions.

        Print link for deployment manager dashboard
        Print link to go to G Suite service account and enable DWD

        Args:
            deploy_success (bool): Whether deployment was successful
            deployment_name (str): Name of the deployment
            deployment_tpl_path (str): Deployment template path
            forseti_conf_path (str): Forseti configuration file path
            bucket_name (str): Name of the GCS bucket
        """
        utils.print_banner('Forseti {} post-setup instructions'
                           .format(self.config.installer_type))

        if self.config.dry_run:
            print('This was a dry run, so a deployment was not attempted. '
                  'You can still create the deployment manually.\n')
        elif deploy_success:
            print(constants.MESSAGE_FORSETI_BRANCH_DEPLOYED.format(
                self.branch))
        else:
            print(constants.MESSAGE_DEPLOYMENT_HAD_ISSUES)

        deploy_tpl_gcs_path = constants.DEPLOYMENT_TEMPLATE_OUTPUT_PATH.format(
            bucket_name)

        print(constants.MESSAGE_DEPLOYMENT_TEMPLATE_LOCATION.format(
            deployment_tpl_path, deploy_tpl_gcs_path))

        if self.config.dry_run:
            print(constants.MESSAGE_FORSETI_CONFIGURATION_GENERATED_DRY_RUN
                  .format(forseti_conf_path, bucket_name))
        else:
            print(constants.MESSAGE_VIEW_DEPLOYMENT_DETAILS.format(
                deployment_name,
                self.project_id,
                self.organization_id))

            print(constants.MESSAGE_FORSETI_CONFIGURATION_GENERATED.format(
                installer_type=self.config.installer_type,
                datetimestamp=self.config.datetimestamp,
                bucket_name=bucket_name))
<commit_msg>Added missing docstring<commit_after># Copyright 2017 The Forseti Security Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

""" Forseti Installer."""

from __future__ import print_function
from abc import ABCMeta
from abc import abstractmethod

import sys

from configs.config import Config
from util import constants
from util import files
from util import gcloud
from util import utils

class ForsetiInstaller(object):
    """Forseti installer base class (abstract)"""
    __metaclass__ = ABCMeta

    # Class variables initialization
    branch = None
    project_id = None
    organization_id = None
    gcp_service_account = None
    user_can_grant_roles = True
    config = Config()

    @abstractmethod
    def __init__(self):
        """Initialize."""
        pass

    def run_setup(self):
        """Run the setup steps."""
        utils.print_banner('Installing Forseti {} v{}'.format(
            self.config.installer_type, utils.get_forseti_version()))

        # Preflight checks.
        self.preflight_checks()

        # Create/Reuse service account(s).
        self.create_or_reuse_service_accts()

        # Create configuration file and deployment template.
        (conf_file_path,
         deployment_tpl_path) = self.create_conf_file_dpl_template()

        # Deployment.
        bucket_name = self.generate_bucket_name()
        deploy_success, deployment_name = self.deploy(deployment_tpl_path,
                                                      conf_file_path,
                                                      bucket_name)

        # After deployment.
        self.post_install_instructions(deploy_success,
                                       deployment_name,
                                       deployment_tpl_path,
                                       conf_file_path,
                                       bucket_name)

    def create_conf_file_dpl_template(self):
        """Create configuration file and deployment template.

        Returns:
            str: Configuration file path.
            ]str: Deployment template path.
        """
        utils.print_banner('Creating configuration file and'
                           ' deployment template')
        conf_file_path = self.generate_forseti_conf()
        deployment_tpl_path = self.generate_deployment_templates()

        return conf_file_path, deployment_tpl_path

    def preflight_checks(self):
        """Pre-flight checks"""
        self.check_run_properties()
        self.branch = utils.infer_version(self.config.advanced_mode)
        self.project_id, authed_user, is_cloudshell = gcloud.get_gcloud_info()
        gcloud.verify_gcloud_information(self.project_id,
                                         authed_user,
                                         self.config.force_no_cloudshell,
                                         is_cloudshell)
        self.organization_id = gcloud.lookup_organization(self.project_id)
        gcloud.check_billing_enabled(self.project_id, self.organization_id)

    def create_or_reuse_service_accts(self):
        """Create or reuse service accounts."""
        utils.print_banner('Create/Reuse service account(s)')
        self.format_gcp_service_acct_id()
        self.gcp_service_account = gcloud.create_or_reuse_service_acct(
            'gcp_service_account',
            self.gcp_service_account,
            self.config.advanced_mode,
            self.config.dry_run)

    def deploy(self, deployment_tpl_path, conf_file_path, bucket_name):
        """Deploy Forseti using the deployment template

        Args:
            deployment_tpl_path (str): Deployment template path
            conf_file_path (str): Configuration file path
            bucket_name (str): Name of the GCS bucket

        Returns:
            bool: Whether or not the deployment was successful
            str: Deployment name
        """
        deployment_name, return_code = gcloud.create_deployment(
            self.project_id,
            self.organization_id,
            deployment_tpl_path,
            self.config.installer_type,
            self.config.datetimestamp,
            self.config.dry_run)

        deployment_completed = not return_code

        if deployment_completed:
            # If deployed successfully, make sure the VM has been initialized,
            # copy configuration file, deployment template file and
            # rule files to the GCS bucket

            if self.config.dry_run:
                print('This is a dry run, will not copy any files.')

            utils.print_banner('Copying configuration file/deployment'
                               ' template to GCS')

            conf_output_path = constants.FORSETI_CONF_PATH.format(
                bucket_name=bucket_name,
                installer_type=self.config.installer_type)

            print('Copying {} to {}'.format(conf_file_path, conf_output_path))

            files.copy_file_to_destination(
                conf_file_path, conf_output_path,
                is_directory=False, dry_run=self.config.dry_run)

            deployment_tpl_output_path = (
                constants.DEPLOYMENT_TEMPLATE_OUTPUT_PATH.format(bucket_name))

            print('Copying {} to {}'.format(deployment_tpl_path,
                                            deployment_tpl_output_path))

            files.copy_file_to_destination(
                deployment_tpl_path, deployment_tpl_output_path,
                is_directory=False, dry_run=self.config.dry_run)

        return deployment_completed, deployment_name

    def wait_until_vm_initialized(self, vm_name):
        """Check vm init status.

        Args:
            vm_name (str): Name of the VM instance.
        """
        utils.print_banner('Forseti {} VM initialization'.format(
            self.config.installer_type))
        _, zone, name = gcloud.get_vm_instance_info(vm_name)

        # VT100 control codes, use to remove the last line
        erase_line = '\x1b[2K'

        for i in range(0, constants.MAXIMUM_LOOP_COUNT):
            dots = '.' * (i % 10)
            sys.stdout.write('\r{}This may take a few minutes. '
                             'Waiting for Forseti server to start{}'
                             .format(erase_line, dots))
            sys.stdout.flush()
            if gcloud.check_vm_init_status(name, zone):
                break
        # print new line
        print ('Done.\n')

    def check_run_properties(self):
        """Check script run properties."""
        print('Dry run? %s' % self.config.dry_run)
        print('Advanced mode? %s' % self.config.advanced_mode)

    def format_gcp_service_acct_id(self):
        """Format the service account ids."""
        self.gcp_service_account = utils.format_service_acct_id(
            'gcp',
            'reader',
            self.config.timestamp,
            self.project_id)

    def generate_bucket_name(self):
        """Generate GCS bucket name.

        Returns:
            str: Name of the GCS bucket
        """
        return constants.DEFAULT_BUCKET_FMT_V2.format(
            self.project_id,
            self.config.installer_type,
            self.config.timestamp)

    @abstractmethod
    def get_deployment_values(self):
        """Get deployment values

        Returns:
            dict: A dictionary of values needed to generate
                the forseti deployment template
        """
        return {}

    @abstractmethod
    def get_configuration_values(self):
        """Get configuration values

        Returns:
            dict: A dictionary of values needed to generate
                the forseti configuration file
        """
        return {}

    def generate_deployment_templates(self):
        """Generate deployment templates.

        Returns:
            str: Deployment template path
        """
        print('Generate Deployment Manager templates...')

        deploy_values = self.get_deployment_values()

        deployment_tpl_path = files.generate_deployment_templates(
            self.config.installer_type,
            deploy_values,
            self.config.datetimestamp)

        print('\nCreated a deployment template:\n    %s\n' %
              deployment_tpl_path)
        return deployment_tpl_path

    def generate_forseti_conf(self):
        """Generate Forseti conf file.

        Returns:
            str: Forseti configuration file path
        """
        # Create a forseti_conf_{INSTALLER_TYPE}_$TIMESTAMP.yaml config file
        # with values filled in.
        print('\nGenerate forseti_conf_{}_{}.yaml...'
              .format(self.config.installer_type, self.config.datetimestamp))

        conf_values = self.get_configuration_values()

        forseti_conf_path = files.generate_forseti_conf(
            self.config.installer_type,
            conf_values,
            self.config.datetimestamp)

        print('\nCreated forseti_conf_{}_{}.yaml config file:\n    {}\n'.
              format(self.config.installer_type,
                     self.config.datetimestamp,
                     forseti_conf_path))
        return forseti_conf_path

    def post_install_instructions(self, deploy_success, deployment_name,
                                  deployment_tpl_path, forseti_conf_path,
                                  bucket_name):
        """Show post-install instructions.

        Print link for deployment manager dashboard
        Print link to go to G Suite service account and enable DWD

        Args:
            deploy_success (bool): Whether deployment was successful
            deployment_name (str): Name of the deployment
            deployment_tpl_path (str): Deployment template path
            forseti_conf_path (str): Forseti configuration file path
            bucket_name (str): Name of the GCS bucket
        """
        utils.print_banner('Forseti {} post-setup instructions'
                           .format(self.config.installer_type))

        if self.config.dry_run:
            print('This was a dry run, so a deployment was not attempted. '
                  'You can still create the deployment manually.\n')
        elif deploy_success:
            print(constants.MESSAGE_FORSETI_BRANCH_DEPLOYED.format(
                self.branch))
        else:
            print(constants.MESSAGE_DEPLOYMENT_HAD_ISSUES)

        deploy_tpl_gcs_path = constants.DEPLOYMENT_TEMPLATE_OUTPUT_PATH.format(
            bucket_name)

        print(constants.MESSAGE_DEPLOYMENT_TEMPLATE_LOCATION.format(
            deployment_tpl_path, deploy_tpl_gcs_path))

        if self.config.dry_run:
            print(constants.MESSAGE_FORSETI_CONFIGURATION_GENERATED_DRY_RUN
                  .format(forseti_conf_path, bucket_name))
        else:
            print(constants.MESSAGE_VIEW_DEPLOYMENT_DETAILS.format(
                deployment_name,
                self.project_id,
                self.organization_id))

            print(constants.MESSAGE_FORSETI_CONFIGURATION_GENERATED.format(
                installer_type=self.config.installer_type,
                datetimestamp=self.config.datetimestamp,
                bucket_name=bucket_name))
<|endoftext|>


<commit_before>import click
import os
import sys

PY3 = sys.version_info[0] == 3
OLD_PY2 = sys.version_info[:2] < (2, 7)

if PY3:  # pragma: no cover
    input_str = 'builtins.input'
    iteritems = lambda d: iter(d.items())
    from unittest.mock import patch
    from io import StringIO

else:  # pragma: no cover
    from __builtin__ import raw_input
    input = raw_input
    input_str = '__builtin__.raw_input'
    iteritems = lambda d: d.iteritems()
    from mock import patch
    from cStringIO import StringIO

if PY3:  # Forced testing

    from shutil import which

else:  # Forced testing

    def is_exe(program):
        """
        Returns whether or not a file is an executable.
        """
        return os.path.isfile(program) and os.access(program, os.X_OK)

    def which(cmd, mode=os.F_OK | os.X_OK, path=None):
        """Given a command, mode, and a PATH string, return the path which
        conforms to the given mode on the PATH, or None if there is no such
        file.
        `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result
        of os.environ.get("PATH"), or can be overridden with a custom search
        path.

        Note: This function was backported from the Python 3 source code.
        """
        # Check that a given file can be accessed with the correct mode.
        # Additionally check that `file` is not a directory, as on Windows
        # directories pass the os.access check.
        def _access_check(fn, mode):
            return (os.path.exists(fn) and os.access(fn, mode) and
                    not os.path.isdir(fn))

        # If we're given a path with a directory part, look it up directly
        # rather than referring to PATH directories. This includes checking
        # relative to the current directory, e.g. ./script
        if os.path.dirname(cmd):
            if _access_check(cmd, mode):
                return cmd
            return None

        if path is None:
            path = os.environ.get("PATH", os.defpath)
        if not path:
            return None
        path = path.split(os.pathsep)

        if sys.platform == "win32":
            # The current directory takes precedence on Windows.
            if os.curdir not in path:
                path.insert(0, os.curdir)

            # PATHEXT is necessary to check on Windows.
            pathext = os.environ.get("PATHEXT", "").split(os.pathsep)
            # See if the given file matches any of the expected path
            # extensions. This will allow us to short circuit when given
            # "python.exe". If it does match, only test that one, otherwise we
            # have to try others.
            if any(cmd.lower().endswith(ext.lower()) for ext in pathext):
                files = [cmd]
            else:
                files = [cmd + ext for ext in pathext]
        else:
            # On other platforms you don't have things like PATHEXT to tell you
            # what file suffixes are executable, so just pass on cmd as-is.
            files = [cmd]

        seen = set()
        for dir in path:
            normdir = os.path.normcase(dir)
            if normdir not in seen:
                seen.add(normdir)
                for thefile in files:
                    name = os.path.join(dir, thefile)
                    if _access_check(name, mode):
                        return name
        return None

def is_string(obj):
    """Determine if an object is a string."""
    return isinstance(obj, str if PY3 else basestring)

_hush_pyflakes = (patch, StringIO, which)
<commit_msg>Remove unused click import from compat.py<commit_after>import os
import sys

PY3 = sys.version_info[0] == 3
OLD_PY2 = sys.version_info[:2] < (2, 7)

if PY3:  # pragma: no cover
    input_str = 'builtins.input'
    iteritems = lambda d: iter(d.items())
    from unittest.mock import patch
    from io import StringIO

else:  # pragma: no cover
    from __builtin__ import raw_input
    input = raw_input
    input_str = '__builtin__.raw_input'
    iteritems = lambda d: d.iteritems()
    from mock import patch
    from cStringIO import StringIO

if PY3:  # Forced testing

    from shutil import which

else:  # Forced testing

    def is_exe(program):
        """
        Returns whether or not a file is an executable.
        """
        return os.path.isfile(program) and os.access(program, os.X_OK)

    def which(cmd, mode=os.F_OK | os.X_OK, path=None):
        """Given a command, mode, and a PATH string, return the path which
        conforms to the given mode on the PATH, or None if there is no such
        file.
        `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result
        of os.environ.get("PATH"), or can be overridden with a custom search
        path.

        Note: This function was backported from the Python 3 source code.
        """
        # Check that a given file can be accessed with the correct mode.
        # Additionally check that `file` is not a directory, as on Windows
        # directories pass the os.access check.
        def _access_check(fn, mode):
            return (os.path.exists(fn) and os.access(fn, mode) and
                    not os.path.isdir(fn))

        # If we're given a path with a directory part, look it up directly
        # rather than referring to PATH directories. This includes checking
        # relative to the current directory, e.g. ./script
        if os.path.dirname(cmd):
            if _access_check(cmd, mode):
                return cmd
            return None

        if path is None:
            path = os.environ.get("PATH", os.defpath)
        if not path:
            return None
        path = path.split(os.pathsep)

        if sys.platform == "win32":
            # The current directory takes precedence on Windows.
            if os.curdir not in path:
                path.insert(0, os.curdir)

            # PATHEXT is necessary to check on Windows.
            pathext = os.environ.get("PATHEXT", "").split(os.pathsep)
            # See if the given file matches any of the expected path
            # extensions. This will allow us to short circuit when given
            # "python.exe". If it does match, only test that one, otherwise we
            # have to try others.
            if any(cmd.lower().endswith(ext.lower()) for ext in pathext):
                files = [cmd]
            else:
                files = [cmd + ext for ext in pathext]
        else:
            # On other platforms you don't have things like PATHEXT to tell you
            # what file suffixes are executable, so just pass on cmd as-is.
            files = [cmd]

        seen = set()
        for dir in path:
            normdir = os.path.normcase(dir)
            if normdir not in seen:
                seen.add(normdir)
                for thefile in files:
                    name = os.path.join(dir, thefile)
                    if _access_check(name, mode):
                        return name
        return None

def is_string(obj):
    """Determine if an object is a string."""
    return isinstance(obj, str if PY3 else basestring)

_hush_pyflakes = (patch, StringIO, which)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
import os
import argparse
import sys
from time import sleep
import subprocess
import imp

parser = argparse.ArgumentParser(
        description="This is a python3 script to\
        find the lowest energy frame and cooresponding pdb")

parser.add_argument("protein", help="The name of the protein")
# parser.add_argument("template", help="the name of template file")
# parser.add_argument("-n", "--number", type=int, default=20,
#                     help="Number of simulation run")
# parser.add_argument("-m", "--movie", type=int, default=-1,
#                     help="generate the movie,defalut is all")
# parser.add_argument("-p", "--plotOnly", help="only generate the plot",
#                     action="store_true")
# parser.add_argument("-s", "--steps", type=int, default=4,
#                     help="Simulation steps in unit of million,\
#                     default is 4 million, -1 means test run")
parser.add_argument("-o", "--offAuto", help="turn off from Read from \
                    config file", action="store_true")
args = parser.parse_args()

list_of_lowest_potential_energy = []
protein_name = args.protein.split('.')[0]
# n = args.number
# steps = args.steps*1000*1000
# if args.steps == -1:
#     n = 1  # also set n to be 1 ,this is for debug
#     steps = 10*1000
# # imp.load_source('run_paramter.py', '')
if(not args.offAuto):
    exec (open("config.py").read())
    # print(n, x, y, type(y))
    n = number_of_run
    steps = simulation_steps
    # print(n, steps)
    # sys.exit(0)

for i in range(n):
    os.chdir("simulation/"+str(i))
    sys.stdout = open("lowest_energy.txt", "w")
    record_time = 0
    with open('energy.log') as input_data:
        # Skips text before the beginning of the interesting block:
        next(input_data)
        record_time = 0
        lowest_potential_energy = 0
        last_potential_energy = 0
        next(input_data)
        for line in input_data:
            # time, q = line.strip().split()

            time = int(line.strip().split()[0])
            potenial_energy = float(line.strip().split()[-1])
            # print(time, potenial_energy)
            if(potenial_energy < lowest_potential_energy):
                record_time = time
                lowest_potential_energy = potenial_energy
            last_potential_energy = potenial_energy
        list_of_lowest_potential_energy += [(lowest_potential_energy, record_time, last_potential_energy)]
    time_step = record_time

    print('ITEM: TIMESTEP')
    with open('dump.lammpstrj') as input_data:
        # Skips text before the beginning of the interesting block:
        for line in input_data:
            if line.strip() == str(time_step):
                print(line.strip())  # Or whatever test is needed
                break
        # Reads text until the end of the block:
        for line in input_data:  # This keeps reading the file
            if line.strip() == 'ITEM: TIMESTEP':
                break
            print(line.strip())
    sys.stdout.close()
    os.system(
        "python2 ~/opt/script/BuildAllAtomsFromLammps_seq.py \
        lowest_energy.txt lowest_energy "+protein_name+".seq")
    os.chdir("../..")
sys.stdout = open("list_of_lowest_potential_energy", "w")
# store the globally loweest energy frame
global_min_idx = -1
energy_temp = 0
for idx, q in enumerate(list_of_lowest_potential_energy):
    print(q[0], q[1], q[2], idx)  # max q, timestep of max q, last q
    if(q[0] < energy_temp):
        energy_temp = q[0]
        global_min_idx = idx
sys.stdout.close()
os.system("cp simulation/{}/lowest_energy.pdb global_lowest_energy.pdb".format(global_min_idx))
<commit_msg>bug fix<commit_after>#!/usr/bin/env python3
import os
import argparse
import sys
from time import sleep
import subprocess
import imp

parser = argparse.ArgumentParser(
        description="This is a python3 script to\
        find the lowest energy frame and cooresponding pdb")

parser.add_argument("protein", help="The name of the protein")
# parser.add_argument("template", help="the name of template file")
# parser.add_argument("-n", "--number", type=int, default=20,
#                     help="Number of simulation run")
# parser.add_argument("-m", "--movie", type=int, default=-1,
#                     help="generate the movie,defalut is all")
# parser.add_argument("-p", "--plotOnly", help="only generate the plot",
#                     action="store_true")
# parser.add_argument("-s", "--steps", type=int, default=4,
#                     help="Simulation steps in unit of million,\
#                     default is 4 million, -1 means test run")
parser.add_argument("-o", "--offAuto", help="turn off from Read from \
                    config file", action="store_true")
args = parser.parse_args()

list_of_lowest_potential_energy = []
protein_name = args.protein.split('.')[0]
# n = args.number
# steps = args.steps*1000*1000
# if args.steps == -1:
#     n = 1  # also set n to be 1 ,this is for debug
#     steps = 10*1000
# # imp.load_source('run_paramter.py', '')
if(not args.offAuto):
    exec (open("config.py").read())
    # print(n, x, y, type(y))
    n = number_of_run
    steps = simulation_steps
    # print(n, steps)
    # sys.exit(0)

for i in range(n):
    os.chdir("simulation_iteration_1/"+str(i))
    sys.stdout = open("lowest_energy.txt", "w")
    record_time = 0
    with open('energy.log') as input_data:
        # Skips text before the beginning of the interesting block:
        next(input_data)
        record_time = 0
        lowest_potential_energy = 0
        last_potential_energy = 0
        next(input_data)
        for line in input_data:
            # time, q = line.strip().split()

            time = int(line.strip().split()[0])
            potenial_energy = float(line.strip().split()[-1])
            # print(time, potenial_energy)
            if(potenial_energy < lowest_potential_energy):
                record_time = time
                lowest_potential_energy = potenial_energy
            last_potential_energy = potenial_energy
        list_of_lowest_potential_energy += [(lowest_potential_energy, record_time, last_potential_energy)]
    time_step = record_time

    print('ITEM: TIMESTEP')
    with open('dump.lammpstrj') as input_data:
        # Skips text before the beginning of the interesting block:
        for line in input_data:
            if line.strip() == str(time_step):
                print(line.strip())  # Or whatever test is needed
                break
        # Reads text until the end of the block:
        for line in input_data:  # This keeps reading the file
            if line.strip() == 'ITEM: TIMESTEP':
                break
            print(line.strip())
    sys.stdout.close()
    os.system(
        "python2 ~/opt/script/BuildAllAtomsFromLammps_seq.py \
        lowest_energy.txt lowest_energy "+protein_name+".seq")
    os.chdir("../..")
sys.stdout = open("list_of_lowest_potential_energy", "w")
# store the globally loweest energy frame
global_min_idx = -1
energy_temp = 0
for idx, q in enumerate(list_of_lowest_potential_energy):
    print(q[0], q[1], q[2], idx)  # max q, timestep of max q, last q
    if(q[0] < energy_temp):
        energy_temp = q[0]
        global_min_idx = idx
sys.stdout.close()
os.system("cp simulation/{}/lowest_energy.pdb global_lowest_energy.pdb".format(global_min_idx))
<|endoftext|>


<commit_before>#!/usr/bin/python
import lutin.module as module
import lutin.tools as tools
import lutin.debug as debug
import os
import lutin.multiprocess as lutinMultiprocess

def get_type():
	return "LIBRARY"

def get_desc():
	return "esignal is signal management for all class"

def get_licence():
	return "APACHE-2"

def get_compagny_type():
	return "com"

def get_compagny_name():
	return "atria-soft"

def get_maintainer():
	return ["Mr DUPIN Edouard <yui.heero@gmail.com>"]

def get_version():
	return [0,3,0]

def create(target, module_name):
	my_module = module.Module(__file__, module_name, get_type())
	my_module.add_extra_compile_flags()
	my_module.add_src_file([
		'esignal/debug.cpp',
		'esignal/Connection.cpp',
		'esignal/InterfaceData.cpp',
		'esignal/Base.cpp',
		'esignal/details/Signal.cpp',
		])
	my_module.add_header_file([
		'esignal/debug.h',
		'esignal/Interface.h',
		'esignal/InterfaceData.h',
		'esignal/Base.h',
		'esignal/Signal.h',
		'esignal/Connection.h',
		'esignal/details/Signal.hxx',
		])
	my_module.compile_version("c++", 2011)
	my_module.add_module_depend([
	    'etk',
	    'ememory'
	    ])
	my_module.add_path(tools.get_current_path(__file__))
	my_module.compile_flags('c++', [
		"-DESIGNAL_VERSION=\"\\\"" + tools.version_to_string(get_version()) + "\\\"\""
		])
	my_module.add_tools(['esignal-test'])
	return my_module

<commit_msg>[DEV] update dev tag version<commit_after>#!/usr/bin/python
import lutin.module as module
import lutin.tools as tools
import lutin.debug as debug
import os
import lutin.multiprocess as lutinMultiprocess

def get_type():
	return "LIBRARY"

def get_desc():
	return "esignal is signal management for all class"

def get_licence():
	return "APACHE-2"

def get_compagny_type():
	return "com"

def get_compagny_name():
	return "atria-soft"

def get_maintainer():
	return ["Mr DUPIN Edouard <yui.heero@gmail.com>"]

def get_version():
	return [0,3,"dev"]

def create(target, module_name):
	my_module = module.Module(__file__, module_name, get_type())
	my_module.add_extra_compile_flags()
	my_module.add_src_file([
		'esignal/debug.cpp',
		'esignal/Connection.cpp',
		'esignal/InterfaceData.cpp',
		'esignal/Base.cpp',
		'esignal/details/Signal.cpp',
		])
	my_module.add_header_file([
		'esignal/debug.h',
		'esignal/Interface.h',
		'esignal/InterfaceData.h',
		'esignal/Base.h',
		'esignal/Signal.h',
		'esignal/Connection.h',
		'esignal/details/Signal.hxx',
		])
	my_module.compile_version("c++", 2011)
	my_module.add_module_depend([
	    'etk',
	    'ememory'
	    ])
	my_module.add_path(tools.get_current_path(__file__))
	my_module.compile_flags('c++', [
		"-DESIGNAL_VERSION=\"\\\"" + tools.version_to_string(get_version()) + "\\\"\""
		])
	my_module.add_tools(['esignal-test'])
	return my_module

<|endoftext|>


<commit_before>#!/usr/bin/env python
# Khosrow Ebrahimpour - Sep 2012

"""
lvsm - LVS Manager
LVS Manager is a shell that eases the management of a linux virtual server.

Using it without arguments will enter an interactive shell. Supplying one or
more command-line arguments will run lvsm for a "single-shot" use.

Usage: lvsm [-h] [-c <conffile>][commands]

Options:
  -h, --help            Show this help message and exit

  -c <conffile>         Specify which configuration file to use.
  -config=<connfile>    The default is /etc/lvsm.conf

Commands:
  configure
  status
  help

Use 'lvsm help <command>' for information on a specific command.
"""

import getopt
import sys
import __init__ as appinfo
import lvsm
import utils

def usage(code, msg=''):
    if code:
        fd = sys.stderr
    else:
        fd = sys.stdout
    print >> fd, __doc__
    if msg:
        print >> fd, msg
    sys.exit(code)

def main():
    CONFFILE = "/etc/lvsm.conf"

    try:
        opts, args = getopt.getopt(sys.argv[1:], "hvc:d",
                                   ["help", "version", "config=", "debug"])
    except getopt.error, msg:
        usage(2, msg)

    for opt, arg in opts:
        if opt in ("-h", "--help"):
            usage(0)
        elif opt in ("-v", "--version"):
            print appinfo.__version__
        elif opt in ("-c", "--config"):
            CONFFILE = arg
        elif opt in ("-d", "--debug"):
            utils.DEBUG = True

    #open config file and read it
    config = utils.parse_config(CONFFILE)
    utils.log("Parsed config file")
    utils.log(str(config))

    shell = lvsm.MainPrompt(config)
    if args:
        shell.onecmd(' '.join(args[:]))
    else:
        shell.cmdloop()

if __name__ == "__main__":
    main()
<commit_msg>Fixed --version option<commit_after>#!/usr/bin/env python
# Khosrow Ebrahimpour - Sep 2012

"""
lvsm - LVS Manager
LVS Manager is a shell that eases the management of a linux virtual server.

Using it without arguments will enter an interactive shell. Supplying one or
more command-line arguments will run lvsm for a "single-shot" use.

Usage: lvsm [-h] [-c <conffile>][commands]

Options:
  -h, --help            Show this help message and exit

  -c <conffile>         Specify which configuration file to use.
  -config=<connfile>    The default is /etc/lvsm.conf

Commands:
  configure
  status
  help

Use 'lvsm help <command>' for information on a specific command.
"""

import getopt
import sys
import __init__ as appinfo
import lvsm
import utils

def usage(code, msg=''):
    if code:
        fd = sys.stderr
    else:
        fd = sys.stdout
    print >> fd, __doc__
    if msg:
        print >> fd, msg
    sys.exit(code)

def main():
    CONFFILE = "/etc/lvsm.conf"

    try:
        opts, args = getopt.getopt(sys.argv[1:], "hvc:d",
                                   ["help", "version", "config=", "debug"])
    except getopt.error, msg:
        usage(2, msg)

    for opt, arg in opts:
        if opt in ("-h", "--help"):
            usage(0)
        elif opt in ("-v", "--version"):
            print appinfo.__version__
            sys.exit(0)
        elif opt in ("-c", "--config"):
            CONFFILE = arg
        elif opt in ("-d", "--debug"):
            utils.DEBUG = True

    #open config file and read it
    config = utils.parse_config(CONFFILE)
    utils.log("Parsed config file")
    utils.log(str(config))

    shell = lvsm.MainPrompt(config)
    if args:
        shell.onecmd(' '.join(args[:]))
    else:
        shell.cmdloop()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># ----------------------------------------------------------------------------
# Copyright (c) 2013--, scikit-bio development team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
# ----------------------------------------------------------------------------

from __future__ import absolute_import, division, print_function
import six

from io import StringIO
from unittest import TestCase, main

import numpy as np
import numpy.testing as npt

from skbio.io import OrdinationFormatError
from skbio.io.format.ordination import (
    _ordination_to_ordination_results, _ordination_results_to_ordination,
    _ordination_sniffer)
from skbio.stats.ordination import (
    OrdinationResults, assert_ordination_results_equal)
from skbio.util import get_data_path

class OrdinationTestData(TestCase):
    def setUp(self):
        self.valid_fps = map(
            get_data_path,
            ['ordination_L&L_CA_data_scores', 'ordination_example3_scores',
             'ordination_PCoA_sample_data_3_scores',
             'ordination_example2_scores'])

        # Store filepath, regex for matching the error message that should be
        # raised when reading the file, and whether the file should be matched
        # by the sniffer (True) or not (False).
        self.invalid_fps = map(lambda e: (get_data_path(e[0]), e[1], e[2]), [
            ('empty', 'end of file.*Eigvals header', False),
            ('whitespace_only', 'Eigvals header not found', False),
            ('ordination_error1', 'Eigvals header not found', False),
            ('ordination_error2',
             'Proportion explained header not found', False),
            ('ordination_error3', 'Species header not found', True),
            ('ordination_error4', 'Site header not found', True),
            ('ordination_error5', 'Biplot header not found', True),
            ('ordination_error6', 'Site constraints header not found', True),
            ('ordination_error7', 'empty line', False),
            ('ordination_error8', '9.*Proportion explained.*8', True),
            ('ordination_error9', '2 values.*1 in row 1', True),
            ('ordination_error10', '2 values.*1 in row 1', True),
            ('ordination_error11', 'Site constraints ids and site ids', True),
            ('ordination_error12', '9.*Eigvals.*8', True),
            ('ordination_error13', '9.*Proportion explained.*8', True),
            ('ordination_error14', 'Site is 0: 9 x 0', True),
            ('ordination_error15', '9 values.*8 in row 1', True),
            ('ordination_error16', 'Biplot is 0: 3 x 0', True),
            ('ordination_error17', '3 values.*2 in row 1', True),
            ('ordination_error18',
             'proportion explained.*eigvals: 8 != 9', True),
            ('ordination_error19',
             'coordinates.*species.*eigvals: 1 != 2', True),
            ('ordination_error20', 'coordinates.*site.*eigvals: 1 != 2', True),
            ('ordination_error21', 'one eigval', False),
            ('ordination_error22', 'end of file.*blank line', False),
            ('ordination_error23', 'end of file.*Proportion explained section',
             True),
            ('ordination_error24', 'end of file.*row 2.*Species section', True)
        ])

class OrdinationResultsReaderWriterTests(OrdinationTestData):
    def setUp(self):
        super(OrdinationResultsReaderWriterTests, self).setUp()

        # define in-memory results, one for each of the valid files in
        # self.valid_fps

        # CA results
        eigvals = np.array([0.0961330159181, 0.0409418140138])
        species = np.array([[0.408869425742, 0.0695518116298],
                            [-0.1153860437, -0.299767683538],
                            [-0.309967102571, 0.187391917117]])
        site = np.array([[-0.848956053187, 0.882764759014],
                         [-0.220458650578, -1.34482000302],
                         [1.66697179591, 0.470324389808]])
        biplot = None
        site_constraints = None
        prop_explained = None
        species_ids = ['Species1', 'Species2', 'Species3']
        site_ids = ['Site1', 'Site2', 'Site3']
        ca_scores = OrdinationResults(eigvals=eigvals, species=species,
                                      site=site, biplot=biplot,
                                      site_constraints=site_constraints,
                                      proportion_explained=prop_explained,
                                      species_ids=species_ids,
                                      site_ids=site_ids)
        # CCA results
        eigvals = np.array([0.366135830393, 0.186887643052, 0.0788466514249,
                            0.082287840501, 0.0351348475787, 0.0233265839374,
                            0.0099048981912, 0.00122461669234,
                            0.000417454724117])
        species = np.loadtxt(
            get_data_path('ordination_exp_Ordination_CCA_species'))
        site = np.loadtxt(get_data_path('ordination_exp_Ordination_CCA_site'))
        biplot = np.array([[-0.169746767979, 0.63069090084, 0.760769036049],
                           [-0.994016563505, 0.0609533148724,
                            -0.0449369418179],
                           [0.184352565909, -0.974867543612, 0.0309865007541]])
        site_constraints = np.loadtxt(
            get_data_path('ordination_exp_Ordination_CCA_site_constraints'))
        prop_explained = None
        species_ids = ['Species0', 'Species1', 'Species2', 'Species3',
                       'Species4', 'Species5', 'Species6', 'Species7',
                       'Species8']
        site_ids = ['Site0', 'Site1', 'Site2', 'Site3', 'Site4', 'Site5',
                    'Site6', 'Site7', 'Site8', 'Site9']
        cca_scores = OrdinationResults(eigvals=eigvals, species=species,
                                       site=site, biplot=biplot,
                                       site_constraints=site_constraints,
                                       proportion_explained=prop_explained,
                                       species_ids=species_ids,
                                       site_ids=site_ids)
        # PCoA results
        eigvals = np.array([0.512367260461, 0.300719094427, 0.267912066004,
                            0.208988681078, 0.19169895326, 0.16054234528,
                            0.15017695712, 0.122457748167, 0.0])
        species = None
        site = np.loadtxt(get_data_path('ordination_exp_Ordination_PCoA_site'))
        biplot = None
        site_constraints = None
        prop_explained = np.array([0.267573832777, 0.15704469605,
                                   0.139911863774, 0.109140272454,
                                   0.100111048503, 0.0838401161912,
                                   0.0784269939011, 0.0639511763509, 0.0])
        species_ids = None
        site_ids = ['PC.636', 'PC.635', 'PC.356', 'PC.481', 'PC.354', 'PC.593',
                    'PC.355', 'PC.607', 'PC.634']
        pcoa_scores = OrdinationResults(eigvals=eigvals, species=species,
                                        site=site, biplot=biplot,
                                        site_constraints=site_constraints,
                                        proportion_explained=prop_explained,
                                        species_ids=species_ids,
                                        site_ids=site_ids)
        # RDA results
        eigvals = np.array([25.8979540892, 14.9825779819, 8.93784077262,
                            6.13995623072, 1.68070536498, 0.57735026919,
                            0.275983624351])
        species = np.loadtxt(
            get_data_path('ordination_exp_Ordination_RDA_species'))
        site = np.loadtxt(get_data_path('ordination_exp_Ordination_RDA_site'))
        biplot = np.array([[0.422650019179, -0.559142585857, -0.713250678211],
                           [0.988495963777, 0.150787422017, -0.0117848614073],
                           [-0.556516618887, 0.817599992718, 0.147714267459],
                           [-0.404079676685, -0.9058434809, -0.127150316558]])
        site_constraints = np.loadtxt(
            get_data_path('ordination_exp_Ordination_RDA_site_constraints'))
        prop_explained = None
        species_ids = ['Species0', 'Species1', 'Species2', 'Species3',
                       'Species4', 'Species5']
        site_ids = ['Site0', 'Site1', 'Site2', 'Site3', 'Site4', 'Site5',
                    'Site6', 'Site7', 'Site8', 'Site9']
        rda_scores = OrdinationResults(eigvals=eigvals, species=species,
                                       site=site, biplot=biplot,
                                       site_constraints=site_constraints,
                                       proportion_explained=prop_explained,
                                       species_ids=species_ids,
                                       site_ids=site_ids)

        self.ordination_results_objs = [ca_scores, cca_scores, pcoa_scores,
                                        rda_scores]

    def test_read_valid_files(self):
        for fp, obj in zip(self.valid_fps, self.ordination_results_objs):
                obs = _ordination_to_ordination_results(fp)
                assert_ordination_results_equal(obs, obj)

    def test_read_invalid_files(self):
        for invalid_fp, error_msg_regexp, _ in self.invalid_fps:
            with six.assertRaisesRegex(self, OrdinationFormatError,
                                       error_msg_regexp):
                _ordination_to_ordination_results(invalid_fp)

    def test_write(self):
        for fp, obj in zip(self.valid_fps, self.ordination_results_objs):
            fh = StringIO()
            _ordination_results_to_ordination(obj, fh)
            obs = fh.getvalue()
            fh.close()

            with open(fp, 'U') as fh:
                exp = fh.read()

            npt.assert_equal(obs, exp)

    def test_roundtrip_read_write(self):
        for fp in self.valid_fps:
            # Read.
            obj1 = _ordination_to_ordination_results(fp)

            # Write.
            fh = StringIO()
            _ordination_results_to_ordination(obj1, fh)
            fh.seek(0)

            # Read.
            obj2 = _ordination_to_ordination_results(fh)
            fh.close()

            assert_ordination_results_equal(obj1, obj2)

class SnifferTests(OrdinationTestData):
    def setUp(self):
        super(SnifferTests, self).setUp()

    def test_matches_and_nonmatches(self):
        # Sniffer should match all valid files, and will match some invalid
        # ones too because it doesn't exhaustively check the entire file.
        for fp in self.valid_fps:
            self.assertEqual(_ordination_sniffer(fp), (True, {}))

        for fp, _, expected_sniffer_match in self.invalid_fps:
            self.assertEqual(_ordination_sniffer(fp),
                             (expected_sniffer_match, {}))

if __name__ == '__main__':
    main()
<commit_msg>ENH: test_ordination no longer uses deprecated open<commit_after># ----------------------------------------------------------------------------
# Copyright (c) 2013--, scikit-bio development team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
# ----------------------------------------------------------------------------

from __future__ import absolute_import, division, print_function
import six

import io
from unittest import TestCase, main

import numpy as np
import numpy.testing as npt

from skbio.io import OrdinationFormatError
from skbio.io.format.ordination import (
    _ordination_to_ordination_results, _ordination_results_to_ordination,
    _ordination_sniffer)
from skbio.stats.ordination import (
    OrdinationResults, assert_ordination_results_equal)
from skbio.util import get_data_path

class OrdinationTestData(TestCase):
    def setUp(self):
        self.valid_fps = map(
            get_data_path,
            ['ordination_L&L_CA_data_scores', 'ordination_example3_scores',
             'ordination_PCoA_sample_data_3_scores',
             'ordination_example2_scores'])

        # Store filepath, regex for matching the error message that should be
        # raised when reading the file, and whether the file should be matched
        # by the sniffer (True) or not (False).
        self.invalid_fps = map(lambda e: (get_data_path(e[0]), e[1], e[2]), [
            ('empty', 'end of file.*Eigvals header', False),
            ('whitespace_only', 'Eigvals header not found', False),
            ('ordination_error1', 'Eigvals header not found', False),
            ('ordination_error2',
             'Proportion explained header not found', False),
            ('ordination_error3', 'Species header not found', True),
            ('ordination_error4', 'Site header not found', True),
            ('ordination_error5', 'Biplot header not found', True),
            ('ordination_error6', 'Site constraints header not found', True),
            ('ordination_error7', 'empty line', False),
            ('ordination_error8', '9.*Proportion explained.*8', True),
            ('ordination_error9', '2 values.*1 in row 1', True),
            ('ordination_error10', '2 values.*1 in row 1', True),
            ('ordination_error11', 'Site constraints ids and site ids', True),
            ('ordination_error12', '9.*Eigvals.*8', True),
            ('ordination_error13', '9.*Proportion explained.*8', True),
            ('ordination_error14', 'Site is 0: 9 x 0', True),
            ('ordination_error15', '9 values.*8 in row 1', True),
            ('ordination_error16', 'Biplot is 0: 3 x 0', True),
            ('ordination_error17', '3 values.*2 in row 1', True),
            ('ordination_error18',
             'proportion explained.*eigvals: 8 != 9', True),
            ('ordination_error19',
             'coordinates.*species.*eigvals: 1 != 2', True),
            ('ordination_error20', 'coordinates.*site.*eigvals: 1 != 2', True),
            ('ordination_error21', 'one eigval', False),
            ('ordination_error22', 'end of file.*blank line', False),
            ('ordination_error23', 'end of file.*Proportion explained section',
             True),
            ('ordination_error24', 'end of file.*row 2.*Species section', True)
        ])

class OrdinationResultsReaderWriterTests(OrdinationTestData):
    def setUp(self):
        super(OrdinationResultsReaderWriterTests, self).setUp()

        # define in-memory results, one for each of the valid files in
        # self.valid_fps

        # CA results
        eigvals = np.array([0.0961330159181, 0.0409418140138])
        species = np.array([[0.408869425742, 0.0695518116298],
                            [-0.1153860437, -0.299767683538],
                            [-0.309967102571, 0.187391917117]])
        site = np.array([[-0.848956053187, 0.882764759014],
                         [-0.220458650578, -1.34482000302],
                         [1.66697179591, 0.470324389808]])
        biplot = None
        site_constraints = None
        prop_explained = None
        species_ids = ['Species1', 'Species2', 'Species3']
        site_ids = ['Site1', 'Site2', 'Site3']
        ca_scores = OrdinationResults(eigvals=eigvals, species=species,
                                      site=site, biplot=biplot,
                                      site_constraints=site_constraints,
                                      proportion_explained=prop_explained,
                                      species_ids=species_ids,
                                      site_ids=site_ids)
        # CCA results
        eigvals = np.array([0.366135830393, 0.186887643052, 0.0788466514249,
                            0.082287840501, 0.0351348475787, 0.0233265839374,
                            0.0099048981912, 0.00122461669234,
                            0.000417454724117])
        species = np.loadtxt(
            get_data_path('ordination_exp_Ordination_CCA_species'))
        site = np.loadtxt(get_data_path('ordination_exp_Ordination_CCA_site'))
        biplot = np.array([[-0.169746767979, 0.63069090084, 0.760769036049],
                           [-0.994016563505, 0.0609533148724,
                            -0.0449369418179],
                           [0.184352565909, -0.974867543612, 0.0309865007541]])
        site_constraints = np.loadtxt(
            get_data_path('ordination_exp_Ordination_CCA_site_constraints'))
        prop_explained = None
        species_ids = ['Species0', 'Species1', 'Species2', 'Species3',
                       'Species4', 'Species5', 'Species6', 'Species7',
                       'Species8']
        site_ids = ['Site0', 'Site1', 'Site2', 'Site3', 'Site4', 'Site5',
                    'Site6', 'Site7', 'Site8', 'Site9']
        cca_scores = OrdinationResults(eigvals=eigvals, species=species,
                                       site=site, biplot=biplot,
                                       site_constraints=site_constraints,
                                       proportion_explained=prop_explained,
                                       species_ids=species_ids,
                                       site_ids=site_ids)
        # PCoA results
        eigvals = np.array([0.512367260461, 0.300719094427, 0.267912066004,
                            0.208988681078, 0.19169895326, 0.16054234528,
                            0.15017695712, 0.122457748167, 0.0])
        species = None
        site = np.loadtxt(get_data_path('ordination_exp_Ordination_PCoA_site'))
        biplot = None
        site_constraints = None
        prop_explained = np.array([0.267573832777, 0.15704469605,
                                   0.139911863774, 0.109140272454,
                                   0.100111048503, 0.0838401161912,
                                   0.0784269939011, 0.0639511763509, 0.0])
        species_ids = None
        site_ids = ['PC.636', 'PC.635', 'PC.356', 'PC.481', 'PC.354', 'PC.593',
                    'PC.355', 'PC.607', 'PC.634']
        pcoa_scores = OrdinationResults(eigvals=eigvals, species=species,
                                        site=site, biplot=biplot,
                                        site_constraints=site_constraints,
                                        proportion_explained=prop_explained,
                                        species_ids=species_ids,
                                        site_ids=site_ids)
        # RDA results
        eigvals = np.array([25.8979540892, 14.9825779819, 8.93784077262,
                            6.13995623072, 1.68070536498, 0.57735026919,
                            0.275983624351])
        species = np.loadtxt(
            get_data_path('ordination_exp_Ordination_RDA_species'))
        site = np.loadtxt(get_data_path('ordination_exp_Ordination_RDA_site'))
        biplot = np.array([[0.422650019179, -0.559142585857, -0.713250678211],
                           [0.988495963777, 0.150787422017, -0.0117848614073],
                           [-0.556516618887, 0.817599992718, 0.147714267459],
                           [-0.404079676685, -0.9058434809, -0.127150316558]])
        site_constraints = np.loadtxt(
            get_data_path('ordination_exp_Ordination_RDA_site_constraints'))
        prop_explained = None
        species_ids = ['Species0', 'Species1', 'Species2', 'Species3',
                       'Species4', 'Species5']
        site_ids = ['Site0', 'Site1', 'Site2', 'Site3', 'Site4', 'Site5',
                    'Site6', 'Site7', 'Site8', 'Site9']
        rda_scores = OrdinationResults(eigvals=eigvals, species=species,
                                       site=site, biplot=biplot,
                                       site_constraints=site_constraints,
                                       proportion_explained=prop_explained,
                                       species_ids=species_ids,
                                       site_ids=site_ids)

        self.ordination_results_objs = [ca_scores, cca_scores, pcoa_scores,
                                        rda_scores]

    def test_read_valid_files(self):
        for fp, obj in zip(self.valid_fps, self.ordination_results_objs):
                obs = _ordination_to_ordination_results(fp)
                assert_ordination_results_equal(obs, obj)

    def test_read_invalid_files(self):
        for invalid_fp, error_msg_regexp, _ in self.invalid_fps:
            with six.assertRaisesRegex(self, OrdinationFormatError,
                                       error_msg_regexp):
                _ordination_to_ordination_results(invalid_fp)

    def test_write(self):
        for fp, obj in zip(self.valid_fps, self.ordination_results_objs):
            fh = io.StringIO()
            _ordination_results_to_ordination(obj, fh)
            obs = fh.getvalue()
            fh.close()

            with io.open(fp) as fh:
                exp = fh.read()

            npt.assert_equal(obs, exp)

    def test_roundtrip_read_write(self):
        for fp in self.valid_fps:
            # Read.
            obj1 = _ordination_to_ordination_results(fp)

            # Write.
            fh = io.StringIO()
            _ordination_results_to_ordination(obj1, fh)
            fh.seek(0)

            # Read.
            obj2 = _ordination_to_ordination_results(fh)
            fh.close()

            assert_ordination_results_equal(obj1, obj2)

class SnifferTests(OrdinationTestData):
    def setUp(self):
        super(SnifferTests, self).setUp()

    def test_matches_and_nonmatches(self):
        # Sniffer should match all valid files, and will match some invalid
        # ones too because it doesn't exhaustively check the entire file.
        for fp in self.valid_fps:
            self.assertEqual(_ordination_sniffer(fp), (True, {}))

        for fp, _, expected_sniffer_match in self.invalid_fps:
            self.assertEqual(_ordination_sniffer(fp),
                             (expected_sniffer_match, {}))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
Testing for mean shift clustering methods

"""

import numpy as np
import warnings
import pytest

from scipy import sparse

from sklearn.utils._testing import assert_array_equal
from sklearn.utils._testing import assert_array_almost_equal
from sklearn.utils._testing import assert_allclose

from sklearn.cluster import MeanShift
from sklearn.cluster import mean_shift
from sklearn.cluster import estimate_bandwidth
from sklearn.cluster import get_bin_seeds
from sklearn.datasets import make_blobs
from sklearn.metrics import v_measure_score

n_clusters = 3
centers = np.array([[1, 1], [-1, -1], [1, -1]]) + 10
X, _ = make_blobs(
    n_samples=300,
    n_features=2,
    centers=centers,
    cluster_std=0.4,
    shuffle=True,
    random_state=11,
)

def test_estimate_bandwidth():
    # Test estimate_bandwidth
    bandwidth = estimate_bandwidth(X, n_samples=200)
    assert 0.9 <= bandwidth <= 1.5

def test_estimate_bandwidth_1sample():
    # Test estimate_bandwidth when n_samples=1 and quantile<1, so that
    # n_neighbors is set to 1.
    bandwidth = estimate_bandwidth(X, n_samples=1, quantile=0.3)
    assert bandwidth == pytest.approx(0.0, abs=1e-5)

@pytest.mark.parametrize(
    "bandwidth, cluster_all, expected, first_cluster_label",
    [(1.2, True, 3, 0), (1.2, False, 4, -1)],
)
def test_mean_shift(bandwidth, cluster_all, expected, first_cluster_label):
    # Test MeanShift algorithm
    ms = MeanShift(bandwidth=bandwidth, cluster_all=cluster_all)
    labels = ms.fit(X).labels_
    labels_unique = np.unique(labels)
    n_clusters_ = len(labels_unique)
    assert n_clusters_ == expected
    assert labels_unique[0] == first_cluster_label

    cluster_centers, labels_mean_shift = mean_shift(X, cluster_all=cluster_all)
    labels_mean_shift_unique = np.unique(labels_mean_shift)
    n_clusters_mean_shift = len(labels_mean_shift_unique)
    assert n_clusters_mean_shift == expected
    assert labels_mean_shift_unique[0] == first_cluster_label

def test_mean_shift_negative_bandwidth():
    bandwidth = -1
    ms = MeanShift(bandwidth=bandwidth)
    msg = r"bandwidth needs to be greater than zero or None," r" got -1\.000000"
    with pytest.raises(ValueError, match=msg):
        ms.fit(X)

def test_estimate_bandwidth_with_sparse_matrix():
    # Test estimate_bandwidth with sparse matrix
    X = sparse.lil_matrix((1000, 1000))
    msg = "A sparse matrix was passed, but dense data is required."
    with pytest.raises(TypeError, match=msg):
        estimate_bandwidth(X)

def test_parallel():
    centers = np.array([[1, 1], [-1, -1], [1, -1]]) + 10
    X, _ = make_blobs(
        n_samples=50,
        n_features=2,
        centers=centers,
        cluster_std=0.4,
        shuffle=True,
        random_state=11,
    )

    ms1 = MeanShift(n_jobs=2)
    ms1.fit(X)

    ms2 = MeanShift()
    ms2.fit(X)

    assert_array_almost_equal(ms1.cluster_centers_, ms2.cluster_centers_)
    assert_array_equal(ms1.labels_, ms2.labels_)

def test_meanshift_predict():
    # Test MeanShift.predict
    ms = MeanShift(bandwidth=1.2)
    labels = ms.fit_predict(X)
    labels2 = ms.predict(X)
    assert_array_equal(labels, labels2)

def test_meanshift_all_orphans():
    # init away from the data, crash with a sensible warning
    ms = MeanShift(bandwidth=0.1, seeds=[[-9, -9], [-10, -10]])
    msg = "No point was within bandwidth=0.1"
    with pytest.raises(ValueError, match=msg):
        ms.fit(
            X,
        )

def test_unfitted():
    # Non-regression: before fit, there should be not fitted attributes.
    ms = MeanShift()
    assert not hasattr(ms, "cluster_centers_")
    assert not hasattr(ms, "labels_")

def test_cluster_intensity_tie():
    X = np.array([[1, 1], [2, 1], [1, 0], [4, 7], [3, 5], [3, 6]])
    c1 = MeanShift(bandwidth=2).fit(X)

    X = np.array([[4, 7], [3, 5], [3, 6], [1, 1], [2, 1], [1, 0]])
    c2 = MeanShift(bandwidth=2).fit(X)
    assert_array_equal(c1.labels_, [1, 1, 1, 0, 0, 0])
    assert_array_equal(c2.labels_, [0, 0, 0, 1, 1, 1])

def test_bin_seeds():
    # Test the bin seeding technique which can be used in the mean shift
    # algorithm
    # Data is just 6 points in the plane
    X = np.array(
        [[1.0, 1.0], [1.4, 1.4], [1.8, 1.2], [2.0, 1.0], [2.1, 1.1], [0.0, 0.0]]
    )

    # With a bin coarseness of 1.0 and min_bin_freq of 1, 3 bins should be
    # found
    ground_truth = {(1.0, 1.0), (2.0, 1.0), (0.0, 0.0)}
    test_bins = get_bin_seeds(X, 1, 1)
    test_result = set(tuple(p) for p in test_bins)
    assert len(ground_truth.symmetric_difference(test_result)) == 0

    # With a bin coarseness of 1.0 and min_bin_freq of 2, 2 bins should be
    # found
    ground_truth = {(1.0, 1.0), (2.0, 1.0)}
    test_bins = get_bin_seeds(X, 1, 2)
    test_result = set(tuple(p) for p in test_bins)
    assert len(ground_truth.symmetric_difference(test_result)) == 0

    # With a bin size of 0.01 and min_bin_freq of 1, 6 bins should be found
    # we bail and use the whole data here.
    with warnings.catch_warnings(record=True):
        test_bins = get_bin_seeds(X, 0.01, 1)
    assert_array_almost_equal(test_bins, X)

    # tight clusters around [0, 0] and [1, 1], only get two bins
    X, _ = make_blobs(
        n_samples=100,
        n_features=2,
        centers=[[0, 0], [1, 1]],
        cluster_std=0.1,
        random_state=0,
    )
    test_bins = get_bin_seeds(X, 1)
    assert_array_equal(test_bins, [[0, 0], [1, 1]])

@pytest.mark.parametrize("max_iter", [1, 100])
def test_max_iter(max_iter):
    clusters1, _ = mean_shift(X, max_iter=max_iter)
    ms = MeanShift(max_iter=max_iter).fit(X)
    clusters2 = ms.cluster_centers_

    assert ms.n_iter_ <= ms.max_iter
    assert len(clusters1) == len(clusters2)

    for c1, c2 in zip(clusters1, clusters2):
        assert np.allclose(c1, c2)

def test_mean_shift_zero_bandwidth():
    # Check that mean shift works when the estimated bandwidth is 0.
    X = np.array([1, 1, 1, 2, 2, 2, 3, 3]).reshape(-1, 1)

    # estimate_bandwidth with default args returns 0 on this dataset
    bandwidth = estimate_bandwidth(X)
    assert bandwidth == 0

    # get_bin_seeds with a 0 bin_size should return the dataset itself
    assert get_bin_seeds(X, bin_size=bandwidth) is X

    # MeanShift with binning and a 0 estimated bandwidth should be equivalent
    # to no binning.
    ms_binning = MeanShift(bin_seeding=True, bandwidth=None).fit(X)
    ms_nobinning = MeanShift(bin_seeding=False).fit(X)
    expected_labels = np.array([0, 0, 0, 1, 1, 1, 2, 2])

    assert v_measure_score(ms_binning.labels_, expected_labels) == 1
    assert v_measure_score(ms_nobinning.labels_, expected_labels) == 1
    assert_allclose(ms_binning.cluster_centers_, ms_nobinning.cluster_centers_)
<commit_msg>TST use global_dtype in sklearn/cluster/tests/test_mean_shift.py (#22672)<commit_after>"""
Testing for mean shift clustering methods

"""

import numpy as np
import warnings
import pytest

from scipy import sparse

from sklearn.utils._testing import assert_array_equal
from sklearn.utils._testing import assert_allclose

from sklearn.cluster import MeanShift
from sklearn.cluster import mean_shift
from sklearn.cluster import estimate_bandwidth
from sklearn.cluster import get_bin_seeds
from sklearn.datasets import make_blobs
from sklearn.metrics import v_measure_score

n_clusters = 3
centers = np.array([[1, 1], [-1, -1], [1, -1]]) + 10
X, _ = make_blobs(
    n_samples=300,
    n_features=2,
    centers=centers,
    cluster_std=0.4,
    shuffle=True,
    random_state=11,
)

def test_estimate_bandwidth():
    # Test estimate_bandwidth
    bandwidth = estimate_bandwidth(X, n_samples=200)
    assert 0.9 <= bandwidth <= 1.5

def test_estimate_bandwidth_1sample(global_dtype):
    # Test estimate_bandwidth when n_samples=1 and quantile<1, so that
    # n_neighbors is set to 1.
    bandwidth = estimate_bandwidth(
        X.astype(global_dtype, copy=False), n_samples=1, quantile=0.3
    )

    assert bandwidth.dtype == X.dtype
    assert bandwidth == pytest.approx(0.0, abs=1e-5)

@pytest.mark.parametrize(
    "bandwidth, cluster_all, expected, first_cluster_label",
    [(1.2, True, 3, 0), (1.2, False, 4, -1)],
)
def test_mean_shift(
    global_dtype, bandwidth, cluster_all, expected, first_cluster_label
):
    # Test MeanShift algorithm
    X_with_global_dtype = X.astype(global_dtype, copy=False)
    ms = MeanShift(bandwidth=bandwidth, cluster_all=cluster_all)
    labels = ms.fit(X_with_global_dtype).labels_
    labels_unique = np.unique(labels)
    n_clusters_ = len(labels_unique)
    assert n_clusters_ == expected
    assert labels_unique[0] == first_cluster_label
    assert ms.cluster_centers_.dtype == global_dtype

    cluster_centers, labels_mean_shift = mean_shift(
        X_with_global_dtype, cluster_all=cluster_all
    )
    labels_mean_shift_unique = np.unique(labels_mean_shift)
    n_clusters_mean_shift = len(labels_mean_shift_unique)
    assert n_clusters_mean_shift == expected
    assert labels_mean_shift_unique[0] == first_cluster_label
    assert cluster_centers.dtype == global_dtype

def test_mean_shift_negative_bandwidth():
    bandwidth = -1
    ms = MeanShift(bandwidth=bandwidth)
    msg = r"bandwidth needs to be greater than zero or None," r" got -1\.000000"
    with pytest.raises(ValueError, match=msg):
        ms.fit(X)

def test_estimate_bandwidth_with_sparse_matrix():
    # Test estimate_bandwidth with sparse matrix
    X = sparse.lil_matrix((1000, 1000))
    msg = "A sparse matrix was passed, but dense data is required."
    with pytest.raises(TypeError, match=msg):
        estimate_bandwidth(X)

def test_parallel(global_dtype):
    centers = np.array([[1, 1], [-1, -1], [1, -1]]) + 10
    X, _ = make_blobs(
        n_samples=50,
        n_features=2,
        centers=centers,
        cluster_std=0.4,
        shuffle=True,
        random_state=11,
    )

    X = X.astype(global_dtype, copy=False)

    ms1 = MeanShift(n_jobs=2)
    ms1.fit(X)

    ms2 = MeanShift()
    ms2.fit(X)

    assert_allclose(ms1.cluster_centers_, ms2.cluster_centers_)
    assert ms1.cluster_centers_.dtype == ms2.cluster_centers_.dtype
    assert_array_equal(ms1.labels_, ms2.labels_)

def test_meanshift_predict(global_dtype):
    # Test MeanShift.predict
    ms = MeanShift(bandwidth=1.2)
    X_with_global_dtype = X.astype(global_dtype, copy=False)
    labels = ms.fit_predict(X_with_global_dtype)
    labels2 = ms.predict(X_with_global_dtype)
    assert_array_equal(labels, labels2)

def test_meanshift_all_orphans():
    # init away from the data, crash with a sensible warning
    ms = MeanShift(bandwidth=0.1, seeds=[[-9, -9], [-10, -10]])
    msg = "No point was within bandwidth=0.1"
    with pytest.raises(ValueError, match=msg):
        ms.fit(
            X,
        )

def test_unfitted():
    # Non-regression: before fit, there should be not fitted attributes.
    ms = MeanShift()
    assert not hasattr(ms, "cluster_centers_")
    assert not hasattr(ms, "labels_")

def test_cluster_intensity_tie(global_dtype):
    X = np.array([[1, 1], [2, 1], [1, 0], [4, 7], [3, 5], [3, 6]], dtype=global_dtype)
    c1 = MeanShift(bandwidth=2).fit(X)

    X = np.array([[4, 7], [3, 5], [3, 6], [1, 1], [2, 1], [1, 0]], dtype=global_dtype)
    c2 = MeanShift(bandwidth=2).fit(X)
    assert_array_equal(c1.labels_, [1, 1, 1, 0, 0, 0])
    assert_array_equal(c2.labels_, [0, 0, 0, 1, 1, 1])

def test_bin_seeds(global_dtype):
    # Test the bin seeding technique which can be used in the mean shift
    # algorithm
    # Data is just 6 points in the plane
    X = np.array(
        [[1.0, 1.0], [1.4, 1.4], [1.8, 1.2], [2.0, 1.0], [2.1, 1.1], [0.0, 0.0]],
        dtype=global_dtype,
    )

    # With a bin coarseness of 1.0 and min_bin_freq of 1, 3 bins should be
    # found
    ground_truth = {(1.0, 1.0), (2.0, 1.0), (0.0, 0.0)}
    test_bins = get_bin_seeds(X, 1, 1)
    test_result = set(tuple(p) for p in test_bins)
    assert len(ground_truth.symmetric_difference(test_result)) == 0

    # With a bin coarseness of 1.0 and min_bin_freq of 2, 2 bins should be
    # found
    ground_truth = {(1.0, 1.0), (2.0, 1.0)}
    test_bins = get_bin_seeds(X, 1, 2)
    test_result = set(tuple(p) for p in test_bins)
    assert len(ground_truth.symmetric_difference(test_result)) == 0

    # With a bin size of 0.01 and min_bin_freq of 1, 6 bins should be found
    # we bail and use the whole data here.
    with warnings.catch_warnings(record=True):
        test_bins = get_bin_seeds(X, 0.01, 1)
    assert_allclose(test_bins, X)

    # tight clusters around [0, 0] and [1, 1], only get two bins
    X, _ = make_blobs(
        n_samples=100,
        n_features=2,
        centers=[[0, 0], [1, 1]],
        cluster_std=0.1,
        random_state=0,
    )
    X = X.astype(global_dtype, copy=False)
    test_bins = get_bin_seeds(X, 1)
    assert_array_equal(test_bins, [[0, 0], [1, 1]])

@pytest.mark.parametrize("max_iter", [1, 100])
def test_max_iter(max_iter):
    clusters1, _ = mean_shift(X, max_iter=max_iter)
    ms = MeanShift(max_iter=max_iter).fit(X)
    clusters2 = ms.cluster_centers_

    assert ms.n_iter_ <= ms.max_iter
    assert len(clusters1) == len(clusters2)

    for c1, c2 in zip(clusters1, clusters2):
        assert np.allclose(c1, c2)

def test_mean_shift_zero_bandwidth(global_dtype):
    # Check that mean shift works when the estimated bandwidth is 0.
    X = np.array([1, 1, 1, 2, 2, 2, 3, 3], dtype=global_dtype).reshape(-1, 1)

    # estimate_bandwidth with default args returns 0 on this dataset
    bandwidth = estimate_bandwidth(X)
    assert bandwidth == 0

    # get_bin_seeds with a 0 bin_size should return the dataset itself
    assert get_bin_seeds(X, bin_size=bandwidth) is X

    # MeanShift with binning and a 0 estimated bandwidth should be equivalent
    # to no binning.
    ms_binning = MeanShift(bin_seeding=True, bandwidth=None).fit(X)
    ms_nobinning = MeanShift(bin_seeding=False).fit(X)
    expected_labels = np.array([0, 0, 0, 1, 1, 1, 2, 2])

    assert v_measure_score(ms_binning.labels_, expected_labels) == pytest.approx(1)
    assert v_measure_score(ms_nobinning.labels_, expected_labels) == pytest.approx(1)
    assert_allclose(ms_binning.cluster_centers_, ms_nobinning.cluster_centers_)
<|endoftext|>


<commit_before>"""
codemonkey@jleadbetter.com
License: AGPLv3

Helper libraries for integer programming (IP) created specifically for
postchange.
"""

from intprog import IP<commit_msg>[shared libs] import correction<commit_after>"""
codemonkey@jleadbetter.com
License: AGPLv3

Helper libraries for integer programming (IP) created specifically for
postchange.
"""

from intprog import IPBase<|endoftext|>


<commit_before>import numpy as np
from numpy.testing import (assert_equal, assert_array_almost_equal,
                           assert_raises)
from skimage.transform._geometric import _stackcopy
from skimage.transform._geometric import GeometricTransform
from skimage.transform import (estimate_transform, matrix_transform,
                               SimilarityTransform, AffineTransform,
                               ProjectiveTransform, PolynomialTransform,
                               PiecewiseAffineTransform)

SRC = np.array([
    [-12.3705, -10.5075],
    [-10.7865, 15.4305],
    [8.6985, 10.8675],
    [11.4975, -9.5715],
    [7.8435, 7.4835],
    [-5.3325, 6.5025],
    [6.7905, -6.3765],
    [-6.1695, -0.8235],
])
DST = np.array([
    [0, 0],
    [0, 5800],
    [4900, 5800],
    [4900, 0],
    [4479, 4580],
    [1176, 3660],
    [3754, 790],
    [1024, 1931],
])

def test_stackcopy():
    layers = 4
    x = np.empty((3, 3, layers))
    y = np.eye(3, 3)
    _stackcopy(x, y)
    for i in range(layers):
        assert_array_almost_equal(x[..., i], y)

def test_estimate_transform():
    for tform in ('similarity', 'affine', 'projective', 'polynomial'):
        estimate_transform(tform, SRC[:2, :], DST[:2, :])
    assert_raises(ValueError, estimate_transform, 'foobar',
                  SRC[:2, :], DST[:2, :])

def test_matrix_transform():
    tform = AffineTransform(scale=(0.1, 0.5), rotation=2)
    assert_equal(tform(SRC), matrix_transform(SRC, tform._matrix))

def test_similarity_estimation():
    # exact solution
    tform = estimate_transform('similarity', SRC[:2, :], DST[:2, :])
    assert_array_almost_equal(tform(SRC[:2, :]), DST[:2, :])
    assert_equal(tform.params[0, 0], tform.params[1, 1])
    assert_equal(tform.params[0, 1], - tform.params[1, 0])

    # over-determined
    tform2 = estimate_transform('similarity', SRC, DST)
    assert_array_almost_equal(tform2.inverse(tform2(SRC)), SRC)
    assert_equal(tform2.params[0, 0], tform2.params[1, 1])
    assert_equal(tform2.params[0, 1], - tform2.params[1, 0])

    # via estimate method
    tform3 = SimilarityTransform()
    tform3.estimate(SRC, DST)
    assert_array_almost_equal(tform3.params, tform2.params)

def test_similarity_init():
    # init with implicit parameters
    scale = 0.1
    rotation = 1
    translation = (1, 1)
    tform = SimilarityTransform(scale=scale, rotation=rotation,
                                translation=translation)
    assert_array_almost_equal(tform.scale, scale)
    assert_array_almost_equal(tform.rotation, rotation)
    assert_array_almost_equal(tform.translation, translation)

    # init with transformation matrix
    tform2 = SimilarityTransform(tform.params)
    assert_array_almost_equal(tform2.scale, scale)
    assert_array_almost_equal(tform2.rotation, rotation)
    assert_array_almost_equal(tform2.translation, translation)

    # test special case for scale if rotation=0
    scale = 0.1
    rotation = 0
    translation = (1, 1)
    tform = SimilarityTransform(scale=scale, rotation=rotation,
                                translation=translation)
    assert_array_almost_equal(tform.scale, scale)
    assert_array_almost_equal(tform.rotation, rotation)
    assert_array_almost_equal(tform.translation, translation)

def test_affine_estimation():
    # exact solution
    tform = estimate_transform('affine', SRC[:3, :], DST[:3, :])
    assert_array_almost_equal(tform(SRC[:3, :]), DST[:3, :])

    # over-determined
    tform2 = estimate_transform('affine', SRC, DST)
    assert_array_almost_equal(tform2.inverse(tform2(SRC)), SRC)

    # via estimate method
    tform3 = AffineTransform()
    tform3.estimate(SRC, DST)
    assert_array_almost_equal(tform3.params, tform2.params)

def test_affine_init():
    # init with implicit parameters
    scale = (0.1, 0.13)
    rotation = 1
    shear = 0.1
    translation = (1, 1)
    tform = AffineTransform(scale=scale, rotation=rotation, shear=shear,
                            translation=translation)
    assert_array_almost_equal(tform.scale, scale)
    assert_array_almost_equal(tform.rotation, rotation)
    assert_array_almost_equal(tform.shear, shear)
    assert_array_almost_equal(tform.translation, translation)

    # init with transformation matrix
    tform2 = AffineTransform(tform.params)
    assert_array_almost_equal(tform2.scale, scale)
    assert_array_almost_equal(tform2.rotation, rotation)
    assert_array_almost_equal(tform2.shear, shear)
    assert_array_almost_equal(tform2.translation, translation)

def test_piecewise_affine():
    tform = PiecewiseAffineTransform()
    tform.estimate(SRC, DST)
    # make sure each single affine transform is exactly estimated
    assert_array_almost_equal(tform(SRC), DST)
    assert_array_almost_equal(tform.inverse(DST), SRC)

def test_projective_estimation():
    # exact solution
    tform = estimate_transform('projective', SRC[:4, :], DST[:4, :])
    assert_array_almost_equal(tform(SRC[:4, :]), DST[:4, :])

    # over-determined
    tform2 = estimate_transform('projective', SRC, DST)
    assert_array_almost_equal(tform2.inverse(tform2(SRC)), SRC)

    # via estimate method
    tform3 = ProjectiveTransform()
    tform3.estimate(SRC, DST)
    assert_array_almost_equal(tform3.params, tform2.params)

def test_projective_init():
    tform = estimate_transform('projective', SRC, DST)
    # init with transformation matrix
    tform2 = ProjectiveTransform(tform.params)
    assert_array_almost_equal(tform2.params, tform.params)

def test_polynomial_estimation():
    # over-determined
    tform = estimate_transform('polynomial', SRC, DST, order=10)
    assert_array_almost_equal(tform(SRC), DST, 6)

    # via estimate method
    tform2 = PolynomialTransform()
    tform2.estimate(SRC, DST, order=10)
    assert_array_almost_equal(tform2.params, tform.params)

def test_polynomial_init():
    tform = estimate_transform('polynomial', SRC, DST, order=10)
    # init with transformation parameters
    tform2 = PolynomialTransform(tform.params)
    assert_array_almost_equal(tform2.params, tform.params)

def test_polynomial_default_order():
    tform = estimate_transform('polynomial', SRC, DST)
    tform2 = estimate_transform('polynomial', SRC, DST, order=2)
    assert_array_almost_equal(tform2.params, tform.params)

def test_polynomial_inverse():
    assert_raises(Exception, PolynomialTransform().inverse, 0)

def test_union():
    tform1 = SimilarityTransform(scale=0.1, rotation=0.3)
    tform2 = SimilarityTransform(scale=0.1, rotation=0.9)
    tform3 = SimilarityTransform(scale=0.1 ** 2, rotation=0.3 + 0.9)
    tform = tform1 + tform2
    assert_array_almost_equal(tform._matrix, tform3._matrix)

    tform1 = AffineTransform(scale=(0.1, 0.1), rotation=0.3)
    tform2 = SimilarityTransform(scale=0.1, rotation=0.9)
    tform3 = SimilarityTransform(scale=0.1 ** 2, rotation=0.3 + 0.9)
    tform = tform1 + tform2
    assert_array_almost_equal(tform._matrix, tform3._matrix)
    assert tform.__class__ == ProjectiveTransform

def test_geometric_tform():
    tform = GeometricTransform()
    assert_raises(NotImplementedError, tform, 0)
    assert_raises(NotImplementedError, tform.inverse, 0)
    assert_raises(NotImplementedError, tform.__add__, 0)

def test_invalid_input():
    assert_raises(ValueError, ProjectiveTransform, np.zeros((2, 3)))
    assert_raises(ValueError, AffineTransform, np.zeros((2, 3)))
    assert_raises(ValueError, SimilarityTransform, np.zeros((2, 3)))

    assert_raises(ValueError, AffineTransform,
                  matrix=np.zeros((2, 3)), scale=1)
    assert_raises(ValueError, SimilarityTransform,
                  matrix=np.zeros((2, 3)), scale=1)

    assert_raises(ValueError, PolynomialTransform, np.zeros((3, 3)))

def test_deprecated_params_attributes():
    for t in ('projective', 'affine', 'similarity'):
        tform = estimate_transform(t, SRC, DST)
        assert_equal(tform._matrix, tform.params)

    tform = estimate_transform('polynomial', SRC, DST, order=3)
    assert_equal(tform._params, tform.params)

if __name__ == "__main__":
    from numpy.testing import run_module_suite
    run_module_suite()
<commit_msg>Test union of differing geometric transforms<commit_after>import numpy as np
from numpy.testing import (assert_equal, assert_array_almost_equal,
                           assert_raises)
from skimage.transform._geometric import _stackcopy
from skimage.transform._geometric import GeometricTransform
from skimage.transform import (estimate_transform, matrix_transform,
                               SimilarityTransform, AffineTransform,
                               ProjectiveTransform, PolynomialTransform,
                               PiecewiseAffineTransform)

SRC = np.array([
    [-12.3705, -10.5075],
    [-10.7865, 15.4305],
    [8.6985, 10.8675],
    [11.4975, -9.5715],
    [7.8435, 7.4835],
    [-5.3325, 6.5025],
    [6.7905, -6.3765],
    [-6.1695, -0.8235],
])
DST = np.array([
    [0, 0],
    [0, 5800],
    [4900, 5800],
    [4900, 0],
    [4479, 4580],
    [1176, 3660],
    [3754, 790],
    [1024, 1931],
])

def test_stackcopy():
    layers = 4
    x = np.empty((3, 3, layers))
    y = np.eye(3, 3)
    _stackcopy(x, y)
    for i in range(layers):
        assert_array_almost_equal(x[..., i], y)

def test_estimate_transform():
    for tform in ('similarity', 'affine', 'projective', 'polynomial'):
        estimate_transform(tform, SRC[:2, :], DST[:2, :])
    assert_raises(ValueError, estimate_transform, 'foobar',
                  SRC[:2, :], DST[:2, :])

def test_matrix_transform():
    tform = AffineTransform(scale=(0.1, 0.5), rotation=2)
    assert_equal(tform(SRC), matrix_transform(SRC, tform._matrix))

def test_similarity_estimation():
    # exact solution
    tform = estimate_transform('similarity', SRC[:2, :], DST[:2, :])
    assert_array_almost_equal(tform(SRC[:2, :]), DST[:2, :])
    assert_equal(tform.params[0, 0], tform.params[1, 1])
    assert_equal(tform.params[0, 1], - tform.params[1, 0])

    # over-determined
    tform2 = estimate_transform('similarity', SRC, DST)
    assert_array_almost_equal(tform2.inverse(tform2(SRC)), SRC)
    assert_equal(tform2.params[0, 0], tform2.params[1, 1])
    assert_equal(tform2.params[0, 1], - tform2.params[1, 0])

    # via estimate method
    tform3 = SimilarityTransform()
    tform3.estimate(SRC, DST)
    assert_array_almost_equal(tform3.params, tform2.params)

def test_similarity_init():
    # init with implicit parameters
    scale = 0.1
    rotation = 1
    translation = (1, 1)
    tform = SimilarityTransform(scale=scale, rotation=rotation,
                                translation=translation)
    assert_array_almost_equal(tform.scale, scale)
    assert_array_almost_equal(tform.rotation, rotation)
    assert_array_almost_equal(tform.translation, translation)

    # init with transformation matrix
    tform2 = SimilarityTransform(tform.params)
    assert_array_almost_equal(tform2.scale, scale)
    assert_array_almost_equal(tform2.rotation, rotation)
    assert_array_almost_equal(tform2.translation, translation)

    # test special case for scale if rotation=0
    scale = 0.1
    rotation = 0
    translation = (1, 1)
    tform = SimilarityTransform(scale=scale, rotation=rotation,
                                translation=translation)
    assert_array_almost_equal(tform.scale, scale)
    assert_array_almost_equal(tform.rotation, rotation)
    assert_array_almost_equal(tform.translation, translation)

def test_affine_estimation():
    # exact solution
    tform = estimate_transform('affine', SRC[:3, :], DST[:3, :])
    assert_array_almost_equal(tform(SRC[:3, :]), DST[:3, :])

    # over-determined
    tform2 = estimate_transform('affine', SRC, DST)
    assert_array_almost_equal(tform2.inverse(tform2(SRC)), SRC)

    # via estimate method
    tform3 = AffineTransform()
    tform3.estimate(SRC, DST)
    assert_array_almost_equal(tform3.params, tform2.params)

def test_affine_init():
    # init with implicit parameters
    scale = (0.1, 0.13)
    rotation = 1
    shear = 0.1
    translation = (1, 1)
    tform = AffineTransform(scale=scale, rotation=rotation, shear=shear,
                            translation=translation)
    assert_array_almost_equal(tform.scale, scale)
    assert_array_almost_equal(tform.rotation, rotation)
    assert_array_almost_equal(tform.shear, shear)
    assert_array_almost_equal(tform.translation, translation)

    # init with transformation matrix
    tform2 = AffineTransform(tform.params)
    assert_array_almost_equal(tform2.scale, scale)
    assert_array_almost_equal(tform2.rotation, rotation)
    assert_array_almost_equal(tform2.shear, shear)
    assert_array_almost_equal(tform2.translation, translation)

def test_piecewise_affine():
    tform = PiecewiseAffineTransform()
    tform.estimate(SRC, DST)
    # make sure each single affine transform is exactly estimated
    assert_array_almost_equal(tform(SRC), DST)
    assert_array_almost_equal(tform.inverse(DST), SRC)

def test_projective_estimation():
    # exact solution
    tform = estimate_transform('projective', SRC[:4, :], DST[:4, :])
    assert_array_almost_equal(tform(SRC[:4, :]), DST[:4, :])

    # over-determined
    tform2 = estimate_transform('projective', SRC, DST)
    assert_array_almost_equal(tform2.inverse(tform2(SRC)), SRC)

    # via estimate method
    tform3 = ProjectiveTransform()
    tform3.estimate(SRC, DST)
    assert_array_almost_equal(tform3.params, tform2.params)

def test_projective_init():
    tform = estimate_transform('projective', SRC, DST)
    # init with transformation matrix
    tform2 = ProjectiveTransform(tform.params)
    assert_array_almost_equal(tform2.params, tform.params)

def test_polynomial_estimation():
    # over-determined
    tform = estimate_transform('polynomial', SRC, DST, order=10)
    assert_array_almost_equal(tform(SRC), DST, 6)

    # via estimate method
    tform2 = PolynomialTransform()
    tform2.estimate(SRC, DST, order=10)
    assert_array_almost_equal(tform2.params, tform.params)

def test_polynomial_init():
    tform = estimate_transform('polynomial', SRC, DST, order=10)
    # init with transformation parameters
    tform2 = PolynomialTransform(tform.params)
    assert_array_almost_equal(tform2.params, tform.params)

def test_polynomial_default_order():
    tform = estimate_transform('polynomial', SRC, DST)
    tform2 = estimate_transform('polynomial', SRC, DST, order=2)
    assert_array_almost_equal(tform2.params, tform.params)

def test_polynomial_inverse():
    assert_raises(Exception, PolynomialTransform().inverse, 0)

def test_union():
    tform1 = SimilarityTransform(scale=0.1, rotation=0.3)
    tform2 = SimilarityTransform(scale=0.1, rotation=0.9)
    tform3 = SimilarityTransform(scale=0.1 ** 2, rotation=0.3 + 0.9)
    tform = tform1 + tform2
    assert_array_almost_equal(tform._matrix, tform3._matrix)

    tform1 = AffineTransform(scale=(0.1, 0.1), rotation=0.3)
    tform2 = SimilarityTransform(scale=0.1, rotation=0.9)
    tform3 = SimilarityTransform(scale=0.1 ** 2, rotation=0.3 + 0.9)
    tform = tform1 + tform2
    assert_array_almost_equal(tform._matrix, tform3._matrix)
    assert tform.__class__ == ProjectiveTransform

def test_union_differing_types():
    tform1 = SimilarityTransform()
    tform2 = PolynomialTransform()
    assert_raises(TypeError, tform1.__add__, tform2)

def test_geometric_tform():
    tform = GeometricTransform()
    assert_raises(NotImplementedError, tform, 0)
    assert_raises(NotImplementedError, tform.inverse, 0)
    assert_raises(NotImplementedError, tform.__add__, 0)

def test_invalid_input():
    assert_raises(ValueError, ProjectiveTransform, np.zeros((2, 3)))
    assert_raises(ValueError, AffineTransform, np.zeros((2, 3)))
    assert_raises(ValueError, SimilarityTransform, np.zeros((2, 3)))

    assert_raises(ValueError, AffineTransform,
                  matrix=np.zeros((2, 3)), scale=1)
    assert_raises(ValueError, SimilarityTransform,
                  matrix=np.zeros((2, 3)), scale=1)

    assert_raises(ValueError, PolynomialTransform, np.zeros((3, 3)))

def test_deprecated_params_attributes():
    for t in ('projective', 'affine', 'similarity'):
        tform = estimate_transform(t, SRC, DST)
        assert_equal(tform._matrix, tform.params)

    tform = estimate_transform('polynomial', SRC, DST, order=3)
    assert_equal(tform._params, tform.params)

if __name__ == "__main__":
    from numpy.testing import run_module_suite
    run_module_suite()
<|endoftext|>