<commit_before># Copyright 2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Integration test for App Engine.

Primarily tests the deploy operation and upsert load balancer pipeline stage,
which are relatively complex and not well covered by unit tests.

Sample Usage:
    Assuming you have created $PASSPHRASE_FILE (which you should chmod 400)
    and $CITEST_ROOT points to the root directory of the citest library.
    The passphrase file can be omited if you run ssh-agent and add
    .ssh/compute_google_engine.

  PYTHONPATH=$CITEST_ROOT \
    python spinnaker/testing/citest/tests/appengine_smoke_test.py \
    --gce_ssh_passphrase_file=$PASSPHRASE_FILE \
    --gce_project=$PROJECT \
    --gce_zone=$ZONE \
    --gce_instance=$INSTANCE
or
  PYTHONPATH=$CITEST_ROOT \
    python spinnaker/testing/citest/tests/appengine_smoke_test.py \
    --native_hostname=host-running-smoke-test
"""

import logging
import os
import shutil
import subprocess
import sys
import tempfile

import citest.gcp_testing as gcp
import citest.json_contract as jc
import citest.json_predicate as jp
import citest.service_testing as st
ov_factory = jc.ObservationPredicateFactory()

import spinnaker_testing as sk
import spinnaker_testing.gate as gate
import spinnaker_testing.frigga as frigga
import citest.base

class AppengineSmokeTestScenario(sk.SpinnakerTestScenario):
  """Defines the scenario for the integration test.

  We're going to:
    Create a Spinnaker Application
    Create a Spinnaker Server Group (implicitly creates a Load Balancer)
    Create a Pipeline with the following stages
      - Deploy
      - Upsert Load Balancer
    Delete Load Balancer (implicitly destroys the Server Groups
    created within this test)
    Delete Application
  """
  @classmethod
  def new_agent(cls, bindings):
    return gate.new_agent(bindings)

  @classmethod
  def initArgumentParser(cls, parser, defaults=None):
    """Initialize command line argument parser."""
    super(AppengineSmokeTestScenario, cls).initArgumentParser(
          parser, defaults=defaults)
    parser.add_argument(
        '--test_gcs_bucket', default=None,
        help='URL to use for testing appengine deployment from a bucket.'
             ' The test will write into this bucket'
             ' then deploy what it writes.')

    parser.add_argument('--git_repo_url', default=None,
                        help='URL of a GIT source code repository used by Spinnaker to deploy to App Engine.')
    parser.add_argument('--branch', default='master',
                        help='Git branch to be used when deploying from source code repository.')
    parser.add_argument('--app_directory_root', default=None,
                        help='Path from the root of source code repository to the application directory.')

  def __init__(self, bindings, agent=None):
    super(AppengineSmokeTestScenario, self).__init__(bindings, agent)

    if not bindings['GIT_REPO_URL']:
      raise ValueError('Must supply value for --git_repo_url')

    if not bindings['APP_DIRECTORY_ROOT']:
      raise ValueError('Must supply value for --app_directory_root')

    self.TEST_APP = bindings['TEST_APP']
    self.TEST_STACK = bindings['TEST_STACK']
    self.__path = 'applications/%s/tasks' % self.TEST_APP
    self.__gcp_project = bindings['APPENGINE_PRIMARY_MANAGED_PROJECT_ID']
    self.__cluster_name = frigga.Naming.cluster(self.TEST_APP, self.TEST_STACK)
    self.__server_group_name = frigga.Naming.server_group(self.TEST_APP, self.TEST_STACK)
    self.__lb_name = self.__cluster_name

    # Python is clearly hard-coded as the runtime here, but we're just asking App Engine to be a static file server.
    self.__app_yaml = ('\n'.join(['runtime: python27',
                                  'api_version: 1',
                                  'threadsafe: true',
                                  'service: {service}',
                                  'handlers:',
                                  ' - url: /.*',
                                  '   static_dir: .']).format(service=self.__lb_name))

    self.__app_directory_root = bindings['APP_DIRECTORY_ROOT']
    self.__branch = bindings['BRANCH']
    self.pipeline_id = None

    test_bucket = bindings['TEST_GCS_BUCKET']
    if test_bucket:
      self.__prepare_bucket(test_bucket)
      self.__test_repository_url = 'gs://' + test_bucket
    else:
      self.__test_repository_url = bindings['GIT_REPO_URL']

  def __prepare_bucket(self, bucket):
    root = self.bindings['APP_DIRECTORY_ROOT']
    temp = tempfile.mkdtemp()
    local_path = os.path.join(temp, root)

    branch = self.bindings['BRANCH']
    git_repo = self.bindings['GIT_REPO_URL']
    gcs_path = 'gs://{bucket}/{root}'.format(
        bucket=self.bindings['TEST_GCS_BUCKET'], root=root)

    try:
      command = 'git clone {repo} -b {branch} {dir}'.format(
          repo=git_repo, branch=branch, dir=temp)
      logging.info('Fetching %s', git_repo)
      subprocess.Popen(command, stderr=sys.stderr, shell=True).wait()

      command = 'gsutil -m rsync {local} {gcs}'.format(
          local=local_path, gcs=gcs_path)
      logging.info('Preparing %s', gcs_path)
      subprocess.Popen(command, stderr=sys.stderr, shell=True).wait()
    finally:
      shutil.rmtree(local_path)

  def create_app(self):
    # Not testing create_app, since the operation is well tested elsewhere.
    contract = jc.Contract()
    return st.OperationContract(
      self.agent.make_create_app_operation(
        bindings=self.bindings,
        application=self.TEST_APP,
        account_name=self.bindings['SPINNAKER_APPENGINE_ACCOUNT']),
      contract=contract)

  def delete_app(self):
    # Not testing delete_app, since the operation is well tested elsewhere.
    contract = jc.Contract()
    return st.OperationContract(
        self.agent.make_delete_app_operation(
            application=self.TEST_APP,
            account_name=self.bindings['SPINNAKER_APPENGINE_ACCOUNT']),
        contract=contract)

  def create_server_group(self):
    group_name = frigga.Naming.server_group(
        app=self.TEST_APP,
        stack=self.bindings['TEST_STACK'],
        version='v000')
    job_spec = {
        'application': self.TEST_APP,
        'stack': self.TEST_STACK,
        'credentials': self.bindings['SPINNAKER_APPENGINE_ACCOUNT'],
        'repositoryUrl': self.__test_repository_url,
        'applicationDirectoryRoot': self.__app_directory_root,
        'configFiles': [self.__app_yaml],
        'type': 'createServerGroup',
        'cloudProvider': 'appengine',
        'region': 'us-central'
      }
    if not self.__test_repository_url.startswith('gs://'):
      job_spec.update({
          'gitCredentialType': 'NONE',
          'branch': self.__branch
      })

    payload = self.agent.make_json_payload_from_kwargs(job=[job_spec],
      description='Create Server Group in ' + group_name,
      application=self.TEST_APP)

    builder = gcp.GcpContractBuilder(self.appengine_observer)
    (builder.new_clause_builder('Version Added', retryable_for_secs=30)
      .inspect_resource('apps.services.versions',
                        group_name,
                        appsId=self.__gcp_project,
                        servicesId=self.__lb_name)
     .EXPECT(ov_factory.value_list_path_contains(
         'servingStatus', jp.STR_EQ('SERVING'))))

    return st.OperationContract(
        self.new_post_operation(
            title='create_server_group', data=payload, path='tasks'),
        contract=builder.build())

  def make_deploy_stage(self):
    cluster_spec = {
        'account': self.bindings['SPINNAKER_APPENGINE_ACCOUNT'],
        'applicationDirectoryRoot': self.__app_directory_root,
        'configFiles': [self.__app_yaml],
        'application': self.TEST_APP,
        'cloudProvider': 'appengine',
        'provider': 'appengine',
        'region': 'us-central',
        'repositoryUrl': self.__test_repository_url,
        'stack': self.TEST_STACK
    }
    if not self.__test_repository_url.startswith('gs://'):
      cluster_spec.update({
          'gitCredentialType': 'NONE',
          'branch': self.__branch
      })

    result = {
      'clusters': [cluster_spec],
      'name': 'Deploy',
      'refId': '1',
      'requisiteStageRefIds': [],
      'type': 'deploy'
    }
    return result

  def make_upsert_load_balancer_stage(self):
    result = {
        'cloudProvider': 'appengine',
        'loadBalancers': [
        {
            'cloudProvider': 'appengine',
            'credentials': self.bindings['SPINNAKER_APPENGINE_ACCOUNT'],
            'loadBalancerName': self.__lb_name,
            'migrateTraffic': False,
            'name': self.__lb_name,
            'region': 'us-central',
            'splitDescription': {
                'allocationDescriptions': [
                {
                    'allocation': 0.1,
                    'cluster': self.__cluster_name,
                    'locatorType': 'targetCoordinate',
                   'target': 'current_asg_dynamic'
                },
                {
                    'allocation': 0.9,
                    'cluster': self.__cluster_name,
                    'locatorType': 'targetCoordinate',
                    'target': 'ancestor_asg_dynamic'
                }
                ],
                'shardBy': 'IP'
            }
        }
      ],
      'name': 'Edit Load Balancer',
      'refId': '2',
      'requisiteStageRefIds': ['1'],
      'type': 'upsertAppEngineLoadBalancers'
    }
    return result

  def create_deploy_upsert_load_balancer_pipeline(self):
    name = 'promoteServerGroupPipeline'
    self.pipeline_id = name
    deploy_stage = self.make_deploy_stage()
    upsert_load_balancer_stage = self.make_upsert_load_balancer_stage()

    pipeline_spec = dict(
        name=name,
        stages=[deploy_stage, upsert_load_balancer_stage],
        triggers=[],
        application=self.TEST_APP,
        stageCounter=2,
        parallel=True,
        limitConcurrent=True,
        appConfig={},
        index=0
    )

    payload = self.agent.make_json_payload_from_kwargs(**pipeline_spec)

    builder = st.HttpContractBuilder(self.agent)
    (builder.new_clause_builder('Has Pipeline',
                                retryable_for_secs=5)
     .get_url_path('applications/{0}/pipelineConfigs'.format(self.TEST_APP))
     .contains_path_value(None, pipeline_spec))

    return st.OperationContract(
      self.new_post_operation(
        title='create_deploy_upsert_load_balancer_pipeline', data=payload, path='pipelines',
        status_class=st.SynchronousHttpOperationStatus),
      contract=builder.build())

  def run_deploy_upsert_load_balancer_pipeline(self):
    url_path = 'pipelines/{0}/{1}'.format(self.TEST_APP, self.pipeline_id)

    previous_group_name = frigga.Naming.server_group(
        app=self.TEST_APP,
        stack=self.TEST_STACK,
        version='v000')

    deployed_group_name = frigga.Naming.server_group(
        app=self.TEST_APP,
        stack=self.TEST_STACK,
        version='v001')

    payload = self.agent.make_json_payload_from_kwargs(
        type='manual',
        user='[anonymous]')

    builder = gcp.GcpContractBuilder(self.appengine_observer)
    (builder.new_clause_builder('Service Modified', retryable_for_secs=30)
     .inspect_resource('apps.services',
                       self.__lb_name,
                       appsId=self.__gcp_project)
     .EXPECT(
         ov_factory.value_list_path_contains(
             jp.build_path('split', 'allocations'),
             jp.DICT_MATCHES({previous_group_name: jp.NUM_EQ(0.9),
                              deployed_group_name: jp.NUM_EQ(0.1)}))))

    return st.OperationContract(
        self.new_post_operation(
            title='run_deploy_upsert_load_balancer_pipeline',
            data=payload, path=url_path),
        builder.build())

  def delete_load_balancer(self):
    bindings = self.bindings
    payload = self.agent.make_json_payload_from_kwargs(
        job=[{
            'type': 'deleteLoadBalancer',
            'cloudProvider': 'appengine',
            'loadBalancerName': self.__lb_name,
            'account': bindings['SPINNAKER_APPENGINE_ACCOUNT'],
            'credentials': bindings['SPINNAKER_APPENGINE_ACCOUNT'],
            'user': '[anonymous]'
        }],
        description='Delete Load Balancer: {0} in {1}'.format(
            self.__lb_name,
            bindings['SPINNAKER_APPENGINE_ACCOUNT']),
        application=self.TEST_APP)

    builder = gcp.GcpContractBuilder(self.appengine_observer)
    (builder.new_clause_builder('Service Deleted', retryable_for_secs=30)
     .inspect_resource('apps.services',
                       self.__lb_name,
                       appsId=self.__gcp_project)
     .EXPECT(
         ov_factory.error_list_contains(gcp.HttpErrorPredicate(http_code=404))))

    return st.OperationContract(
        self.new_post_operation(
            title='delete_load_balancer', data=payload, path='tasks'),
        contract=builder.build())

class AppengineSmokeTest(st.AgentTestCase):

  @property
  def scenario(self):
    return citest.base.TestRunner.global_runner().get_shared_data(AppengineSmokeTestScenario)

  def test_a_create_app(self):
    self.run_test_case(self.scenario.create_app())

  def test_b_create_server_group(self):
    self.run_test_case(self.scenario.create_server_group())

  def test_c_create_pipeline(self):
    self.run_test_case(self.scenario.create_deploy_upsert_load_balancer_pipeline())

  def test_d_run_pipeline(self):
    self.run_test_case(self.scenario.run_deploy_upsert_load_balancer_pipeline())

  def test_y_delete_load_balancer(self):
    self.run_test_case(self.scenario.delete_load_balancer(),
                       retry_interval_secs=8, max_retries=8)

  def test_z_delete_app(self):
    self.run_test_case(self.scenario.delete_app(),
                       retry_interval_secs=8, max_retries=8)

def main():
  defaults = {
      'TEST_STACK': AppengineSmokeTestScenario.DEFAULT_TEST_ID,
      'TEST_APP': 'gaesmoketest' + AppengineSmokeTestScenario.DEFAULT_TEST_ID,
  }

  return citest.base.TestRunner.main(
      parser_inits=[AppengineSmokeTestScenario.initArgumentParser],
      default_binding_overrides=defaults,
      test_case_list=[AppengineSmokeTest])

if __name__ == '__main__':
  sys.exit(main())
<commit_msg>docs(tests/appengine): prerequisites for running app engine smoke test (#1936)<commit_after># Copyright 2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Integration test for App Engine.

Primarily tests the deploy operation and upsert load balancer pipeline stage,
which are relatively complex and not well covered by unit tests.

Sample Usage:
    Before running this test, verify that the App Engine application
    in your GCP project has a default service. If it does not, deploy
    any App Engine version to your application that will use the default service.

    Assuming you have created $PASSPHRASE_FILE (which you should chmod 400)
    and $CITEST_ROOT points to the root directory of the citest library.
    The passphrase file can be omited if you run ssh-agent and add
    .ssh/compute_google_engine.

  PYTHONPATH=$CITEST_ROOT \
    python spinnaker/testing/citest/tests/appengine_smoke_test.py \
    --gce_ssh_passphrase_file=$PASSPHRASE_FILE \
    --gce_project=$PROJECT \
    --gce_zone=$ZONE \
    --gce_instance=$INSTANCE
or
  PYTHONPATH=$CITEST_ROOT \
    python spinnaker/testing/citest/tests/appengine_smoke_test.py \
    --native_hostname=host-running-smoke-test
"""

import logging
import os
import shutil
import subprocess
import sys
import tempfile

import citest.gcp_testing as gcp
import citest.json_contract as jc
import citest.json_predicate as jp
import citest.service_testing as st
ov_factory = jc.ObservationPredicateFactory()

import spinnaker_testing as sk
import spinnaker_testing.gate as gate
import spinnaker_testing.frigga as frigga
import citest.base

class AppengineSmokeTestScenario(sk.SpinnakerTestScenario):
  """Defines the scenario for the integration test.

  We're going to:
    Create a Spinnaker Application
    Create a Spinnaker Server Group (implicitly creates a Load Balancer)
    Create a Pipeline with the following stages
      - Deploy
      - Upsert Load Balancer
    Delete Load Balancer (implicitly destroys the Server Groups
    created within this test)
    Delete Application
  """
  @classmethod
  def new_agent(cls, bindings):
    return gate.new_agent(bindings)

  @classmethod
  def initArgumentParser(cls, parser, defaults=None):
    """Initialize command line argument parser."""
    super(AppengineSmokeTestScenario, cls).initArgumentParser(
          parser, defaults=defaults)
    parser.add_argument(
        '--test_gcs_bucket', default=None,
        help='URL to use for testing appengine deployment from a bucket.'
             ' The test will write into this bucket'
             ' then deploy what it writes.')

    parser.add_argument('--git_repo_url', default=None,
                        help='URL of a GIT source code repository used by Spinnaker to deploy to App Engine.')
    parser.add_argument('--branch', default='master',
                        help='Git branch to be used when deploying from source code repository.')
    parser.add_argument('--app_directory_root', default=None,
                        help='Path from the root of source code repository to the application directory.')

  def __init__(self, bindings, agent=None):
    super(AppengineSmokeTestScenario, self).__init__(bindings, agent)

    if not bindings['GIT_REPO_URL']:
      raise ValueError('Must supply value for --git_repo_url')

    if not bindings['APP_DIRECTORY_ROOT']:
      raise ValueError('Must supply value for --app_directory_root')

    self.TEST_APP = bindings['TEST_APP']
    self.TEST_STACK = bindings['TEST_STACK']
    self.__path = 'applications/%s/tasks' % self.TEST_APP
    self.__gcp_project = bindings['APPENGINE_PRIMARY_MANAGED_PROJECT_ID']
    self.__cluster_name = frigga.Naming.cluster(self.TEST_APP, self.TEST_STACK)
    self.__server_group_name = frigga.Naming.server_group(self.TEST_APP, self.TEST_STACK)
    self.__lb_name = self.__cluster_name

    # Python is clearly hard-coded as the runtime here, but we're just asking App Engine to be a static file server.
    self.__app_yaml = ('\n'.join(['runtime: python27',
                                  'api_version: 1',
                                  'threadsafe: true',
                                  'service: {service}',
                                  'handlers:',
                                  ' - url: /.*',
                                  '   static_dir: .']).format(service=self.__lb_name))

    self.__app_directory_root = bindings['APP_DIRECTORY_ROOT']
    self.__branch = bindings['BRANCH']
    self.pipeline_id = None

    test_bucket = bindings['TEST_GCS_BUCKET']
    if test_bucket:
      self.__prepare_bucket(test_bucket)
      self.__test_repository_url = 'gs://' + test_bucket
    else:
      self.__test_repository_url = bindings['GIT_REPO_URL']

  def __prepare_bucket(self, bucket):
    root = self.bindings['APP_DIRECTORY_ROOT']
    temp = tempfile.mkdtemp()
    local_path = os.path.join(temp, root)

    branch = self.bindings['BRANCH']
    git_repo = self.bindings['GIT_REPO_URL']
    gcs_path = 'gs://{bucket}/{root}'.format(
        bucket=self.bindings['TEST_GCS_BUCKET'], root=root)

    try:
      command = 'git clone {repo} -b {branch} {dir}'.format(
          repo=git_repo, branch=branch, dir=temp)
      logging.info('Fetching %s', git_repo)
      subprocess.Popen(command, stderr=sys.stderr, shell=True).wait()

      command = 'gsutil -m rsync {local} {gcs}'.format(
          local=local_path, gcs=gcs_path)
      logging.info('Preparing %s', gcs_path)
      subprocess.Popen(command, stderr=sys.stderr, shell=True).wait()
    finally:
      shutil.rmtree(local_path)

  def create_app(self):
    # Not testing create_app, since the operation is well tested elsewhere.
    contract = jc.Contract()
    return st.OperationContract(
      self.agent.make_create_app_operation(
        bindings=self.bindings,
        application=self.TEST_APP,
        account_name=self.bindings['SPINNAKER_APPENGINE_ACCOUNT']),
      contract=contract)

  def delete_app(self):
    # Not testing delete_app, since the operation is well tested elsewhere.
    contract = jc.Contract()
    return st.OperationContract(
        self.agent.make_delete_app_operation(
            application=self.TEST_APP,
            account_name=self.bindings['SPINNAKER_APPENGINE_ACCOUNT']),
        contract=contract)

  def create_server_group(self):
    group_name = frigga.Naming.server_group(
        app=self.TEST_APP,
        stack=self.bindings['TEST_STACK'],
        version='v000')
    job_spec = {
        'application': self.TEST_APP,
        'stack': self.TEST_STACK,
        'credentials': self.bindings['SPINNAKER_APPENGINE_ACCOUNT'],
        'repositoryUrl': self.__test_repository_url,
        'applicationDirectoryRoot': self.__app_directory_root,
        'configFiles': [self.__app_yaml],
        'type': 'createServerGroup',
        'cloudProvider': 'appengine',
        'region': 'us-central'
      }
    if not self.__test_repository_url.startswith('gs://'):
      job_spec.update({
          'gitCredentialType': 'NONE',
          'branch': self.__branch
      })

    payload = self.agent.make_json_payload_from_kwargs(job=[job_spec],
      description='Create Server Group in ' + group_name,
      application=self.TEST_APP)

    builder = gcp.GcpContractBuilder(self.appengine_observer)
    (builder.new_clause_builder('Version Added', retryable_for_secs=30)
      .inspect_resource('apps.services.versions',
                        group_name,
                        appsId=self.__gcp_project,
                        servicesId=self.__lb_name)
     .EXPECT(ov_factory.value_list_path_contains(
         'servingStatus', jp.STR_EQ('SERVING'))))

    return st.OperationContract(
        self.new_post_operation(
            title='create_server_group', data=payload, path='tasks'),
        contract=builder.build())

  def make_deploy_stage(self):
    cluster_spec = {
        'account': self.bindings['SPINNAKER_APPENGINE_ACCOUNT'],
        'applicationDirectoryRoot': self.__app_directory_root,
        'configFiles': [self.__app_yaml],
        'application': self.TEST_APP,
        'cloudProvider': 'appengine',
        'provider': 'appengine',
        'region': 'us-central',
        'repositoryUrl': self.__test_repository_url,
        'stack': self.TEST_STACK
    }
    if not self.__test_repository_url.startswith('gs://'):
      cluster_spec.update({
          'gitCredentialType': 'NONE',
          'branch': self.__branch
      })

    result = {
      'clusters': [cluster_spec],
      'name': 'Deploy',
      'refId': '1',
      'requisiteStageRefIds': [],
      'type': 'deploy'
    }
    return result

  def make_upsert_load_balancer_stage(self):
    result = {
        'cloudProvider': 'appengine',
        'loadBalancers': [
        {
            'cloudProvider': 'appengine',
            'credentials': self.bindings['SPINNAKER_APPENGINE_ACCOUNT'],
            'loadBalancerName': self.__lb_name,
            'migrateTraffic': False,
            'name': self.__lb_name,
            'region': 'us-central',
            'splitDescription': {
                'allocationDescriptions': [
                {
                    'allocation': 0.1,
                    'cluster': self.__cluster_name,
                    'locatorType': 'targetCoordinate',
                   'target': 'current_asg_dynamic'
                },
                {
                    'allocation': 0.9,
                    'cluster': self.__cluster_name,
                    'locatorType': 'targetCoordinate',
                    'target': 'ancestor_asg_dynamic'
                }
                ],
                'shardBy': 'IP'
            }
        }
      ],
      'name': 'Edit Load Balancer',
      'refId': '2',
      'requisiteStageRefIds': ['1'],
      'type': 'upsertAppEngineLoadBalancers'
    }
    return result

  def create_deploy_upsert_load_balancer_pipeline(self):
    name = 'promoteServerGroupPipeline'
    self.pipeline_id = name
    deploy_stage = self.make_deploy_stage()
    upsert_load_balancer_stage = self.make_upsert_load_balancer_stage()

    pipeline_spec = dict(
        name=name,
        stages=[deploy_stage, upsert_load_balancer_stage],
        triggers=[],
        application=self.TEST_APP,
        stageCounter=2,
        parallel=True,
        limitConcurrent=True,
        appConfig={},
        index=0
    )

    payload = self.agent.make_json_payload_from_kwargs(**pipeline_spec)

    builder = st.HttpContractBuilder(self.agent)
    (builder.new_clause_builder('Has Pipeline',
                                retryable_for_secs=5)
     .get_url_path('applications/{0}/pipelineConfigs'.format(self.TEST_APP))
     .contains_path_value(None, pipeline_spec))

    return st.OperationContract(
      self.new_post_operation(
        title='create_deploy_upsert_load_balancer_pipeline', data=payload, path='pipelines',
        status_class=st.SynchronousHttpOperationStatus),
      contract=builder.build())

  def run_deploy_upsert_load_balancer_pipeline(self):
    url_path = 'pipelines/{0}/{1}'.format(self.TEST_APP, self.pipeline_id)

    previous_group_name = frigga.Naming.server_group(
        app=self.TEST_APP,
        stack=self.TEST_STACK,
        version='v000')

    deployed_group_name = frigga.Naming.server_group(
        app=self.TEST_APP,
        stack=self.TEST_STACK,
        version='v001')

    payload = self.agent.make_json_payload_from_kwargs(
        type='manual',
        user='[anonymous]')

    builder = gcp.GcpContractBuilder(self.appengine_observer)
    (builder.new_clause_builder('Service Modified', retryable_for_secs=30)
     .inspect_resource('apps.services',
                       self.__lb_name,
                       appsId=self.__gcp_project)
     .EXPECT(
         ov_factory.value_list_path_contains(
             jp.build_path('split', 'allocations'),
             jp.DICT_MATCHES({previous_group_name: jp.NUM_EQ(0.9),
                              deployed_group_name: jp.NUM_EQ(0.1)}))))

    return st.OperationContract(
        self.new_post_operation(
            title='run_deploy_upsert_load_balancer_pipeline',
            data=payload, path=url_path),
        builder.build())

  def delete_load_balancer(self):
    bindings = self.bindings
    payload = self.agent.make_json_payload_from_kwargs(
        job=[{
            'type': 'deleteLoadBalancer',
            'cloudProvider': 'appengine',
            'loadBalancerName': self.__lb_name,
            'account': bindings['SPINNAKER_APPENGINE_ACCOUNT'],
            'credentials': bindings['SPINNAKER_APPENGINE_ACCOUNT'],
            'user': '[anonymous]'
        }],
        description='Delete Load Balancer: {0} in {1}'.format(
            self.__lb_name,
            bindings['SPINNAKER_APPENGINE_ACCOUNT']),
        application=self.TEST_APP)

    builder = gcp.GcpContractBuilder(self.appengine_observer)
    (builder.new_clause_builder('Service Deleted', retryable_for_secs=30)
     .inspect_resource('apps.services',
                       self.__lb_name,
                       appsId=self.__gcp_project)
     .EXPECT(
         ov_factory.error_list_contains(gcp.HttpErrorPredicate(http_code=404))))

    return st.OperationContract(
        self.new_post_operation(
            title='delete_load_balancer', data=payload, path='tasks'),
        contract=builder.build())

class AppengineSmokeTest(st.AgentTestCase):

  @property
  def scenario(self):
    return citest.base.TestRunner.global_runner().get_shared_data(AppengineSmokeTestScenario)

  def test_a_create_app(self):
    self.run_test_case(self.scenario.create_app())

  def test_b_create_server_group(self):
    self.run_test_case(self.scenario.create_server_group())

  def test_c_create_pipeline(self):
    self.run_test_case(self.scenario.create_deploy_upsert_load_balancer_pipeline())

  def test_d_run_pipeline(self):
    self.run_test_case(self.scenario.run_deploy_upsert_load_balancer_pipeline())

  def test_y_delete_load_balancer(self):
    self.run_test_case(self.scenario.delete_load_balancer(),
                       retry_interval_secs=8, max_retries=8)

  def test_z_delete_app(self):
    self.run_test_case(self.scenario.delete_app(),
                       retry_interval_secs=8, max_retries=8)

def main():
  defaults = {
      'TEST_STACK': AppengineSmokeTestScenario.DEFAULT_TEST_ID,
      'TEST_APP': 'gaesmoketest' + AppengineSmokeTestScenario.DEFAULT_TEST_ID,
  }

  return citest.base.TestRunner.main(
      parser_inits=[AppengineSmokeTestScenario.initArgumentParser],
      default_binding_overrides=defaults,
      test_case_list=[AppengineSmokeTest])

if __name__ == '__main__':
  sys.exit(main())
<|endoftext|>


<commit_before>import unittest

import numpy as np

import chainer
from chainer.dataset import tabular
from chainer import testing

class TestFromData(unittest.TestCase):

    def test_unary_array(self):
        dataset = tabular.from_data(np.arange(10))

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(len(dataset.keys), 1)
        self.assertIsNone(dataset.mode)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, [1, 3])
        self.assertIsInstance(output, np.ndarray)

    def test_unary_array_with_key(self):
        dataset = tabular.from_data(('a', np.arange(10)))

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(dataset.keys, ('a',))
        self.assertIsNone(dataset.mode)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, [1, 3])
        self.assertIsInstance(output, np.ndarray)

    def test_unary_list(self):
        dataset = tabular.from_data([2, 7, 1, 8, 4, 5, 9, 0, 3, 6])

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(len(dataset.keys), 1)
        self.assertIsNone(dataset.mode)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, [7, 8])
        self.assertIsInstance(output, list)

    def test_unary_list_with_key(self):
        dataset = tabular.from_data(('a', [2, 7, 1, 8, 4, 5, 9, 0, 3, 6]))

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(dataset.keys, ('a',))
        self.assertIsNone(dataset.mode)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, [7, 8])
        self.assertIsInstance(output, list)

    def test_unary_callable_unary(self):
        dataset = tabular.from_data(('a', lambda i: i * i), size=10)

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(dataset.keys, ('a',))
        self.assertIsNone(dataset.mode)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, [1, 9])
        self.assertIsInstance(output, list)

    def test_unary_callable_tuple(self):
        dataset = tabular.from_data(
            (('a', 'b'), lambda i: (i * i, -i)), size=10)

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(dataset.keys, ('a', 'b'))
        self.assertEqual(dataset.mode, tuple)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, ([1, 9], [-1, -3]))
        for out in output:
            self.assertIsInstance(out, list)

    def test_unary_callable_dict(self):
        dataset = tabular.from_data(
            (('a', 'b'), lambda i: {'a': i * i, 'b': -i}), size=10)

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(dataset.keys, ('a', 'b'))
        self.assertEqual(dataset.mode, dict)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, {'a': [1, 9], 'b': [-1, -3]})
        for out in output.values():
            self.assertIsInstance(out, list)

    def test_unary_callable_without_key(self):
        with self.assertRaises(ValueError):
            tabular.from_data(lambda i: i * i, size=10)

    def test_unary_callable_without_size(self):
        with self.assertRaises(ValueError):
            tabular.from_data(('a', lambda i: i * i))

    def test_tuple_array_list(self):
        dataset = tabular.from_data(
            (np.arange(10), [2, 7, 1, 8, 4, 5, 9, 0, 3, 6]))

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(len(dataset.keys), 2)
        self.assertEqual(dataset.mode, tuple)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, ([1, 3], [7, 8]))
        self.assertIsInstance(output[0], np.ndarray)
        self.assertIsInstance(output[1], list)

    def test_tuple_array_with_key_list(self):
        dataset = tabular.from_data(
            (('a', np.arange(10)), [2, 7, 1, 8, 4, 5, 9, 0, 3, 6]))

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(len(dataset.keys), 2)
        self.assertEqual(dataset.keys[0], 'a')
        self.assertEqual(dataset.mode, tuple)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, ([1, 3], [7, 8]))
        self.assertIsInstance(output[0], np.ndarray)
        self.assertIsInstance(output[1], list)

    def test_tuple_array_list_with_key(self):
        dataset = tabular.from_data(
            (np.arange(10), ('b', [2, 7, 1, 8, 4, 5, 9, 0, 3, 6])))

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(len(dataset.keys), 2)
        self.assertEqual(dataset.keys[1], 'b')
        self.assertEqual(dataset.mode, tuple)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, ([1, 3], [7, 8]))
        self.assertIsInstance(output[0], np.ndarray)
        self.assertIsInstance(output[1], list)

    def test_tuple_array_callable_unary(self):
        dataset = tabular.from_data((np.arange(10), ('b', lambda i: i * i)))

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(len(dataset.keys), 2)
        self.assertEqual(dataset.keys[1], 'b')
        self.assertEqual(dataset.mode, tuple)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, ([1, 3], [1, 9]))
        self.assertIsInstance(output[0], np.ndarray)
        self.assertIsInstance(output[1], list)

    def test_tuple_array_callable_tuple(self):
        dataset = tabular.from_data(
            (np.arange(10), (('b', 'c'), lambda i: (i * i, -i))))

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(len(dataset.keys), 3)
        self.assertEqual(dataset.keys[1:], ('b', 'c'))
        self.assertEqual(dataset.mode, tuple)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, ([1, 3], [1, 9], [-1, -3]))
        self.assertIsInstance(output[0], np.ndarray)
        self.assertIsInstance(output[1], list)

    def test_tuple_array_callable_dict(self):
        dataset = tabular.from_data(
            (np.arange(10), (('b', 'c'), lambda i: {'b': i * i, 'c': -i})))

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(len(dataset.keys), 3)
        self.assertEqual(dataset.keys[1:], ('b', 'c'))
        self.assertEqual(dataset.mode, tuple)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, ([1, 3], [1, 9], [-1, -3]))
        self.assertIsInstance(output[0], np.ndarray)
        self.assertIsInstance(output[1], list)

    def test_tuple_array_with_key_callable_unary(self):
        dataset = tabular.from_data(
            (('a', np.arange(10)), ('b', lambda i: i * i)))

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(dataset.keys, ('a', 'b'))
        self.assertEqual(dataset.mode, tuple)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, ([1, 3], [1, 9]))
        self.assertIsInstance(output[0], np.ndarray)
        self.assertIsInstance(output[1], list)

    def test_tuple_callable_unary_callable_unary(self):
        dataset = tabular.from_data(
            (('a', lambda i: i * i), ('b', lambda i: -i)), size=10)

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(dataset.keys, ('a', 'b'))
        self.assertEqual(dataset.mode, tuple)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, ([1, 9], [-1, -3]))
        self.assertIsInstance(output[0], list)
        self.assertIsInstance(output[1], list)

    def test_tuple_callable_unary_callable_unary_without_size(self):
        with self.assertRaises(ValueError):
            tabular.from_data((('a', lambda i: i * i), ('b', lambda i: -i)))

    def test_dict_array_list(self):
        dataset = tabular.from_data(
            {'a': np.arange(10), 'b': [2, 7, 1, 8, 4, 5, 9, 0, 3, 6]})

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(set(dataset.keys), {'a', 'b'})
        self.assertEqual(dataset.mode, dict)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, {'a': [1, 3], 'b': [7, 8]})
        self.assertIsInstance(output['a'], np.ndarray)
        self.assertIsInstance(output['b'], list)

    def test_dict_array_callable_unary(self):
        dataset = tabular.from_data({'a': np.arange(10), 'b': lambda i: i * i})

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(set(dataset.keys), {'a', 'b'})
        self.assertEqual(dataset.mode, dict)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, {'a': [1, 3], 'b': [1, 9]})
        self.assertIsInstance(output['a'], np.ndarray)
        self.assertIsInstance(output['b'], list)

    def test_dict_array_callable_tuple(self):
        dataset = tabular.from_data(
            {'a': np.arange(10), ('b', 'c'): lambda i: (i * i, -i)})

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(set(dataset.keys), {'a', 'b', 'c'})
        self.assertEqual(dataset.mode, dict)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(
            output, {'a': [1, 3], 'b': [1, 9], 'c': [-1, -3]})
        self.assertIsInstance(output['a'], np.ndarray)
        self.assertIsInstance(output['b'], list)
        self.assertIsInstance(output['c'], list)

    def test_dict_array_callable_dict(self):
        dataset = tabular.from_data(
            {'a': np.arange(10), ('b', 'c'): lambda i: {'b': i * i, 'c': -i}})

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(set(dataset.keys), {'a', 'b', 'c'})
        self.assertEqual(dataset.mode, dict)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(
            output, {'a': [1, 3], 'b': [1, 9], 'c': [-1, -3]})
        self.assertIsInstance(output['a'], np.ndarray)
        self.assertIsInstance(output['b'], list)
        self.assertIsInstance(output['c'], list)

    def test_dict_callable_unary_callable_unary(self):
        dataset = tabular.from_data(
            {'a': lambda i: i * i, 'b': lambda i: -i}, size=10)

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(dataset.keys, ('a', 'b'))
        self.assertEqual(dataset.mode, dict)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, {'a': [1, 9], 'b': [-1, -3]})
        self.assertIsInstance(output['a'], list)
        self.assertIsInstance(output['b'], list)

    def test_dict_callable_unary_callable_unary_without_size(self):
        with self.assertRaises(ValueError):
            tabular.from_data(({'a': lambda i: i * i, 'b': lambda i: -i}))

    def test_unique(self):
        dataset_a = tabular.from_data(np.arange(10))
        dataset_b = tabular.from_data(np.arange(10))
        self.assertNotEqual(dataset_a.keys, dataset_b.keys)

testing.run_module(__name__, __file__)
<commit_msg>fix test<commit_after>import unittest

import numpy as np

import chainer
from chainer.dataset import tabular
from chainer import testing

class TestFromData(unittest.TestCase):

    def test_unary_array(self):
        dataset = tabular.from_data(np.arange(10))

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(len(dataset.keys), 1)
        self.assertIsNone(dataset.mode)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, [1, 3])
        self.assertIsInstance(output, np.ndarray)

    def test_unary_array_with_key(self):
        dataset = tabular.from_data(('a', np.arange(10)))

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(dataset.keys, ('a',))
        self.assertIsNone(dataset.mode)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, [1, 3])
        self.assertIsInstance(output, np.ndarray)

    def test_unary_list(self):
        dataset = tabular.from_data([2, 7, 1, 8, 4, 5, 9, 0, 3, 6])

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(len(dataset.keys), 1)
        self.assertIsNone(dataset.mode)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, [7, 8])
        self.assertIsInstance(output, list)

    def test_unary_list_with_key(self):
        dataset = tabular.from_data(('a', [2, 7, 1, 8, 4, 5, 9, 0, 3, 6]))

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(dataset.keys, ('a',))
        self.assertIsNone(dataset.mode)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, [7, 8])
        self.assertIsInstance(output, list)

    def test_unary_callable_unary(self):
        dataset = tabular.from_data(('a', lambda i: i * i), size=10)

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(dataset.keys, ('a',))
        self.assertIsNone(dataset.mode)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, [1, 9])
        self.assertIsInstance(output, list)

    def test_unary_callable_tuple(self):
        dataset = tabular.from_data(
            (('a', 'b'), lambda i: (i * i, -i)), size=10)

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(dataset.keys, ('a', 'b'))
        self.assertEqual(dataset.mode, tuple)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, ([1, 9], [-1, -3]))
        for out in output:
            self.assertIsInstance(out, list)

    def test_unary_callable_dict(self):
        dataset = tabular.from_data(
            (('a', 'b'), lambda i: {'a': i * i, 'b': -i}), size=10)

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(dataset.keys, ('a', 'b'))
        self.assertEqual(dataset.mode, dict)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, {'a': [1, 9], 'b': [-1, -3]})
        for out in output.values():
            self.assertIsInstance(out, list)

    def test_unary_callable_without_key(self):
        with self.assertRaises(ValueError):
            tabular.from_data(lambda i: i * i, size=10)

    def test_unary_callable_without_size(self):
        with self.assertRaises(ValueError):
            tabular.from_data(('a', lambda i: i * i))

    def test_tuple_array_list(self):
        dataset = tabular.from_data(
            (np.arange(10), [2, 7, 1, 8, 4, 5, 9, 0, 3, 6]))

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(len(dataset.keys), 2)
        self.assertEqual(dataset.mode, tuple)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, ([1, 3], [7, 8]))
        self.assertIsInstance(output[0], np.ndarray)
        self.assertIsInstance(output[1], list)

    def test_tuple_array_with_key_list(self):
        dataset = tabular.from_data(
            (('a', np.arange(10)), [2, 7, 1, 8, 4, 5, 9, 0, 3, 6]))

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(len(dataset.keys), 2)
        self.assertEqual(dataset.keys[0], 'a')
        self.assertEqual(dataset.mode, tuple)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, ([1, 3], [7, 8]))
        self.assertIsInstance(output[0], np.ndarray)
        self.assertIsInstance(output[1], list)

    def test_tuple_array_list_with_key(self):
        dataset = tabular.from_data(
            (np.arange(10), ('b', [2, 7, 1, 8, 4, 5, 9, 0, 3, 6])))

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(len(dataset.keys), 2)
        self.assertEqual(dataset.keys[1], 'b')
        self.assertEqual(dataset.mode, tuple)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, ([1, 3], [7, 8]))
        self.assertIsInstance(output[0], np.ndarray)
        self.assertIsInstance(output[1], list)

    def test_tuple_array_callable_unary(self):
        dataset = tabular.from_data((np.arange(10), ('b', lambda i: i * i)))

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(len(dataset.keys), 2)
        self.assertEqual(dataset.keys[1], 'b')
        self.assertEqual(dataset.mode, tuple)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, ([1, 3], [1, 9]))
        self.assertIsInstance(output[0], np.ndarray)
        self.assertIsInstance(output[1], list)

    def test_tuple_array_callable_tuple(self):
        dataset = tabular.from_data(
            (np.arange(10), (('b', 'c'), lambda i: (i * i, -i))))

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(len(dataset.keys), 3)
        self.assertEqual(dataset.keys[1:], ('b', 'c'))
        self.assertEqual(dataset.mode, tuple)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, ([1, 3], [1, 9], [-1, -3]))
        self.assertIsInstance(output[0], np.ndarray)
        self.assertIsInstance(output[1], list)

    def test_tuple_array_callable_dict(self):
        dataset = tabular.from_data(
            (np.arange(10), (('b', 'c'), lambda i: {'b': i * i, 'c': -i})))

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(len(dataset.keys), 3)
        self.assertEqual(dataset.keys[1:], ('b', 'c'))
        self.assertEqual(dataset.mode, tuple)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, ([1, 3], [1, 9], [-1, -3]))
        self.assertIsInstance(output[0], np.ndarray)
        self.assertIsInstance(output[1], list)

    def test_tuple_array_with_key_callable_unary(self):
        dataset = tabular.from_data(
            (('a', np.arange(10)), ('b', lambda i: i * i)))

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(dataset.keys, ('a', 'b'))
        self.assertEqual(dataset.mode, tuple)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, ([1, 3], [1, 9]))
        self.assertIsInstance(output[0], np.ndarray)
        self.assertIsInstance(output[1], list)

    def test_tuple_callable_unary_callable_unary(self):
        dataset = tabular.from_data(
            (('a', lambda i: i * i), ('b', lambda i: -i)), size=10)

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(dataset.keys, ('a', 'b'))
        self.assertEqual(dataset.mode, tuple)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, ([1, 9], [-1, -3]))
        self.assertIsInstance(output[0], list)
        self.assertIsInstance(output[1], list)

    def test_tuple_callable_unary_callable_unary_without_size(self):
        with self.assertRaises(ValueError):
            tabular.from_data((('a', lambda i: i * i), ('b', lambda i: -i)))

    def test_dict_array_list(self):
        dataset = tabular.from_data(
            {'a': np.arange(10), 'b': [2, 7, 1, 8, 4, 5, 9, 0, 3, 6]})

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(set(dataset.keys), {'a', 'b'})
        self.assertEqual(dataset.mode, dict)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, {'a': [1, 3], 'b': [7, 8]})
        self.assertIsInstance(output['a'], np.ndarray)
        self.assertIsInstance(output['b'], list)

    def test_dict_array_callable_unary(self):
        dataset = tabular.from_data({'a': np.arange(10), 'b': lambda i: i * i})

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(set(dataset.keys), {'a', 'b'})
        self.assertEqual(dataset.mode, dict)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, {'a': [1, 3], 'b': [1, 9]})
        self.assertIsInstance(output['a'], np.ndarray)
        self.assertIsInstance(output['b'], list)

    def test_dict_array_callable_tuple(self):
        dataset = tabular.from_data(
            {'a': np.arange(10), ('b', 'c'): lambda i: (i * i, -i)})

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(set(dataset.keys), {'a', 'b', 'c'})
        self.assertEqual(dataset.mode, dict)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(
            output, {'a': [1, 3], 'b': [1, 9], 'c': [-1, -3]})
        self.assertIsInstance(output['a'], np.ndarray)
        self.assertIsInstance(output['b'], list)
        self.assertIsInstance(output['c'], list)

    def test_dict_array_callable_dict(self):
        dataset = tabular.from_data(
            {'a': np.arange(10), ('b', 'c'): lambda i: {'b': i * i, 'c': -i}})

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(set(dataset.keys), {'a', 'b', 'c'})
        self.assertEqual(dataset.mode, dict)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(
            output, {'a': [1, 3], 'b': [1, 9], 'c': [-1, -3]})
        self.assertIsInstance(output['a'], np.ndarray)
        self.assertIsInstance(output['b'], list)
        self.assertIsInstance(output['c'], list)

    def test_dict_callable_unary_callable_unary(self):
        dataset = tabular.from_data(
            {'a': lambda i: i * i, 'b': lambda i: -i}, size=10)

        self.assertIsInstance(dataset, chainer.dataset.TabularDataset)
        self.assertEqual(len(dataset), 10)
        self.assertEqual(set(dataset.keys), {'a', 'b'})
        self.assertEqual(dataset.mode, dict)

        output = dataset.slice[[1, 3]].fetch()
        np.testing.assert_equal(output, {'a': [1, 9], 'b': [-1, -3]})
        self.assertIsInstance(output['a'], list)
        self.assertIsInstance(output['b'], list)

    def test_dict_callable_unary_callable_unary_without_size(self):
        with self.assertRaises(ValueError):
            tabular.from_data(({'a': lambda i: i * i, 'b': lambda i: -i}))

    def test_unique(self):
        dataset_a = tabular.from_data(np.arange(10))
        dataset_b = tabular.from_data(np.arange(10))
        self.assertNotEqual(dataset_a.keys, dataset_b.keys)

testing.run_module(__name__, __file__)
<|endoftext|>


<commit_before>import unittest

import numpy

import chainer
from chainer.backends import cuda
from chainer import functions
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr

@testing.parameterize(*testing.product_dict(
    [{'shape': (3,), 'dtype': 'f', 'axis': 0, 'inv': False},
     {'shape': (3,), 'dtype': 'f', 'axis': -1, 'inv': True},
     {'shape': (3, 4), 'dtype': 'd', 'axis': 1, 'inv': True},
     {'shape': (3, 4, 5), 'dtype': 'f', 'axis': 2, 'inv': False}],
    [{'label_dtype': numpy.int8},
     {'label_dtype': numpy.int16},
     {'label_dtype': numpy.int32},
     {'label_dtype': numpy.int64}]
))
class TestPermutate(unittest.TestCase):

    def setUp(self):
        self.x = numpy.random.uniform(-1, 1, self.shape).astype(self.dtype)
        self.g = numpy.random.uniform(-1, 1, self.shape).astype(self.dtype)
        self.indices = numpy.random.permutation(
            self.shape[self.axis]).astype(self.label_dtype)

    def check_forward(self, x_data, ind_data):
        x = chainer.Variable(x_data)
        indices = chainer.Variable(ind_data)
        y = functions.permutate(x, indices, axis=self.axis, inv=self.inv)

        y_cpu = cuda.to_cpu(y.data)
        y_cpu = numpy.rollaxis(y_cpu, axis=self.axis)
        x_data = numpy.rollaxis(self.x, axis=self.axis)
        for i, ind in enumerate(self.indices):
            if self.inv:
                numpy.testing.assert_array_equal(y_cpu[ind], x_data[i])
            else:
                numpy.testing.assert_array_equal(y_cpu[i], x_data[ind])

    def test_forward_cpu(self):
        self.check_forward(self.x, self.indices)

    @attr.gpu
    def test_forward_gpu(self):
        self.check_forward(cuda.to_gpu(self.x), cuda.to_gpu(self.indices))

    def check_backward(self, x_data, ind_data, g_data):
        def fun(x, ind):
            return functions.permutate(x, ind, self.axis, self.inv)
        gradient_check.check_backward(
            fun, (x_data, ind_data), g_data, dtype='d', atol=0.001, rtol=0.001)

    def test_backward_cpu(self):
        self.check_backward(self.x, self.indices, self.g)

    @attr.gpu
    def test_backward_gpu(self):
        self.check_backward(cuda.to_gpu(self.x),
                            cuda.to_gpu(self.indices),
                            cuda.to_gpu(self.g))

@testing.parameterize(
    {'indices': [0, 0]},
    {'indices': [-1, 0]},
    {'indices': [0, 2]},
)
class TestPermutateInvalidIndices(unittest.TestCase):

    def setUp(self):
        self.x = numpy.arange(10).reshape((2, 5)).astype('f')
        self.ind = numpy.array(self.indices, 'i')
        self.debug = chainer.is_debug()
        chainer.set_debug(True)

    def tearDown(self):
        chainer.set_debug(self.debug)

    def check_invalid(self, x_data, ind_data):
        x = chainer.Variable(x_data)
        ind = chainer.Variable(ind_data)
        with self.assertRaises(ValueError):
            functions.permutate(x, ind)

    def test_invlaid_cpu(self):
        self.check_invalid(self.x, self.ind)

    @attr.gpu
    def test_invlaid_gpu(self):
        self.check_invalid(cuda.to_gpu(self.x), cuda.to_gpu(self.ind))

testing.run_module(__name__, __file__)
<commit_msg>Add test: permutate gpu array by cpu indices<commit_after>import unittest

import numpy

import chainer
from chainer.backends import cuda
from chainer import functions
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr

@testing.parameterize(*testing.product_dict(
    [{'shape': (3,), 'dtype': 'f', 'axis': 0, 'inv': False},
     {'shape': (3,), 'dtype': 'f', 'axis': -1, 'inv': True},
     {'shape': (3, 4), 'dtype': 'd', 'axis': 1, 'inv': True},
     {'shape': (3, 4, 5), 'dtype': 'f', 'axis': 2, 'inv': False}],
    [{'label_dtype': numpy.int8},
     {'label_dtype': numpy.int16},
     {'label_dtype': numpy.int32},
     {'label_dtype': numpy.int64}]
))
class TestPermutate(unittest.TestCase):

    def setUp(self):
        self.x = numpy.random.uniform(-1, 1, self.shape).astype(self.dtype)
        self.g = numpy.random.uniform(-1, 1, self.shape).astype(self.dtype)
        self.indices = numpy.random.permutation(
            self.shape[self.axis]).astype(self.label_dtype)

    def check_forward(self, x_data, ind_data):
        x = chainer.Variable(x_data)
        indices = chainer.Variable(ind_data)
        y = functions.permutate(x, indices, axis=self.axis, inv=self.inv)

        y_cpu = cuda.to_cpu(y.data)
        y_cpu = numpy.rollaxis(y_cpu, axis=self.axis)
        x_data = numpy.rollaxis(self.x, axis=self.axis)
        for i, ind in enumerate(self.indices):
            if self.inv:
                numpy.testing.assert_array_equal(y_cpu[ind], x_data[i])
            else:
                numpy.testing.assert_array_equal(y_cpu[i], x_data[ind])

    def test_forward_cpu(self):
        self.check_forward(self.x, self.indices)

    @attr.gpu
    def test_forward_gpu(self):
        self.check_forward(cuda.to_gpu(self.x), cuda.to_gpu(self.indices))

    @attr.gpu
    def test_forward_mixed(self):
        self.check_forward(cuda.to_gpu(self.x), self.indices)

    def check_backward(self, x_data, ind_data, g_data):
        def fun(x, ind):
            return functions.permutate(x, ind, self.axis, self.inv)
        gradient_check.check_backward(
            fun, (x_data, ind_data), g_data, dtype='d', atol=0.001, rtol=0.001)

    def test_backward_cpu(self):
        self.check_backward(self.x, self.indices, self.g)

    @attr.gpu
    def test_backward_gpu(self):
        self.check_backward(cuda.to_gpu(self.x),
                            cuda.to_gpu(self.indices),
                            cuda.to_gpu(self.g))

@testing.parameterize(
    {'indices': [0, 0]},
    {'indices': [-1, 0]},
    {'indices': [0, 2]},
)
class TestPermutateInvalidIndices(unittest.TestCase):

    def setUp(self):
        self.x = numpy.arange(10).reshape((2, 5)).astype('f')
        self.ind = numpy.array(self.indices, 'i')
        self.debug = chainer.is_debug()
        chainer.set_debug(True)

    def tearDown(self):
        chainer.set_debug(self.debug)

    def check_invalid(self, x_data, ind_data):
        x = chainer.Variable(x_data)
        ind = chainer.Variable(ind_data)
        with self.assertRaises(ValueError):
            functions.permutate(x, ind)

    def test_invlaid_cpu(self):
        self.check_invalid(self.x, self.ind)

    @attr.gpu
    def test_invlaid_gpu(self):
        self.check_invalid(cuda.to_gpu(self.x), cuda.to_gpu(self.ind))

testing.run_module(__name__, __file__)
<|endoftext|>


<commit_before><commit_msg>deleted connector folder<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#     This file is part of connector, an Odoo module.
#
#     Author: Stéphane Bidoul <stephane.bidoul@acsone.eu>
#     Copyright (c) 2015 ACSONE SA/NV (<http://acsone.eu>)
#
#     connector is free software: you can redistribute it and/or
#     modify it under the terms of the GNU Affero General Public License
#     as published by the Free Software Foundation, either version 3 of
#     the License, or (at your option) any later version.
#
#     connector is distributed in the hope that it will be useful,
#     but WITHOUT ANY WARRANTY; without even the implied warranty of
#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#     GNU Affero General Public License for more details.
#
#     You should have received a copy of the
#     GNU Affero General Public License
#     along with connector.
#     If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from datetime import datetime
from heapq import heappush, heappop
import logging
from weakref import WeakValueDictionary

_logger = logging.getLogger(__name__)

STATE_PENDING = 'pending'
STATE_ENQUEUED = 'enqueued'
STATE_STARTED = 'started'
STATE_FAILED = 'failed'
STATE_DONE = 'done'

STATES_NOT_DONE = (STATE_PENDING, STATE_ENQUEUED, STATE_STARTED, STATE_FAILED)

class PriorityQueue:
    """A priority queue that supports removing arbitrary objects.

    Adding an object already in the queue is a no op.
    Popping an empty queue returns None.

    >>> q = PriorityQueue()
    >>> q.add(2)
    >>> q.add(3)
    >>> q.add(3)
    >>> q.add(1)
    >>> q[0]
    1
    >>> len(q)
    3
    >>> q.pop()
    1
    >>> q.remove(2)
    >>> len(q)
    1
    >>> q[0]
    3
    >>> q.pop()
    3
    >>> q.pop()
    >>> q.add(2)
    >>> q.remove(2)
    >>> q.add(2)
    >>> q.pop()
    2
    """

    def __init__(self):
        self._heap = []
        self._known = set()    # all objects in the heap (including removed)
        self._removed = set()  # all objects that have been removed

    def __len__(self):
        return len(self._known) - len(self._removed)

    def __getitem__(self, i):
        if i != 0:
            raise IndexError()
        while True:
            if not self._heap:
                raise IndexError()
            o = self._heap[0]
            if o in self._removed:
                o2 = heappop(self._heap)
                assert o2 == o
                self._removed.remove(o)
                self._known.remove(o)
            else:
                return o

    def __contains__(self, o):
        return o in self._known and o not in self._removed

    def add(self, o):
        if o is None:
            raise ValueError()
        if o in self._removed:
            self._removed.remove(o)
        if o in self._known:
            return
        self._known.add(o)
        heappush(self._heap, o)

    def remove(self, o):
        if o is None:
            raise ValueError()
        if o not in self._known:
            return
        if o not in self._removed:
            self._removed.add(o)

    def pop(self):
        while True:
            try:
                o = heappop(self._heap)
            except IndexError:
                # queue is empty
                return None
            self._known.remove(o)
            if o in self._removed:
                self._removed.remove(o)
            else:
                return o

class SafeSet(set):
    """A set that does not raise KeyError when removing non-existent items.

    >>> s = SafeSet()
    >>> s.remove(1)
    >>> len(s)
    0
    >>> s.remove(1)
    """
    def remove(self, o):
        try:
            super(SafeSet, self).remove(o)
        except KeyError:
            pass

class ChannelJob:
    """A channel job is attached to a channel and holds the properties of a
    job that are necessary to prioritise them.

    Channel jobs are comparable according to the following rules:
        * jobs with an eta come before all other jobs
        * then jobs with a smaller eta come first
        * then jobs with smaller priority come first
        * then jobs with a smaller creation time come first
        * then jobs with a samller sequence come first

    Here are some examples.

    j1 comes before j2 before it has a smaller date_created
    >>> j1 = ChannelJob(None, 1, seq=0, date_created=1, priority=9, eta=None)
    >>> j1
    <ChannelJob 1>
    >>> j2 = ChannelJob(None, 2, seq=0, date_created=2, priority=9, eta=None)
    >>> j1 < j2
    True

    j3 comes first because it has lower priority,
    despite having a creation date after j1 and j2
    >>> j3 = ChannelJob(None, 3, seq=0, date_created=3, priority=2, eta=None)
    >>> j3 < j1
    True

    j4 and j5 comes even before j3, because they have an eta
    >>> j4 = ChannelJob(None, 4, seq=0, date_created=4, priority=9, eta=9)
    >>> j5 = ChannelJob(None, 5, seq=0, date_created=5, priority=9, eta=9)
    >>> j4 < j5 < j3
    True

    j6 has same date_created and priority as j5 but a smaller eta
    >>> j6 = ChannelJob(None, 6, seq=0, date_created=5, priority=9, eta=2)
    >>> j6 < j4 < j5
    True

    Here is the complete suite:
    >>> j6 < j4 < j5 < j3 < j1 < j2
    True

    j0 has the same properties as j1 but they are not considered
    equal as they are different instances
    >>> j0 = ChannelJob(None, 1, seq=0, date_created=1, priority=9, eta=None)
    >>> j0 == j1
    False
    >>> j0 == j0
    True
    """

    def __init__(self, channel, uuid, seq, date_created, priority, eta):
        self.uuid = uuid
        self.channel = channel
        self.seq = seq
        self.date_created = date_created
        self.priority = priority
        self.eta = eta

    def __repr__(self):
        return "<ChannelJob %s>" % self.uuid

    def __eq__(self, other):
        return id(self) == id(other)

    def __hash__(self):
        return id(self)

    def __cmp__(self, other):
        if self.eta and not other.eta:
            return -1
        elif not self.eta and other.eta:
            return 1
        else:
            return (cmp(self.eta, other.eta) or
                    cmp(self.priority, other.priority) or
                    cmp(self.date_created, other.date_created))

class ChannelQueue:
    """A channel queue is a priority queue for jobs, that returns
    jobs with a past ETA first.

    >>> q = ChannelQueue()
    >>> j1 = ChannelJob(None, 1, seq=0, date_created=1, priority=1, eta=10)
    >>> j2 = ChannelJob(None, 2, seq=0, date_created=2, priority=1, eta=None)
    >>> j3 = ChannelJob(None, 3, seq=0, date_created=3, priority=1, eta=None)
    >>> q.add(j1)
    >>> q.add(j2)
    >>> q.add(j3)
    >>> q.pop(now=1)
    <ChannelJob 2>
    >>> q.pop(now=11)
    <ChannelJob 1>
    >>> q.pop(now=12)
    <ChannelJob 3>
    """

    def __init__(self):
        self._queue = PriorityQueue()
        self._eta_queue = PriorityQueue()

    def __len__(self):
        return len(self._eta_queue) + len(self._queue)

    def __contains__(self, o):
        return o in self._eta_queue or o in self._queue

    def add(self, job):
        if job.eta:
            self._eta_queue.add(job)
        else:
            self._queue.add(job)

    def remove(self, job):
        self._eta_queue.remove(job)
        self._queue.remove(job)

    def pop(self, now):
        if len(self._eta_queue) and self._eta_queue[0].eta <= now:
            return self._eta_queue.pop()
        else:
            return self._queue.pop()

class Channel:
    """A channel for jobs, with a maximum number of workers.

    Job channels are joined in a hierarchy down to the root channel.
    When a job channel has free workers, jobs are dequeued, marked
    as running in the channel and are inserted into the queue of the
    parent channel where they wait for free workers and so on.

    Job channels can be visualized as water channels with a given flow
    limit (= workers). Channels are joined together in a downstream channel
    and the flow limit of the downstream channel limit upstream channels.

    ---------------------\
                         \
                         \-----------------------
     Ch. A W:4,Q:12,R:4

    ---------------------\  Ch. root W:5,Q:0,R:4
                         |
    ---------------------/
     Ch. B W:1,Q:0,R:0    -----------------------
    ---------------------/

    The above diagram illustrates two channels joining in the root channel.
    The root channel has 5 workers, and 4 running jobs coming from Channel A.
    Channel A has maximum 4 workers, all in use (passed down to the root
    channel), and 12 jobs enqueued. Channel B has maximum 1 worker,
    none in use. This means that whenever a new job comes in channel B,
    there will be available room for it to run in the root channel.

    Should a downstream channel have less capacity than its upstream channels,
    jobs going downstream will be enqueued in the downstream channel,
    and compete normally according to their properties (priority, etc).

    Using this technique, it is possible to enforce sequence in a channel
    with 1 worker. It is also possible to dedicate a channel with a
    limited number of workers for application-autocreated subchannels
    without risking to overflow the system.
    """

    def __init__(self, name, parent, workers=1, sequential=False):
        self.name = name
        self.parent = parent
        if self.parent:
            self.parent.children.append(self)
        self.children = []
        self.workers = workers
        if sequential and workers != 1:
            raise ValueError("A sequential channel can have only one worker")
        self.sequential = sequential
        self._queue = ChannelQueue()
        self._running = SafeSet()
        self._failed = SafeSet()

    def __str__(self):
        return "%s(W:%d,Q:%d,R:%d,F:%d)" % (self.name,
                                            self.workers,
                                            len(self._queue),
                                            len(self._running),
                                            len(self._failed))

    def remove(self, job):
        self._queue.remove(job)
        self._running.remove(job)
        self._failed.remove(job)
        if self.parent:
            self.parent.remove(job)

    def set_done(self, job):
        self.remove(job)
        _logger.debug("job %s marked done", job.uuid)

    def set_pending(self, job):
        if job not in self._queue:
            self._queue.add(job)
            self._running.remove(job)
            self._failed.remove(job)
            if self.parent:
                self.parent.remove(job)
            _logger.debug("job %s marked pending", job.uuid)

    def set_running(self, job):
        if job not in self._running:
            self._queue.remove(job)
            self._running.add(job)
            self._failed.remove(job)
            if self.parent:
                self.parent.set_running(job)
            _logger.debug("job %s marked running", job.uuid)

    def set_failed(self, job):
        if job not in self._failed:
            self._queue.remove(job)
            self._running.remove(job)
            self._failed.add(job)
            if self.parent:
                self.parent.remove(job)
            _logger.debug("job %s marked failed", job.uuid)

    def get_jobs_to_run(self):
        # enqueue jobs of children channels
        for child in self.children:
            for job in child.get_jobs_to_run():
                self._queue.add(job)
        # sequential channels block when there are failed jobs
        if self.sequential and len(self._failed):
            return
        # yield jobs that are ready to run
        while len(self._running) < self.workers:
            job = self._queue.pop(now=datetime.now())
            if not job:
                return
            self._running.add(job)
            _logger.debug("job %s marked running", job.uuid)
            yield job

class ChannelManager:

    def __init__(self, workers):
        # TODO: config
        self._jobs_by_uuid = WeakValueDictionary()
        self._root_channel = Channel(name='root', parent=None, workers=workers)

    def get_channel_by_name(self, channel_name):
        # TODO: channels by name
        # TODO: autocreate/autodestroy channels
        return self._root_channel

    def notify(self, channel_name, uuid,
               seq, date_created, priority, eta, state):
        channel = self.get_channel_by_name(channel_name)
        job = self._jobs_by_uuid.get(uuid)
        if not job:
            job = ChannelJob(channel, uuid, seq, date_created, priority, eta)
            self._jobs_by_uuid[uuid] = job
        # TODO: handle sequence change
        assert job.seq == seq
        # date_created is invariant
        assert job.date_created == date_created
        # TODO: handle priority change
        assert job.priority == priority
        # TODO: handle eta change
        assert job.eta == eta
        # TODO: handle channel change
        assert job.channel == channel
        # state transitions
        if not state or state == STATE_DONE:
            channel.set_done(job)
        elif state == STATE_PENDING:
            channel.set_pending(job)
        elif state in (STATE_ENQUEUED, STATE_STARTED):
            channel.set_running(job)
        elif state == STATE_FAILED:
            channel.set_failed(job)
        else:
            _logger.error("unexpected state %s for job %s", state, job)
        _logger.debug("channel %s", self._root_channel)

    def get_jobs_to_run(self):
        for job in self._root_channel.get_jobs_to_run():
            yield job

if __name__ == '__main__':
    import doctest
    doctest.testmod()
<commit_msg>[IMP] odoo-connector-running: a clarification about job channels<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#     This file is part of connector, an Odoo module.
#
#     Author: Stéphane Bidoul <stephane.bidoul@acsone.eu>
#     Copyright (c) 2015 ACSONE SA/NV (<http://acsone.eu>)
#
#     connector is free software: you can redistribute it and/or
#     modify it under the terms of the GNU Affero General Public License
#     as published by the Free Software Foundation, either version 3 of
#     the License, or (at your option) any later version.
#
#     connector is distributed in the hope that it will be useful,
#     but WITHOUT ANY WARRANTY; without even the implied warranty of
#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#     GNU Affero General Public License for more details.
#
#     You should have received a copy of the
#     GNU Affero General Public License
#     along with connector.
#     If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from datetime import datetime
from heapq import heappush, heappop
import logging
from weakref import WeakValueDictionary

_logger = logging.getLogger(__name__)

STATE_PENDING = 'pending'
STATE_ENQUEUED = 'enqueued'
STATE_STARTED = 'started'
STATE_FAILED = 'failed'
STATE_DONE = 'done'

STATES_NOT_DONE = (STATE_PENDING, STATE_ENQUEUED, STATE_STARTED, STATE_FAILED)

class PriorityQueue:
    """A priority queue that supports removing arbitrary objects.

    Adding an object already in the queue is a no op.
    Popping an empty queue returns None.

    >>> q = PriorityQueue()
    >>> q.add(2)
    >>> q.add(3)
    >>> q.add(3)
    >>> q.add(1)
    >>> q[0]
    1
    >>> len(q)
    3
    >>> q.pop()
    1
    >>> q.remove(2)
    >>> len(q)
    1
    >>> q[0]
    3
    >>> q.pop()
    3
    >>> q.pop()
    >>> q.add(2)
    >>> q.remove(2)
    >>> q.add(2)
    >>> q.pop()
    2
    """

    def __init__(self):
        self._heap = []
        self._known = set()    # all objects in the heap (including removed)
        self._removed = set()  # all objects that have been removed

    def __len__(self):
        return len(self._known) - len(self._removed)

    def __getitem__(self, i):
        if i != 0:
            raise IndexError()
        while True:
            if not self._heap:
                raise IndexError()
            o = self._heap[0]
            if o in self._removed:
                o2 = heappop(self._heap)
                assert o2 == o
                self._removed.remove(o)
                self._known.remove(o)
            else:
                return o

    def __contains__(self, o):
        return o in self._known and o not in self._removed

    def add(self, o):
        if o is None:
            raise ValueError()
        if o in self._removed:
            self._removed.remove(o)
        if o in self._known:
            return
        self._known.add(o)
        heappush(self._heap, o)

    def remove(self, o):
        if o is None:
            raise ValueError()
        if o not in self._known:
            return
        if o not in self._removed:
            self._removed.add(o)

    def pop(self):
        while True:
            try:
                o = heappop(self._heap)
            except IndexError:
                # queue is empty
                return None
            self._known.remove(o)
            if o in self._removed:
                self._removed.remove(o)
            else:
                return o

class SafeSet(set):
    """A set that does not raise KeyError when removing non-existent items.

    >>> s = SafeSet()
    >>> s.remove(1)
    >>> len(s)
    0
    >>> s.remove(1)
    """
    def remove(self, o):
        try:
            super(SafeSet, self).remove(o)
        except KeyError:
            pass

class ChannelJob:
    """A channel job is attached to a channel and holds the properties of a
    job that are necessary to prioritise them.

    Channel jobs are comparable according to the following rules:
        * jobs with an eta come before all other jobs
        * then jobs with a smaller eta come first
        * then jobs with smaller priority come first
        * then jobs with a smaller creation time come first
        * then jobs with a samller sequence come first

    Here are some examples.

    j1 comes before j2 before it has a smaller date_created
    >>> j1 = ChannelJob(None, 1, seq=0, date_created=1, priority=9, eta=None)
    >>> j1
    <ChannelJob 1>
    >>> j2 = ChannelJob(None, 2, seq=0, date_created=2, priority=9, eta=None)
    >>> j1 < j2
    True

    j3 comes first because it has lower priority,
    despite having a creation date after j1 and j2
    >>> j3 = ChannelJob(None, 3, seq=0, date_created=3, priority=2, eta=None)
    >>> j3 < j1
    True

    j4 and j5 comes even before j3, because they have an eta
    >>> j4 = ChannelJob(None, 4, seq=0, date_created=4, priority=9, eta=9)
    >>> j5 = ChannelJob(None, 5, seq=0, date_created=5, priority=9, eta=9)
    >>> j4 < j5 < j3
    True

    j6 has same date_created and priority as j5 but a smaller eta
    >>> j6 = ChannelJob(None, 6, seq=0, date_created=5, priority=9, eta=2)
    >>> j6 < j4 < j5
    True

    Here is the complete suite:
    >>> j6 < j4 < j5 < j3 < j1 < j2
    True

    j0 has the same properties as j1 but they are not considered
    equal as they are different instances
    >>> j0 = ChannelJob(None, 1, seq=0, date_created=1, priority=9, eta=None)
    >>> j0 == j1
    False
    >>> j0 == j0
    True
    """

    def __init__(self, channel, uuid, seq, date_created, priority, eta):
        self.uuid = uuid
        self.channel = channel
        self.seq = seq
        self.date_created = date_created
        self.priority = priority
        self.eta = eta

    def __repr__(self):
        return "<ChannelJob %s>" % self.uuid

    def __eq__(self, other):
        return id(self) == id(other)

    def __hash__(self):
        return id(self)

    def __cmp__(self, other):
        if self.eta and not other.eta:
            return -1
        elif not self.eta and other.eta:
            return 1
        else:
            return (cmp(self.eta, other.eta) or
                    cmp(self.priority, other.priority) or
                    cmp(self.date_created, other.date_created))

class ChannelQueue:
    """A channel queue is a priority queue for jobs, that returns
    jobs with a past ETA first.

    >>> q = ChannelQueue()
    >>> j1 = ChannelJob(None, 1, seq=0, date_created=1, priority=1, eta=10)
    >>> j2 = ChannelJob(None, 2, seq=0, date_created=2, priority=1, eta=None)
    >>> j3 = ChannelJob(None, 3, seq=0, date_created=3, priority=1, eta=None)
    >>> q.add(j1)
    >>> q.add(j2)
    >>> q.add(j3)
    >>> q.pop(now=1)
    <ChannelJob 2>
    >>> q.pop(now=11)
    <ChannelJob 1>
    >>> q.pop(now=12)
    <ChannelJob 3>
    """

    def __init__(self):
        self._queue = PriorityQueue()
        self._eta_queue = PriorityQueue()

    def __len__(self):
        return len(self._eta_queue) + len(self._queue)

    def __contains__(self, o):
        return o in self._eta_queue or o in self._queue

    def add(self, job):
        if job.eta:
            self._eta_queue.add(job)
        else:
            self._queue.add(job)

    def remove(self, job):
        self._eta_queue.remove(job)
        self._queue.remove(job)

    def pop(self, now):
        if len(self._eta_queue) and self._eta_queue[0].eta <= now:
            return self._eta_queue.pop()
        else:
            return self._queue.pop()

class Channel:
    """A channel for jobs, with a maximum number of workers.

    Job channels are joined in a hierarchy down to the root channel.
    When a job channel has free workers, jobs are dequeued, marked
    as running in the channel and are inserted into the queue of the
    parent channel where they wait for free workers and so on.

    Job channels can be visualized as water channels with a given flow
    limit (= workers). Channels are joined together in a downstream channel
    and the flow limit of the downstream channel limit upstream channels.

    ---------------------\
                         \
                         \-----------------------
     Ch. A W:4,Q:12,R:4

    ---------------------\  Ch. root W:5,Q:0,R:4
                         |
    ---------------------/
     Ch. B W:1,Q:0,R:0    -----------------------
    ---------------------/

    The above diagram illustrates two channels joining in the root channel.
    The root channel has 5 workers, and 4 running jobs coming from Channel A.
    Channel A has maximum 4 workers, all in use (passed down to the root
    channel), and 12 jobs enqueued. Channel B has maximum 1 worker,
    none in use. This means that whenever a new job comes in channel B,
    there will be available room for it to run in the root channel.

    Note that from the point of view of a channel, 'running' means enqueued
    in the downstream channel. Only jobs marked running in the root channel
    are actually sent to Odoo for execution.

    Should a downstream channel have less capacity than its upstream channels,
    jobs going downstream will be enqueued in the downstream channel,
    and compete normally according to their properties (priority, etc).

    Using this technique, it is possible to enforce sequence in a channel
    with 1 worker. It is also possible to dedicate a channel with a
    limited number of workers for application-autocreated subchannels
    without risking to overflow the system.
    """

    def __init__(self, name, parent, workers=1, sequential=False):
        self.name = name
        self.parent = parent
        if self.parent:
            self.parent.children.append(self)
        self.children = []
        self.workers = workers
        if sequential and workers != 1:
            raise ValueError("A sequential channel can have only one worker")
        self.sequential = sequential
        self._queue = ChannelQueue()
        self._running = SafeSet()
        self._failed = SafeSet()

    def __str__(self):
        return "%s(W:%d,Q:%d,R:%d,F:%d)" % (self.name,
                                            self.workers,
                                            len(self._queue),
                                            len(self._running),
                                            len(self._failed))

    def remove(self, job):
        self._queue.remove(job)
        self._running.remove(job)
        self._failed.remove(job)
        if self.parent:
            self.parent.remove(job)

    def set_done(self, job):
        self.remove(job)
        _logger.debug("job %s marked done", job.uuid)

    def set_pending(self, job):
        if job not in self._queue:
            self._queue.add(job)
            self._running.remove(job)
            self._failed.remove(job)
            if self.parent:
                self.parent.remove(job)
            _logger.debug("job %s marked pending", job.uuid)

    def set_running(self, job):
        if job not in self._running:
            self._queue.remove(job)
            self._running.add(job)
            self._failed.remove(job)
            if self.parent:
                self.parent.set_running(job)
            _logger.debug("job %s marked running", job.uuid)

    def set_failed(self, job):
        if job not in self._failed:
            self._queue.remove(job)
            self._running.remove(job)
            self._failed.add(job)
            if self.parent:
                self.parent.remove(job)
            _logger.debug("job %s marked failed", job.uuid)

    def get_jobs_to_run(self):
        # enqueue jobs of children channels
        for child in self.children:
            for job in child.get_jobs_to_run():
                self._queue.add(job)
        # sequential channels block when there are failed jobs
        if self.sequential and len(self._failed):
            return
        # yield jobs that are ready to run
        while len(self._running) < self.workers:
            job = self._queue.pop(now=datetime.now())
            if not job:
                return
            self._running.add(job)
            _logger.debug("job %s marked running", job.uuid)
            yield job

class ChannelManager:

    def __init__(self, workers):
        # TODO: config
        self._jobs_by_uuid = WeakValueDictionary()
        self._root_channel = Channel(name='root', parent=None, workers=workers)

    def get_channel_by_name(self, channel_name):
        # TODO: channels by name
        # TODO: autocreate/autodestroy channels
        return self._root_channel

    def notify(self, channel_name, uuid,
               seq, date_created, priority, eta, state):
        channel = self.get_channel_by_name(channel_name)
        job = self._jobs_by_uuid.get(uuid)
        if not job:
            job = ChannelJob(channel, uuid, seq, date_created, priority, eta)
            self._jobs_by_uuid[uuid] = job
        # TODO: handle sequence change
        assert job.seq == seq
        # date_created is invariant
        assert job.date_created == date_created
        # TODO: handle priority change
        assert job.priority == priority
        # TODO: handle eta change
        assert job.eta == eta
        # TODO: handle channel change
        assert job.channel == channel
        # state transitions
        if not state or state == STATE_DONE:
            channel.set_done(job)
        elif state == STATE_PENDING:
            channel.set_pending(job)
        elif state in (STATE_ENQUEUED, STATE_STARTED):
            channel.set_running(job)
        elif state == STATE_FAILED:
            channel.set_failed(job)
        else:
            _logger.error("unexpected state %s for job %s", state, job)
        _logger.debug("channel %s", self._root_channel)

    def get_jobs_to_run(self):
        for job in self._root_channel.get_jobs_to_run():
            yield job

if __name__ == '__main__':
    import doctest
    doctest.testmod()
<|endoftext|>