<commit_before># Copyright (C) 2018, HTCondor Team, Computer Sciences Department,
# University of Wisconsin-Madison, WI.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License.  You may
# obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from builtins import str

import sys
import os
import logging
import time
import math

from toil.batchSystems.abstractGridEngineBatchSystem import AbstractGridEngineBatchSystem

import htcondor
import classad

logger = logging.getLogger(__name__)

class HTCondorBatchSystem(AbstractGridEngineBatchSystem):
    # When using HTCondor, the Schedd handles scheduling

    class Worker(AbstractGridEngineBatchSystem.Worker):

        # Override the createJobs method so that we can use htcondor.Submit objects
        # and so that we can get disk allocation requests and ceil the CPU request.
        def createJobs(self, newJob):
            activity = False

            if newJob is not None:
                self.waitingJobs.append(newJob)

            # Queue jobs as necessary:
            while len(self.waitingJobs) > 0 and len(self.runningJobs) < int(self.boss.config.maxLocalJobs):
                activity = True
                jobID, cpu, memory, disk, jobName, command = self.waitingJobs.pop(0)

                # Prepare the htcondor.Submit object
                submitObj = self.prepareSubmission(cpu, memory, disk, jobID, jobName, command)
                logger.debug("Submitting %r", submitObj)

                # Submit job and get batch system ID (i.e. the ClusterId)
                batchJobID = self.submitJob(submitObj)
                logger.debug("Submitted job %s", str(batchJobID))

                # Store dict for mapping Toil job ID to batch job ID
                # TODO: Note that this currently stores a tuple of (batch system
                # ID, Task), but the second value is None by default and doesn't
                # seem to be used
                self.batchJobIDs[jobID] = (batchJobID, None)

                # Add to queue of queued ("running") jobs
                self.runningJobs.add(jobID)

            return activity

        def prepareSubmission(self, cpu, memory, disk, jobID, jobName, command):

            # Convert resource requests
            cpu = int(math.ceil(cpu)) # integer CPUs
            memory = float(memory)/1024 # memory in KB
            disk = float(disk)/1024 # disk in KB

            # Execute the entire command as /bin/sh -c "command"
            # TODO: Transfer the jobStore directory if using a local file store with a relative path.
            submit_parameters = {
                'executable': '/bin/sh',
                'transfer_executable': 'False',
                'arguments': '''"-c '{0}'"'''.format(command).encode('utf-8'),    # Workaround for HTCondor Python bindings Unicode conversion bug
                'environment': self.getEnvString(),
                'getenv': 'True',
                'request_cpus': '{0}'.format(cpu),
                'request_memory': '{0:.3f}KB'.format(memory),
                'request_disk': '{0:.3f}KB'.format(disk),
                'leave_in_queue': '(JobStatus == 4)',
                '+IsToilJob': 'True',
                '+ToilJobID': '{0}'.format(jobID),
                '+ToilJobName': '"{0}"'.format(jobName),
                '+ToilJobKilled': 'False',
            }

            # Extra parameters for HTCondor
            extra_parameters = os.getenv('TOIL_HTCONDOR_PARAMS')
            if extra_parameters is not None:
                logger.debug("Extra HTCondor parameters added to submit file from TOIL_HTCONDOR_PARAMS env. variable: {}".format(extra_parameters))
                for parameter, value in [parameter_value.split('=', 1) for parameter_value in extra_parameters.split(';')]:
                    parameter = parameter.strip()
                    value = value.strip()
                    if parameter in submit_parameters:
                        raise ValueError("Some extra parameters are incompatible: {}".format(extra_parameters))

                    submit_parameters[parameter] = value

            # Return the Submit object
            return htcondor.Submit(submit_parameters)

        def submitJob(self, submitObj):

            # Queue the job using a Schedd transaction
            schedd = self.connectSchedd()
            with schedd.transaction() as txn:
                batchJobID = submitObj.queue(txn)

            # Return the ClusterId
            return batchJobID

        def getRunningJobIDs(self):

            # Get all Toil jobs that are running
            requirements = '(JobStatus == 2) && (IsToilJob)'
            projection = ['ClusterId', 'ToilJobID', 'EnteredCurrentStatus']
            schedd = self.connectSchedd()
            ads = schedd.xquery(requirements = requirements,
                                    projection = projection)

            # Only consider the Toil jobs that are part of this workflow
            batchJobIDs = [batchJobID for (batchJobID, task) in self.batchJobIDs.values()]
            job_runtimes = {}
            for ad in ads:
                batchJobID = int(ad['ClusterId'])
                jobID = int(ad['ToilJobID'])
                if not (batchJobID in batchJobIDs):
                    continue

                # HTCondor stores the start of the runtime as a Unix timestamp
                runtime = time.time() - ad['EnteredCurrentStatus']
                job_runtimes[jobID] = runtime

            return job_runtimes

        def killJob(self, jobID):
            batchJobID = self.batchJobIDs[jobID][0]
            logger.debug("Killing HTCondor job {0}".format(batchJobID))

            # Set the job to be killed when its exit status is checked
            schedd = self.connectSchedd()
            job_spec = '(ClusterId == {0})'.format(batchJobID)
            schedd.edit(job_spec, 'ToilJobKilled', 'True')

        def getJobExitCode(self, batchJobID):
            logger.debug("Getting exit code for HTCondor job {0}".format(batchJobID))

            status = {
                1: 'Idle',
                2: 'Running',
                3: 'Removed',
                4: 'Completed',
                5: 'Held',
                6: 'Transferring Output',
                7: 'Suspended'
            }

            requirements = '(ClusterId == {0})'.format(batchJobID)
            projection = ['JobStatus', 'ToilJobKilled', 'ExitCode',
                              'HoldReason', 'HoldReasonSubCode']

            schedd = self.connectSchedd()
            ads = schedd.xquery(requirements = requirements,  projection = projection)

            # Make sure a ClassAd was returned
            try:
                try:
                    ad = next(ads)
                except TypeError:
                    ad = ads.next()
            except StopIteration:
                logger.error(
                    "No HTCondor ads returned using constraint: {0}".format(requirements))
                raise

            # Make sure only one ClassAd was returned
            try:
                try:
                    next(ads)
                except TypeError:
                    ads.next()
            except StopIteration:
                pass
            else:
                logger.warning(
                    "Multiple HTCondor ads returned using constraint: {0}".format(requirements))

            if ad['ToilJobKilled']:
                logger.debug("HTCondor job {0} was killed by Toil".format(batchJobID))

                # Remove the job from the Schedd and return 1
                job_spec = 'ClusterId == {0}'.format(batchJobID)
                schedd.act(htcondor.JobAction.Remove, job_spec)
                return 1

            elif status[ad['JobStatus']] == 'Completed':
                logger.debug("HTCondor job {0} completed with exit code {1}".format(
                    batchJobID, ad['ExitCode']))

                # Remove the job from the Schedd and return its exit code
                job_spec = 'ClusterId == {0}'.format(batchJobID)
                schedd.act(htcondor.JobAction.Remove, job_spec)
                return int(ad['ExitCode'])

            elif status[ad['JobStatus']] == 'Held':
                logger.error("HTCondor job {0} was held: '{1} (sub code {2})'".format(
                    batchJobID, ad['HoldReason'], ad['HoldReasonSubCode']))

                # Remove the job from the Schedd and return 1
                job_spec = 'ClusterId == {0}'.format(batchJobID)
                schedd.act(htcondor.JobAction.Remove, job_spec)
                return 1

            else: # Job still running or idle or doing something else
                logger.debug("HTCondor job {0} has not completed (Status: {1})".format(
                    batchJobID, status[ad['JobStatus']]))
                return None

        """
        Implementation-specific helper methods
        """

        def connectSchedd(self):
            '''Connect to HTCondor Schedd and return a Schedd object'''

            condor_host = os.getenv('TOIL_HTCONDOR_COLLECTOR')
            schedd_name = os.getenv('TOIL_HTCONDOR_SCHEDD')

            # If TOIL_HTCONDOR_ variables are set, use them to find the Schedd
            if condor_host and schedd_name:
                logger.debug(
                    "Connecting to HTCondor Schedd {0} using Collector at {1}".format(
                        schedd_name, condor_host))
                try:
                    schedd_ad = htcondor.Collector(condor_host).locate(
                        htcondor.DaemonTypes.Schedd, schedd_name)
                except IOError:
                    logger.error(
                        "Could not connect to HTCondor Collector at {0}".format(condor_host))
                    raise
                except ValueError:
                    logger.error(
                        "Could not find HTCondor Schedd with name {0}".format(schedd_name))
                    raise
                else:
                    schedd = htcondor.Schedd(schedd_ad)

            # Otherwise assume the Schedd is on the local machine
            else:
                logger.debug("Connecting to HTCondor Schedd on local machine")
                schedd = htcondor.Schedd()

            # Ping the Schedd to make sure it's there and responding
            try:
                schedd.xquery(limit = 0)
            except RuntimeError:
                logger.error("Could not connect to HTCondor Schedd")
                raise

            return schedd

        def getEnvString(self):
            '''Build an environment string that a HTCondor Submit object can use.

            For examples of valid strings, see:
            http://research.cs.wisc.edu/htcondor/manual/current/condor_submit.html#man-condor-submit-environment

            '''

            env_items = []
            if self.boss.environment:
                for key, value in self.boss.environment.items():

                    # Each variable should be in the form of <key>='<value>'
                    env_string = key + "="

                    # The entire value should be encapsulated in single quotes
                    # Quote marks (single or double) that are part of the value should be duplicated
                    env_string += "'" + value.replace("'", "''").replace('"', '""') + "'"

                    env_items.append(env_string)

            # The entire string should be encapsulated in double quotes
            # Each variable should be separated by a single space
            return '"' + ' '.join(env_items) + '"'

    # Override the issueBatchJob method so HTCondor can be given the disk request
    def issueBatchJob(self, jobNode):
        # Avoid submitting internal jobs to the batch queue, handle locally
        localID = self.handleLocalJob(jobNode)
        if localID:
            return localID
        else:
            self.checkResourceRequest(jobNode.memory, jobNode.cores, jobNode.disk)
            jobID = self.getNextJobID()
            self.currentJobs.add(jobID)

            # Add the jobNode.disk and jobNode.jobName to the job tuple
            self.newJobsQueue.put((jobID, jobNode.cores, jobNode.memory, jobNode.disk, jobNode.jobName, jobNode.command))
            logger.debug("Issued the job command: %s with job id: %s ", jobNode.command, str(jobID))
        return jobID

    @classmethod
    def obtainSystemConstants(cls):

        # Since it's not always clear what the max cpus and max memory available
        # in an HTCondor slot might be, use some reasonable constants for now.
        # TODO: Use a htcondor.Collector().query() to determine reasonable values.
        max_cpu = 4
        max_mem = 4e9
        return max_cpu, max_mem
<commit_msg>Redirect batch system standard output/error/log output for HTCondor<commit_after># Copyright (C) 2018, HTCondor Team, Computer Sciences Department,
# University of Wisconsin-Madison, WI.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License.  You may
# obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from builtins import str

import sys
import os
import logging
import time
import math

from toil.batchSystems.abstractGridEngineBatchSystem import AbstractGridEngineBatchSystem

import htcondor
import classad

logger = logging.getLogger(__name__)

class HTCondorBatchSystem(AbstractGridEngineBatchSystem):
    # When using HTCondor, the Schedd handles scheduling

    class Worker(AbstractGridEngineBatchSystem.Worker):

        # Override the createJobs method so that we can use htcondor.Submit objects
        # and so that we can get disk allocation requests and ceil the CPU request.
        def createJobs(self, newJob):
            activity = False

            if newJob is not None:
                self.waitingJobs.append(newJob)

            # Queue jobs as necessary:
            while len(self.waitingJobs) > 0 and len(self.runningJobs) < int(self.boss.config.maxLocalJobs):
                activity = True
                jobID, cpu, memory, disk, jobName, command = self.waitingJobs.pop(0)

                # Prepare the htcondor.Submit object
                submitObj = self.prepareSubmission(cpu, memory, disk, jobID, jobName, command)
                logger.debug("Submitting %r", submitObj)

                # Submit job and get batch system ID (i.e. the ClusterId)
                batchJobID = self.submitJob(submitObj)
                logger.debug("Submitted job %s", str(batchJobID))

                # Store dict for mapping Toil job ID to batch job ID
                # TODO: Note that this currently stores a tuple of (batch system
                # ID, Task), but the second value is None by default and doesn't
                # seem to be used
                self.batchJobIDs[jobID] = (batchJobID, None)

                # Add to queue of queued ("running") jobs
                self.runningJobs.add(jobID)

            return activity

        def prepareSubmission(self, cpu, memory, disk, jobID, jobName, command):

            # Convert resource requests
            cpu = int(math.ceil(cpu)) # integer CPUs
            memory = float(memory)/1024 # memory in KB
            disk = float(disk)/1024 # disk in KB

            # NOTE: formatStdOutErrPath() puts files in the Toil workflow directory, which defaults
            # to being in the system temporary directory ($TMPDIR, /tmp) which is unlikely to be on
            # a shared filesystem. So to make this work we need to set should_transfer_files = Yes
            # in the submit file, so that HTCondor will write the standard output/error files on the
            # compute node, then transfer back once the job has completed.
            stdoutfile = self.boss.formatStdOutErrPath(jobID, 'htcondor', '$(cluster)', 'std_output')
            stderrfile = self.boss.formatStdOutErrPath(jobID, 'htcondor', '$(cluster)', 'std_error')

            condorlogfile = self.boss.formatStdOutErrPath(jobID, 'htcondor', '$(cluster)', 'job_events')

            # Execute the entire command as /bin/sh -c "command"
            # TODO: Transfer the jobStore directory if using a local file store with a relative path.
            submit_parameters = {
                'executable': '/bin/sh',
                'transfer_executable': 'False',
                'arguments': '''"-c '{0}'"'''.format(command).encode('utf-8'),    # Workaround for HTCondor Python bindings Unicode conversion bug
                'environment': self.getEnvString(),
                'getenv': 'True',
                'should_transfer_files': 'Yes',   # See note above for stdoutfile, stderrfile
                'output': stdoutfile,
                'error': stderrfile,
                'log': condorlogfile,
                'request_cpus': '{0}'.format(cpu),
                'request_memory': '{0:.3f}KB'.format(memory),
                'request_disk': '{0:.3f}KB'.format(disk),
                'leave_in_queue': '(JobStatus == 4)',
                '+IsToilJob': 'True',
                '+ToilJobID': '{0}'.format(jobID),
                '+ToilJobName': '"{0}"'.format(jobName),
                '+ToilJobKilled': 'False',
            }

            # Extra parameters for HTCondor
            extra_parameters = os.getenv('TOIL_HTCONDOR_PARAMS')
            if extra_parameters is not None:
                logger.debug("Extra HTCondor parameters added to submit file from TOIL_HTCONDOR_PARAMS env. variable: {}".format(extra_parameters))
                for parameter, value in [parameter_value.split('=', 1) for parameter_value in extra_parameters.split(';')]:
                    parameter = parameter.strip()
                    value = value.strip()
                    if parameter in submit_parameters:
                        raise ValueError("Some extra parameters are incompatible: {}".format(extra_parameters))

                    submit_parameters[parameter] = value

            # Return the Submit object
            return htcondor.Submit(submit_parameters)

        def submitJob(self, submitObj):

            # Queue the job using a Schedd transaction
            schedd = self.connectSchedd()
            with schedd.transaction() as txn:
                batchJobID = submitObj.queue(txn)

            # Return the ClusterId
            return batchJobID

        def getRunningJobIDs(self):

            # Get all Toil jobs that are running
            requirements = '(JobStatus == 2) && (IsToilJob)'
            projection = ['ClusterId', 'ToilJobID', 'EnteredCurrentStatus']
            schedd = self.connectSchedd()
            ads = schedd.xquery(requirements = requirements,
                                    projection = projection)

            # Only consider the Toil jobs that are part of this workflow
            batchJobIDs = [batchJobID for (batchJobID, task) in self.batchJobIDs.values()]
            job_runtimes = {}
            for ad in ads:
                batchJobID = int(ad['ClusterId'])
                jobID = int(ad['ToilJobID'])
                if not (batchJobID in batchJobIDs):
                    continue

                # HTCondor stores the start of the runtime as a Unix timestamp
                runtime = time.time() - ad['EnteredCurrentStatus']
                job_runtimes[jobID] = runtime

            return job_runtimes

        def killJob(self, jobID):
            batchJobID = self.batchJobIDs[jobID][0]
            logger.debug("Killing HTCondor job {0}".format(batchJobID))

            # Set the job to be killed when its exit status is checked
            schedd = self.connectSchedd()
            job_spec = '(ClusterId == {0})'.format(batchJobID)
            schedd.edit(job_spec, 'ToilJobKilled', 'True')

        def getJobExitCode(self, batchJobID):
            logger.debug("Getting exit code for HTCondor job {0}".format(batchJobID))

            status = {
                1: 'Idle',
                2: 'Running',
                3: 'Removed',
                4: 'Completed',
                5: 'Held',
                6: 'Transferring Output',
                7: 'Suspended'
            }

            requirements = '(ClusterId == {0})'.format(batchJobID)
            projection = ['JobStatus', 'ToilJobKilled', 'ExitCode',
                              'HoldReason', 'HoldReasonSubCode']

            schedd = self.connectSchedd()
            ads = schedd.xquery(requirements = requirements,  projection = projection)

            # Make sure a ClassAd was returned
            try:
                try:
                    ad = next(ads)
                except TypeError:
                    ad = ads.next()
            except StopIteration:
                logger.error(
                    "No HTCondor ads returned using constraint: {0}".format(requirements))
                raise

            # Make sure only one ClassAd was returned
            try:
                try:
                    next(ads)
                except TypeError:
                    ads.next()
            except StopIteration:
                pass
            else:
                logger.warning(
                    "Multiple HTCondor ads returned using constraint: {0}".format(requirements))

            if ad['ToilJobKilled']:
                logger.debug("HTCondor job {0} was killed by Toil".format(batchJobID))

                # Remove the job from the Schedd and return 1
                job_spec = 'ClusterId == {0}'.format(batchJobID)
                schedd.act(htcondor.JobAction.Remove, job_spec)
                return 1

            elif status[ad['JobStatus']] == 'Completed':
                logger.debug("HTCondor job {0} completed with exit code {1}".format(
                    batchJobID, ad['ExitCode']))

                # Remove the job from the Schedd and return its exit code
                job_spec = 'ClusterId == {0}'.format(batchJobID)
                schedd.act(htcondor.JobAction.Remove, job_spec)
                return int(ad['ExitCode'])

            elif status[ad['JobStatus']] == 'Held':
                logger.error("HTCondor job {0} was held: '{1} (sub code {2})'".format(
                    batchJobID, ad['HoldReason'], ad['HoldReasonSubCode']))

                # Remove the job from the Schedd and return 1
                job_spec = 'ClusterId == {0}'.format(batchJobID)
                schedd.act(htcondor.JobAction.Remove, job_spec)
                return 1

            else: # Job still running or idle or doing something else
                logger.debug("HTCondor job {0} has not completed (Status: {1})".format(
                    batchJobID, status[ad['JobStatus']]))
                return None

        """
        Implementation-specific helper methods
        """

        def connectSchedd(self):
            '''Connect to HTCondor Schedd and return a Schedd object'''

            condor_host = os.getenv('TOIL_HTCONDOR_COLLECTOR')
            schedd_name = os.getenv('TOIL_HTCONDOR_SCHEDD')

            # If TOIL_HTCONDOR_ variables are set, use them to find the Schedd
            if condor_host and schedd_name:
                logger.debug(
                    "Connecting to HTCondor Schedd {0} using Collector at {1}".format(
                        schedd_name, condor_host))
                try:
                    schedd_ad = htcondor.Collector(condor_host).locate(
                        htcondor.DaemonTypes.Schedd, schedd_name)
                except IOError:
                    logger.error(
                        "Could not connect to HTCondor Collector at {0}".format(condor_host))
                    raise
                except ValueError:
                    logger.error(
                        "Could not find HTCondor Schedd with name {0}".format(schedd_name))
                    raise
                else:
                    schedd = htcondor.Schedd(schedd_ad)

            # Otherwise assume the Schedd is on the local machine
            else:
                logger.debug("Connecting to HTCondor Schedd on local machine")
                schedd = htcondor.Schedd()

            # Ping the Schedd to make sure it's there and responding
            try:
                schedd.xquery(limit = 0)
            except RuntimeError:
                logger.error("Could not connect to HTCondor Schedd")
                raise

            return schedd

        def getEnvString(self):
            '''Build an environment string that a HTCondor Submit object can use.

            For examples of valid strings, see:
            http://research.cs.wisc.edu/htcondor/manual/current/condor_submit.html#man-condor-submit-environment

            '''

            env_items = []
            if self.boss.environment:
                for key, value in self.boss.environment.items():

                    # Each variable should be in the form of <key>='<value>'
                    env_string = key + "="

                    # The entire value should be encapsulated in single quotes
                    # Quote marks (single or double) that are part of the value should be duplicated
                    env_string += "'" + value.replace("'", "''").replace('"', '""') + "'"

                    env_items.append(env_string)

            # The entire string should be encapsulated in double quotes
            # Each variable should be separated by a single space
            return '"' + ' '.join(env_items) + '"'

    # Override the issueBatchJob method so HTCondor can be given the disk request
    def issueBatchJob(self, jobNode):
        # Avoid submitting internal jobs to the batch queue, handle locally
        localID = self.handleLocalJob(jobNode)
        if localID:
            return localID
        else:
            self.checkResourceRequest(jobNode.memory, jobNode.cores, jobNode.disk)
            jobID = self.getNextJobID()
            self.currentJobs.add(jobID)

            # Add the jobNode.disk and jobNode.jobName to the job tuple
            self.newJobsQueue.put((jobID, jobNode.cores, jobNode.memory, jobNode.disk, jobNode.jobName, jobNode.command))
            logger.debug("Issued the job command: %s with job id: %s ", jobNode.command, str(jobID))
        return jobID

    @classmethod
    def obtainSystemConstants(cls):

        # Since it's not always clear what the max cpus and max memory available
        # in an HTCondor slot might be, use some reasonable constants for now.
        # TODO: Use a htcondor.Collector().query() to determine reasonable values.
        max_cpu = 4
        max_mem = 4e9
        return max_cpu, max_mem
<|endoftext|>


<commit_before># Copyright 2016 Intel Corporation
# Copyright 2013 Hewlett-Packard Development Company, L.P.
# Copyright 2013 International Business Machines Corporation
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from oslo_config import cfg

from ironic.common.i18n import _

opts = [
    cfg.IntOpt('workers_pool_size',
               default=100, min=3,
               help=_('The size of the workers greenthread pool. '
                      'Note that 2 threads will be reserved by the conductor '
                      'itself for handling heart beats and periodic tasks.')),
    cfg.IntOpt('heartbeat_interval',
               default=10,
               help=_('Seconds between conductor heart beats.')),
    cfg.URIOpt('api_url',
               schemes=('http', 'https'),
               deprecated_for_removal=True,
               deprecated_reason=_("Use [service_catalog]endpoint_override "
                                   "option instead if required to use "
                                   "a specific ironic api address, "
                                   "for example in noauth mode."),
               help=_('URL of Ironic API service. If not set ironic can '
                      'get the current value from the keystone service '
                      'catalog. If set, the value must start with either '
                      'http:// or https://.')),
    cfg.IntOpt('heartbeat_timeout',
               default=60,
               help=_('Maximum time (in seconds) since the last check-in '
                      'of a conductor. A conductor is considered inactive '
                      'when this time has been exceeded.')),
    cfg.IntOpt('sync_power_state_interval',
               default=60,
               help=_('Interval between syncing the node power state to the '
                      'database, in seconds. Set to 0 to disable syncing.')),
    cfg.IntOpt('check_provision_state_interval',
               default=60,
               min=0,
               help=_('Interval between checks of provision timeouts, '
                      'in seconds. Set to 0 to disable checks.')),
    cfg.IntOpt('check_rescue_state_interval',
               default=60,
               min=1,
               help=_('Interval (seconds) between checks of rescue '
                      'timeouts.')),
    cfg.IntOpt('deploy_callback_timeout',
               default=1800,
               help=_('Timeout (seconds) to wait for a callback from '
                      'a deploy ramdisk. Set to 0 to disable timeout.')),
    cfg.BoolOpt('force_power_state_during_sync',
                default=True,
                help=_('During sync_power_state, should the hardware power '
                       'state be set to the state recorded in the database '
                       '(True) or should the database be updated based on '
                       'the hardware state (False).')),
    cfg.IntOpt('power_state_sync_max_retries',
               default=3,
               help=_('During sync_power_state failures, limit the '
                      'number of times Ironic should try syncing the '
                      'hardware node power state with the node power state '
                      'in DB')),
    cfg.IntOpt('periodic_max_workers',
               default=8,
               help=_('Maximum number of worker threads that can be started '
                      'simultaneously by a periodic task. Should be less '
                      'than RPC thread pool size.')),
    cfg.IntOpt('node_locked_retry_attempts',
               default=3,
               help=_('Number of attempts to grab a node lock.')),
    cfg.IntOpt('node_locked_retry_interval',
               default=1,
               help=_('Seconds to sleep between node lock attempts.')),
    cfg.BoolOpt('send_sensor_data',
                default=False,
                help=_('Enable sending sensor data message via the '
                       'notification bus')),
    cfg.IntOpt('send_sensor_data_interval',
               default=600,
               min=1,
               help=_('Seconds between conductor sending sensor data message '
                      'to ceilometer via the notification bus.')),
    cfg.IntOpt('send_sensor_data_workers',
               default=4, min=1,
               help=_('The maximum number of workers that can be started '
                      'simultaneously for send data from sensors periodic '
                      'task.')),
    cfg.IntOpt('send_sensor_data_wait_timeout',
               default=300,
               help=_('The time in seconds to wait for send sensors data '
                      'periodic task to be finished before allowing periodic '
                      'call to happen again. Should be less than '
                      'send_sensor_data_interval value.')),
    cfg.ListOpt('send_sensor_data_types',
                default=['ALL'],
                help=_('List of comma separated meter types which need to be'
                       ' sent to Ceilometer. The default value, "ALL", is a '
                       'special value meaning send all the sensor data.')),
    cfg.IntOpt('sync_local_state_interval',
               default=180,
               help=_('When conductors join or leave the cluster, existing '
                      'conductors may need to update any persistent '
                      'local state as nodes are moved around the cluster. '
                      'This option controls how often, in seconds, each '
                      'conductor will check for nodes that it should '
                      '"take over". Set it to 0 (or a negative value) to '
                      'disable the check entirely.')),
    cfg.StrOpt('configdrive_swift_container',
               default='ironic_configdrive_container',
               help=_('Name of the Swift container to store config drive '
                      'data. Used when configdrive_use_object_store is '
                      'True.')),
    cfg.IntOpt('inspect_wait_timeout',
               default=1800,
               help=_('Timeout (seconds) for waiting for node inspection. '
                      '0 - unlimited.')),
    cfg.BoolOpt('automated_clean',
                default=True,
                help=_('Enables or disables automated cleaning. Automated '
                       'cleaning is a configurable set of steps, '
                       'such as erasing disk drives, that are performed on '
                       'the node to ensure it is in a baseline state and '
                       'ready to be deployed to. This is '
                       'done after instance deletion as well as during '
                       'the transition from a "manageable" to "available" '
                       'state. When enabled, the particular steps '
                       'performed to clean a node depend on which driver '
                       'that node is managed by; see the individual '
                       'driver\'s documentation for details. '
                       'NOTE: The introduction of the cleaning operation '
                       'causes instance deletion to take significantly '
                       'longer. In an environment where all tenants are '
                       'trusted (eg, because there is only one tenant), '
                       'this option could be safely disabled.')),
    cfg.IntOpt('clean_callback_timeout',
               default=1800,
               help=_('Timeout (seconds) to wait for a callback from the '
                      'ramdisk doing the cleaning. If the timeout is reached '
                      'the node will be put in the "clean failed" provision '
                      'state. Set to 0 to disable timeout.')),
    cfg.IntOpt('rescue_callback_timeout',
               default=1800,
               min=0,
               help=_('Timeout (seconds) to wait for a callback from the '
                      'rescue ramdisk. If the timeout is reached the node '
                      'will be put in the "rescue failed" provision state. '
                      'Set to 0 to disable timeout.')),
    cfg.IntOpt('soft_power_off_timeout',
               default=600,
               min=1,
               help=_('Timeout (in seconds) of soft reboot and soft power '
                      'off operation. This value always has to be positive.')),
    cfg.IntOpt('power_state_change_timeout',
               min=2, default=30,
               help=_('Number of seconds to wait for power operations to '
                      'complete, i.e., so that a baremetal node is in the '
                      'desired power state. If timed out, the power operation '
                      'is considered a failure.')),
    cfg.IntOpt('power_failure_recovery_interval',
               min=0, default=300,
               help=_('Interval (in seconds) between checking the power '
                      'state for nodes previously put into maintenance mode '
                      'due to power synchronization failure. A node is '
                      'automatically moved out of maintenance mode once its '
                      'power state is retrieved successfully. Set to 0 to '
                      'disable this check.')),
    cfg.StrOpt('conductor_group',
               default='',
               help=_('Name of the conductor group to join. Can be up to '
                      '255 characters and is case insensitive. This '
                      'conductor will only manage nodes with a matching '
                      '"conductor_group" field set on the node.')),
]

def register_opts(conf):
    conf.register_opts(opts, group='conductor')
<commit_msg>Limit the timeout value of heartbeat_timeout<commit_after># Copyright 2016 Intel Corporation
# Copyright 2013 Hewlett-Packard Development Company, L.P.
# Copyright 2013 International Business Machines Corporation
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from oslo_config import cfg

from ironic.common.i18n import _

opts = [
    cfg.IntOpt('workers_pool_size',
               default=100, min=3,
               help=_('The size of the workers greenthread pool. '
                      'Note that 2 threads will be reserved by the conductor '
                      'itself for handling heart beats and periodic tasks.')),
    cfg.IntOpt('heartbeat_interval',
               default=10,
               help=_('Seconds between conductor heart beats.')),
    cfg.URIOpt('api_url',
               schemes=('http', 'https'),
               deprecated_for_removal=True,
               deprecated_reason=_("Use [service_catalog]endpoint_override "
                                   "option instead if required to use "
                                   "a specific ironic api address, "
                                   "for example in noauth mode."),
               help=_('URL of Ironic API service. If not set ironic can '
                      'get the current value from the keystone service '
                      'catalog. If set, the value must start with either '
                      'http:// or https://.')),
    cfg.IntOpt('heartbeat_timeout',
               default=60,
               # We're using timedelta which can overflow if somebody sets this
               # too high, so limit to a sane value of 10 years.
               max=315576000,
               help=_('Maximum time (in seconds) since the last check-in '
                      'of a conductor. A conductor is considered inactive '
                      'when this time has been exceeded.')),
    cfg.IntOpt('sync_power_state_interval',
               default=60,
               help=_('Interval between syncing the node power state to the '
                      'database, in seconds. Set to 0 to disable syncing.')),
    cfg.IntOpt('check_provision_state_interval',
               default=60,
               min=0,
               help=_('Interval between checks of provision timeouts, '
                      'in seconds. Set to 0 to disable checks.')),
    cfg.IntOpt('check_rescue_state_interval',
               default=60,
               min=1,
               help=_('Interval (seconds) between checks of rescue '
                      'timeouts.')),
    cfg.IntOpt('deploy_callback_timeout',
               default=1800,
               help=_('Timeout (seconds) to wait for a callback from '
                      'a deploy ramdisk. Set to 0 to disable timeout.')),
    cfg.BoolOpt('force_power_state_during_sync',
                default=True,
                help=_('During sync_power_state, should the hardware power '
                       'state be set to the state recorded in the database '
                       '(True) or should the database be updated based on '
                       'the hardware state (False).')),
    cfg.IntOpt('power_state_sync_max_retries',
               default=3,
               help=_('During sync_power_state failures, limit the '
                      'number of times Ironic should try syncing the '
                      'hardware node power state with the node power state '
                      'in DB')),
    cfg.IntOpt('periodic_max_workers',
               default=8,
               help=_('Maximum number of worker threads that can be started '
                      'simultaneously by a periodic task. Should be less '
                      'than RPC thread pool size.')),
    cfg.IntOpt('node_locked_retry_attempts',
               default=3,
               help=_('Number of attempts to grab a node lock.')),
    cfg.IntOpt('node_locked_retry_interval',
               default=1,
               help=_('Seconds to sleep between node lock attempts.')),
    cfg.BoolOpt('send_sensor_data',
                default=False,
                help=_('Enable sending sensor data message via the '
                       'notification bus')),
    cfg.IntOpt('send_sensor_data_interval',
               default=600,
               min=1,
               help=_('Seconds between conductor sending sensor data message '
                      'to ceilometer via the notification bus.')),
    cfg.IntOpt('send_sensor_data_workers',
               default=4, min=1,
               help=_('The maximum number of workers that can be started '
                      'simultaneously for send data from sensors periodic '
                      'task.')),
    cfg.IntOpt('send_sensor_data_wait_timeout',
               default=300,
               help=_('The time in seconds to wait for send sensors data '
                      'periodic task to be finished before allowing periodic '
                      'call to happen again. Should be less than '
                      'send_sensor_data_interval value.')),
    cfg.ListOpt('send_sensor_data_types',
                default=['ALL'],
                help=_('List of comma separated meter types which need to be'
                       ' sent to Ceilometer. The default value, "ALL", is a '
                       'special value meaning send all the sensor data.')),
    cfg.IntOpt('sync_local_state_interval',
               default=180,
               help=_('When conductors join or leave the cluster, existing '
                      'conductors may need to update any persistent '
                      'local state as nodes are moved around the cluster. '
                      'This option controls how often, in seconds, each '
                      'conductor will check for nodes that it should '
                      '"take over". Set it to 0 (or a negative value) to '
                      'disable the check entirely.')),
    cfg.StrOpt('configdrive_swift_container',
               default='ironic_configdrive_container',
               help=_('Name of the Swift container to store config drive '
                      'data. Used when configdrive_use_object_store is '
                      'True.')),
    cfg.IntOpt('inspect_wait_timeout',
               default=1800,
               help=_('Timeout (seconds) for waiting for node inspection. '
                      '0 - unlimited.')),
    cfg.BoolOpt('automated_clean',
                default=True,
                help=_('Enables or disables automated cleaning. Automated '
                       'cleaning is a configurable set of steps, '
                       'such as erasing disk drives, that are performed on '
                       'the node to ensure it is in a baseline state and '
                       'ready to be deployed to. This is '
                       'done after instance deletion as well as during '
                       'the transition from a "manageable" to "available" '
                       'state. When enabled, the particular steps '
                       'performed to clean a node depend on which driver '
                       'that node is managed by; see the individual '
                       'driver\'s documentation for details. '
                       'NOTE: The introduction of the cleaning operation '
                       'causes instance deletion to take significantly '
                       'longer. In an environment where all tenants are '
                       'trusted (eg, because there is only one tenant), '
                       'this option could be safely disabled.')),
    cfg.IntOpt('clean_callback_timeout',
               default=1800,
               help=_('Timeout (seconds) to wait for a callback from the '
                      'ramdisk doing the cleaning. If the timeout is reached '
                      'the node will be put in the "clean failed" provision '
                      'state. Set to 0 to disable timeout.')),
    cfg.IntOpt('rescue_callback_timeout',
               default=1800,
               min=0,
               help=_('Timeout (seconds) to wait for a callback from the '
                      'rescue ramdisk. If the timeout is reached the node '
                      'will be put in the "rescue failed" provision state. '
                      'Set to 0 to disable timeout.')),
    cfg.IntOpt('soft_power_off_timeout',
               default=600,
               min=1,
               help=_('Timeout (in seconds) of soft reboot and soft power '
                      'off operation. This value always has to be positive.')),
    cfg.IntOpt('power_state_change_timeout',
               min=2, default=30,
               help=_('Number of seconds to wait for power operations to '
                      'complete, i.e., so that a baremetal node is in the '
                      'desired power state. If timed out, the power operation '
                      'is considered a failure.')),
    cfg.IntOpt('power_failure_recovery_interval',
               min=0, default=300,
               help=_('Interval (in seconds) between checking the power '
                      'state for nodes previously put into maintenance mode '
                      'due to power synchronization failure. A node is '
                      'automatically moved out of maintenance mode once its '
                      'power state is retrieved successfully. Set to 0 to '
                      'disable this check.')),
    cfg.StrOpt('conductor_group',
               default='',
               help=_('Name of the conductor group to join. Can be up to '
                      '255 characters and is case insensitive. This '
                      'conductor will only manage nodes with a matching '
                      '"conductor_group" field set on the node.')),
]

def register_opts(conf):
    conf.register_opts(opts, group='conductor')
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

""" Module for detecting where the IronPython executables exist.

"""

import itertools
import os
import glob
import subprocess
import warnings
import sys

# Original modules
from . import exceptions
from . import constants

def detect_ipy(regkeys = constants.REGKEYS, executable = constants.EXECUTABLE):
    """This function returns the list of the paths to the IronPython directories.

    This function searches in the Windows registry and PATH for
    IronPython.

    :param list regkeys: (optional) The IronPython registry keys that
                         should be looked for.
    :param str executable: (optional) The name of the IronPython
                           executable.
    :rtype: list
    :raises ironpycompiler.exceptions.IronPythonDetectionError: if IronPython cannot be found

    .. versionchanged:: 0.9.0
       This function now calls :func:`search_ipy`.

    .. warning::
       This function is deprecated, and will be removed in the next
       major version. Please use :func:`search_ipy`.

    """

    warnings.warn("Use search_ipy instead.", DeprecationWarning)

    return sorted(search_ipy(regkeys, executable).values(), reverse = True)

def search_ipy_reg(regkeys = None):
    """Search for IronPython regisitry keys.

    This function searches for IronPython keys in the Windows registry,
    and returns a dictionary showing the versions of IronPython and their
    locations (the paths to the IronPython directories).

    :param list regkeys: (optional) The IronPython registry keys that
                         should be looked for.
    :rtype: dict
    :raises ironpycompiler.exceptions.IronPythonDetectionError: if IronPython keys cannot be found

    .. versionadded:: 0.10.1
       Solved the problem that the default value for the argument ``regkeys`` was mutable.

    """

    if regkeys is None:
        regkeys = constants.REGKEYS

    try:
        import _winreg
    except ImportError as e:
        raise exceptions.IronPythonDetectionError(
        msg = "Unable to import a module for accessing the Windows registry.")

    foundipys = dict()
    ipybasekey = None

    # IronPythonキーを読み込む
    for key in regkeys:
        try:
            ipybasekey = _winreg.OpenKey(
            _winreg.HKEY_LOCAL_MACHINE, key)
        except WindowsError as e:
            continue
        else:
            break

    if ipybasekey is None:
        raise exceptions.IronPythonDetectionError(
        msg = "Could not find any IronPython registry key.")
    else:
        itr = itertools.count()
        for idx in itr:
            try:
                foundipys[_winreg.EnumKey(ipybasekey, idx)] = None
            except WindowsError as e: # 対応するサブキーがなくなったら
                break
        if foundipys == dict():
            raise exceptions.IronPythonDetectionError(
            msg = "Could not find any version of IronPython.")
        for ver in foundipys:
            ipypathkey = _winreg.OpenKey(ipybasekey,
            ver + "\\InstallPath")
            foundipys[ver] = os.path.dirname(
            _winreg.QueryValue(ipypathkey, None))
            ipypathkey.Close()
        ipybasekey.Close()

    return foundipys

def search_ipy_env(executable = constants.EXECUTABLE):
    """Search for IronPython directories included in the PATH variable.

    This function searches for IronPython executables in your system,
    reading the PATH environment variable, and gets their version
    numbers, executing the executables.

    This function returns a dictionary showing the versions of
    IronPython and their locations (the paths to the IronPython
    directories).

    :param str executable: (optional) The name of the IronPython
                           executable.
    :rtype: dict
    :raises ironpycompiler.exceptions.IronPythonDetectionError: if IronPython cannot be found

    .. versionadded:: 0.9.0

    """

    ipydirpaths = []
    foundipys = {}

    for path in os.environ["PATH"].split(os.pathsep):
        for match_path in glob.glob(os.path.join(path, executable)):
            if os.access(match_path, os.X_OK):
                ipydirpaths.append(os.path.dirname(match_path))

    if len(ipydirpaths) == 0:
        raise exceptions.IronPythonDetectionError(
        msg = "Could not find any executable file named %s." % executable)

    for directory in ipydirpaths:
        ipy_exe = os.path.abspath(os.path.join(directory, executable))
        sp = subprocess.Popen(
        args = [executable, "-V"],
        executable = ipy_exe, stdin = subprocess.PIPE,
        stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
        (sp_stdout, sp_stderr) = sp.communicate()
        ipy_ver = sp_stdout[11:14] #Todo: 正規表現でx.xのみを抽出
        foundipys[ipy_ver] = directory

    return foundipys

def search_ipy(regkeys = None, executable = constants.EXECUTABLE):
    """Search for IronPython directories.

    This function searches for IronPython directories using both
    :func:`search_ipy_env` and :func:`search_ipy_reg`, and returns a
    dictionary showing the versions of IronPython and their locations
    (the paths to the IronPython directories).

    :param str executable: (optional) The name of the IronPython
                           executable.
    :param list regkeys: (optional) The IronPython registry keys that
                         should be looked for.
    :rtype: dict

    .. versionadded:: 0.9.0

    .. versionadded:: 0.10.1
       Solved the problem that the default value for the argument ``regkeys`` was mutable.

    """

    if regkeys is None:
        regkeys = constants.REGKEYS

    try:
        foundipys = search_ipy_reg(regkeys)
    except exceptions.IronPythonDetectionError as e:
        foundipys = dict()

    try:
        envipys = search_ipy_env(executable)
    except exceptions.IronPythonDetectionError as e:
        envipys = dict()

    for k, v in envipys.items():
        if k not in foundipys:
            foundipys[k] = v

    if len(foundipys) == 0:
        raise exceptions.IronPythonDetectionError(
        msg = "Could not find any IronPython directory.")
    else:
        return foundipys

def auto_detect():
    """Decide the optimum version of IronPython in your system.

    This function decides the most suitable version of IronPython
    in your system for the version of CPython on which IronPyCompiler
    is being run, and returns a tuple showing its version number and
    its location (the path to the IronPython directory).

    Example: On CPython 2.7, first this function searches for
    IronPython 2.7. If this fails, then the newest IronPython 2.x in
    your system will be selected.

    :rtype: tuple
    :raises ironpycompiler.exceptions.IronPythonDetectionError: if this function could
                                                                not decide the optimum
                                                                version

    .. versionadded:: 0.9.0

    """

    cpy_ver = sys.version_info
    cpy_ver_str = "%d.%d" % (cpy_ver[0], cpy_ver[1])
    foundipys = search_ipy()

    if cpy_ver_str in foundipys:
        return (cpy_ver_str, foundipys[cpy_ver_str])
    else:
        #メジャーバージョンは合致するがマイナーバージョンは合致しないバージョンたち
        majoripys = sorted(
        [ver for ver in foundipys.keys() if ver.startswith("%d." % cpy_ver[0])],
        reverse = True)
        if len(majoripys) == 0:
            raise exceptions.IronPythonDetectionError(
            msg = "Could not decide the optimum version of IronPython.")
        else:
            return (majoripys[0], foundipys[majoripys[0]])

<commit_msg>Corrected docstring<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

""" Module for detecting where the IronPython executables exist.

"""

import itertools
import os
import glob
import subprocess
import warnings
import sys

# Original modules
from . import exceptions
from . import constants

def detect_ipy(regkeys = constants.REGKEYS, executable = constants.EXECUTABLE):
    """This function returns the list of the paths to the IronPython directories.

    This function searches in the Windows registry and PATH for
    IronPython.

    :param list regkeys: (optional) The IronPython registry keys that
                         should be looked for.
    :param str executable: (optional) The name of the IronPython
                           executable.
    :rtype: list
    :raises ironpycompiler.exceptions.IronPythonDetectionError: if IronPython cannot be found

    .. versionchanged:: 0.9.0
       This function now calls :func:`search_ipy`.

    .. warning::
       This function is deprecated, and will be removed in the next
       major version. Please use :func:`search_ipy`.

    """

    warnings.warn("Use search_ipy instead.", DeprecationWarning)

    return sorted(search_ipy(regkeys, executable).values(), reverse = True)

def search_ipy_reg(regkeys = None):
    """Search for IronPython regisitry keys.

    This function searches for IronPython keys in the Windows registry,
    and returns a dictionary showing the versions of IronPython and their
    locations (the paths to the IronPython directories).

    :param list regkeys: (optional) The IronPython registry keys that
                         should be looked for.
    :rtype: dict
    :raises ironpycompiler.exceptions.IronPythonDetectionError: if IronPython keys cannot be found

    .. versionadded:: 0.9.0

    .. versionchanged:: 0.10.1
       Solved the problem that the default value for the argument ``regkeys`` was mutable.

    """

    if regkeys is None:
        regkeys = constants.REGKEYS

    try:
        import _winreg
    except ImportError as e:
        raise exceptions.IronPythonDetectionError(
        msg = "Unable to import a module for accessing the Windows registry.")

    foundipys = dict()
    ipybasekey = None

    # IronPythonキーを読み込む
    for key in regkeys:
        try:
            ipybasekey = _winreg.OpenKey(
            _winreg.HKEY_LOCAL_MACHINE, key)
        except WindowsError as e:
            continue
        else:
            break

    if ipybasekey is None:
        raise exceptions.IronPythonDetectionError(
        msg = "Could not find any IronPython registry key.")
    else:
        itr = itertools.count()
        for idx in itr:
            try:
                foundipys[_winreg.EnumKey(ipybasekey, idx)] = None
            except WindowsError as e: # 対応するサブキーがなくなったら
                break
        if foundipys == dict():
            raise exceptions.IronPythonDetectionError(
            msg = "Could not find any version of IronPython.")
        for ver in foundipys:
            ipypathkey = _winreg.OpenKey(ipybasekey,
            ver + "\\InstallPath")
            foundipys[ver] = os.path.dirname(
            _winreg.QueryValue(ipypathkey, None))
            ipypathkey.Close()
        ipybasekey.Close()

    return foundipys

def search_ipy_env(executable = constants.EXECUTABLE):
    """Search for IronPython directories included in the PATH variable.

    This function searches for IronPython executables in your system,
    reading the PATH environment variable, and gets their version
    numbers, executing the executables.

    This function returns a dictionary showing the versions of
    IronPython and their locations (the paths to the IronPython
    directories).

    :param str executable: (optional) The name of the IronPython
                           executable.
    :rtype: dict
    :raises ironpycompiler.exceptions.IronPythonDetectionError: if IronPython cannot be found

    .. versionadded:: 0.9.0

    """

    ipydirpaths = []
    foundipys = {}

    for path in os.environ["PATH"].split(os.pathsep):
        for match_path in glob.glob(os.path.join(path, executable)):
            if os.access(match_path, os.X_OK):
                ipydirpaths.append(os.path.dirname(match_path))

    if len(ipydirpaths) == 0:
        raise exceptions.IronPythonDetectionError(
        msg = "Could not find any executable file named %s." % executable)

    for directory in ipydirpaths:
        ipy_exe = os.path.abspath(os.path.join(directory, executable))
        sp = subprocess.Popen(
        args = [executable, "-V"],
        executable = ipy_exe, stdin = subprocess.PIPE,
        stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
        (sp_stdout, sp_stderr) = sp.communicate()
        ipy_ver = sp_stdout[11:14] #Todo: 正規表現でx.xのみを抽出
        foundipys[ipy_ver] = directory

    return foundipys

def search_ipy(regkeys = None, executable = constants.EXECUTABLE):
    """Search for IronPython directories.

    This function searches for IronPython directories using both
    :func:`search_ipy_env` and :func:`search_ipy_reg`, and returns a
    dictionary showing the versions of IronPython and their locations
    (the paths to the IronPython directories).

    :param str executable: (optional) The name of the IronPython
                           executable.
    :param list regkeys: (optional) The IronPython registry keys that
                         should be looked for.
    :rtype: dict

    .. versionadded:: 0.9.0

    .. versionchanged:: 0.10.1
       Solved the problem that the default value for the argument ``regkeys`` was mutable.

    """

    if regkeys is None:
        regkeys = constants.REGKEYS

    try:
        foundipys = search_ipy_reg(regkeys)
    except exceptions.IronPythonDetectionError as e:
        foundipys = dict()

    try:
        envipys = search_ipy_env(executable)
    except exceptions.IronPythonDetectionError as e:
        envipys = dict()

    for k, v in envipys.items():
        if k not in foundipys:
            foundipys[k] = v

    if len(foundipys) == 0:
        raise exceptions.IronPythonDetectionError(
        msg = "Could not find any IronPython directory.")
    else:
        return foundipys

def auto_detect():
    """Decide the optimum version of IronPython in your system.

    This function decides the most suitable version of IronPython
    in your system for the version of CPython on which IronPyCompiler
    is being run, and returns a tuple showing its version number and
    its location (the path to the IronPython directory).

    Example: On CPython 2.7, first this function searches for
    IronPython 2.7. If this fails, then the newest IronPython 2.x in
    your system will be selected.

    :rtype: tuple
    :raises ironpycompiler.exceptions.IronPythonDetectionError: if this function could
                                                                not decide the optimum
                                                                version

    .. versionadded:: 0.9.0

    """

    cpy_ver = sys.version_info
    cpy_ver_str = "%d.%d" % (cpy_ver[0], cpy_ver[1])
    foundipys = search_ipy()

    if cpy_ver_str in foundipys:
        return (cpy_ver_str, foundipys[cpy_ver_str])
    else:
        #メジャーバージョンは合致するがマイナーバージョンは合致しないバージョンたち
        majoripys = sorted(
        [ver for ver in foundipys.keys() if ver.startswith("%d." % cpy_ver[0])],
        reverse = True)
        if len(majoripys) == 0:
            raise exceptions.IronPythonDetectionError(
            msg = "Could not decide the optimum version of IronPython.")
        else:
            return (majoripys[0], foundipys[majoripys[0]])

<|endoftext|>


<commit_before>import numpy as np

import tomviz.utils

from vtk import vtkImageData

_installed = False
try:
    from databroker import catalog
    from databroker.queries import TimeRange
    _installed = True
except ImportError:
    pass

def installed():
    return _installed

def catalogs():
    cats = []
    for name, cat in catalog.items():
        cats.append({
            "name": name,
            "description": cat.describe()['description']
        })

    cats = sorted(cats, key=lambda c: c['name'])

    return cats

def runs(catalog_name, since, until):
    runs = []

    cat = catalog[catalog_name]

    if since != "" and until != "":
        cat = cat.search(TimeRange(since=since, until=until))

    for uid, run in cat.items():
        runs.append({
            "uid": uid,
            "planName": run.metadata['start']['plan_name'],
            "scanId": run.metadata['start']['scan_id'],
            "startTime": run.metadata['start']['time'],
            "stopTime": run.metadata['stop']['time'],
        })

    runs = sorted(runs, key=lambda r: r['uid'])

    return runs

def tables(catalog_name, run_uid):
    tables = []
    for name, _ in catalog[catalog_name][run_uid].items():
        tables.append({
            "name": name,
        })

    tables = sorted(tables, key=lambda t: t['name'])

    return tables

def variables(catalog_name, run_uid, table):
    variables = []

    # Would be nice to use context manager here, but it doesn't seem to close
    # the dataset.
    dataset = catalog[catalog_name][run_uid][table].read()
    try:
        for name, variable in dataset.data_vars.items():
            variables.append({
                "name": name,
                "size": variable.data.nbytes
            })
    finally:
        dataset.close()

    variables = sorted(variables, key=lambda v: v['name'])

    return variables

def load_variable(catalog_name, run_uid, table, variable):
    dataset = catalog[catalog_name][run_uid][table].read()
    try:
        data = dataset[variable].data
        shape = data.shape
        data = data.reshape((shape[0]*shape[1], shape[2], shape[3]))

        # Convert to Fortran ordering
        data = np.asfortranarray(data)

        image_data = vtkImageData()
        (x, y, z) = data.shape

        # Metadata?
        image_data.SetOrigin(0, 0, 0)
        image_data.SetSpacing(1, 1, 1)
        image_data.SetExtent(0, x - 1, 0, y - 1, 0, z - 1)
        tomviz.utils.set_array(image_data, data)
    finally:
        dataset.close()

    return image_data
<commit_msg>First pass at migrating the code to use tiled<commit_after>import numpy as np

import tomviz.utils

from vtk import vtkImageData

_installed = False
try:
    from tiled.client import from_uri
    c = from_uri("https://tiled.nsls2.bnl.gov")
    from databroker import catalog
    from databroker.queries import TimeRange
    _installed = True
except ImportError:
    pass

def installed():
    return _installed

def catalogs():
    c = from_uri("https://tiled.nsls2.bnl.gov")
    cats = []
    for name in c:
        cats.append({
            "name": name,
            "description": "blank" #cat.describe()['description']
        })

    cats = sorted(cats, key=lambda c: c['name'])

    return cats

def runs(catalog_name, since, until):
    runs = []

    current = c[catalog_name]

    if since != "" and until != "":
        cat = current.search(TimeRange(since=since, until=until))

    cat = [] # [ current[-1], current[-2], current[-3] ]
    for i in range(-1, -21, -1):
        cat.append(current[i])

    for run in cat:
        md = run.metadata
        runs.append({
            "uid": md['start']['uid'],
            "planName": md['start']['plan_name'],
            "scanId": md['start']['scan_id'],
            "startTime": md['start']['time'],
            "stopTime": md['stop']['time'],
        })

    runs = sorted(runs, key=lambda r: r['uid'])

    return runs

def tables(catalog_name, run_uid):
    tables = []
    for name, _ in c[catalog_name][run_uid].items():
        tables.append({
            "name": name,
        })

    tables = sorted(tables, key=lambda t: t['name'])

    return tables

def variables(catalog_name, run_uid, table):
    variables = []

    # Would be nice to use context manager here, but it doesn't seem to close
    # the dataset.
    dataset = c[catalog_name][run_uid][table].read()
    try:
        for name, variable in dataset.data_vars.items():
            variables.append({
                "name": name,
                "size": variable.data.nbytes
            })
    finally:
        dataset.close()

    variables = sorted(variables, key=lambda v: v['name'])

    return variables

def load_variable(catalog_name, run_uid, table, variable):
    dataset = c[catalog_name][run_uid][table].read()
    try:
        data = dataset[variable].data
        shape = data.shape
        data = data.reshape((shape[0]*shape[1], shape[2], shape[3]))

        # Convert to Fortran ordering
        data = np.asfortranarray(data)

        image_data = vtkImageData()
        (x, y, z) = data.shape

        # Metadata?
        image_data.SetOrigin(0, 0, 0)
        image_data.SetSpacing(1, 1, 1)
        image_data.SetExtent(0, x - 1, 0, y - 1, 0, z - 1)
        tomviz.utils.set_array(image_data, data)
    finally:
        dataset.close()

    return image_data
<|endoftext|>