<commit_before>from __future__ import unicode_literals
from __future__ import absolute_import
import logging
from .service import Service

log = logging.getLogger(__name__)

def sort_service_dicts(services):
    # Get all services that are dependant on another.
    dependent_services = [s for s in services if s.get('links')]
    flatten_links = sum([s['links'] for s in dependent_services], [])
    # Get all services that are not linked to and don't link to others.
    non_dependent_sevices = [s for s in services if s['name'] not in flatten_links and not s.get('links')]
    sorted_services = []
    # Topological sort.
    while dependent_services:
        n = dependent_services.pop()
        # Check if a service is dependent on itself, if so raise an error.
        if n['name'] in n.get('links', []):
            raise DependencyError('A service can not link to itself: %s' % n['name'])
        sorted_services.append(n)
        for l in n['links']:
            # Get the linked service.
            linked_service = next(s for s in services if l == s['name'])
            # Check that there isn't a circular import between services.
            if n['name'] in linked_service.get('links', []):
                raise DependencyError('Circular import between %s and %s' % (n['name'], linked_service['name']))
            # Check the linked service has no links and is not already in the
            # sorted service list.
            if not linked_service.get('links') and linked_service not in sorted_services:
                sorted_services.insert(0, linked_service)
    return non_dependent_sevices + sorted_services

class Project(object):
    """
    A collection of services.
    """
    def __init__(self, name, services, client):
        self.name = name
        self.services = services
        self.client = client

    @classmethod
    def from_dicts(cls, name, service_dicts, client):
        """
        Construct a ServiceCollection from a list of dicts representing services.
        """
        project = cls(name, [], client)
        for service_dict in sort_service_dicts(service_dicts):
            # Reference links by object
            links = []
            if 'links' in service_dict:
                for service_name in service_dict.get('links', []):
                    links.append(project.get_service(service_name))
                del service_dict['links']
            project.services.append(Service(client=client, project=name, links=links, **service_dict))
        return project

    @classmethod
    def from_config(cls, name, config, client):
        dicts = []
        for service_name, service in list(config.items()):
            service['name'] = service_name
            dicts.append(service)
        return cls.from_dicts(name, dicts, client)

    def get_service(self, name):
        """
        Retrieve a service by name. Raises NoSuchService
        if the named service does not exist.
        """
        for service in self.services:
            if service.name == name:
                return service

        raise NoSuchService(name)

    def get_services(self, service_names=None):
        """
        Returns a list of this project's services filtered
        by the provided list of names, or all services if
        service_names is None or [].

        Preserves the original order of self.services.

        Raises NoSuchService if any of the named services
        do not exist.
        """
        if service_names is None or len(service_names) == 0:
            return self.services
        else:
            unsorted = [self.get_service(name) for name in service_names]
            return [s for s in self.services if s in unsorted]

    def recreate_containers(self, service_names=None):
        """
        For each service, create or recreate their containers.
        Returns a tuple with two lists. The first is a list of
        (service, old_container) tuples; the second is a list
        of (service, new_container) tuples.
        """
        old = []
        new = []

        for service in self.get_services(service_names):
            (s_old, s_new) = service.recreate_containers()
            old += [(service, container) for container in s_old]
            new += [(service, container) for container in s_new]

        return (old, new)

    def start(self, service_names=None, **options):
        for service in self.get_services(service_names):
            service.start(**options)

    def stop(self, service_names=None, **options):
        for service in self.get_services(service_names):
            service.stop(**options)

    def kill(self, service_names=None, **options):
        for service in self.get_services(service_names):
            service.kill(**options)

    def build(self, service_names=None, **options):
        for service in self.get_services(service_names):
            if service.can_be_built():
                service.build(**options)
            else:
                log.info('%s uses an image, skipping' % service.name)

    def remove_stopped(self, service_names=None, **options):
        for service in self.get_services(service_names):
            service.remove_stopped(**options)

    def containers(self, service_names=None, *args, **kwargs):
        l = []
        for service in self.get_services(service_names):
            for container in service.containers(*args, **kwargs):
                l.append(container)
        return l

class NoSuchService(Exception):
    def __init__(self, name):
        self.name = name
        self.msg = "No such service: %s" % self.name

    def __str__(self):
        return self.msg

class DependencyError(Exception):
    def __init__(self, msg):
        self.msg = msg

    def __str__(self):
        return self.msg<commit_msg>Stop projects in reverse order to starting<commit_after>from __future__ import unicode_literals
from __future__ import absolute_import
import logging
from .service import Service

log = logging.getLogger(__name__)

def sort_service_dicts(services):
    # Get all services that are dependant on another.
    dependent_services = [s for s in services if s.get('links')]
    flatten_links = sum([s['links'] for s in dependent_services], [])
    # Get all services that are not linked to and don't link to others.
    non_dependent_sevices = [s for s in services if s['name'] not in flatten_links and not s.get('links')]
    sorted_services = []
    # Topological sort.
    while dependent_services:
        n = dependent_services.pop()
        # Check if a service is dependent on itself, if so raise an error.
        if n['name'] in n.get('links', []):
            raise DependencyError('A service can not link to itself: %s' % n['name'])
        sorted_services.append(n)
        for l in n['links']:
            # Get the linked service.
            linked_service = next(s for s in services if l == s['name'])
            # Check that there isn't a circular import between services.
            if n['name'] in linked_service.get('links', []):
                raise DependencyError('Circular import between %s and %s' % (n['name'], linked_service['name']))
            # Check the linked service has no links and is not already in the
            # sorted service list.
            if not linked_service.get('links') and linked_service not in sorted_services:
                sorted_services.insert(0, linked_service)
    return non_dependent_sevices + sorted_services

class Project(object):
    """
    A collection of services.
    """
    def __init__(self, name, services, client):
        self.name = name
        self.services = services
        self.client = client

    @classmethod
    def from_dicts(cls, name, service_dicts, client):
        """
        Construct a ServiceCollection from a list of dicts representing services.
        """
        project = cls(name, [], client)
        for service_dict in sort_service_dicts(service_dicts):
            # Reference links by object
            links = []
            if 'links' in service_dict:
                for service_name in service_dict.get('links', []):
                    links.append(project.get_service(service_name))
                del service_dict['links']
            project.services.append(Service(client=client, project=name, links=links, **service_dict))
        return project

    @classmethod
    def from_config(cls, name, config, client):
        dicts = []
        for service_name, service in list(config.items()):
            service['name'] = service_name
            dicts.append(service)
        return cls.from_dicts(name, dicts, client)

    def get_service(self, name):
        """
        Retrieve a service by name. Raises NoSuchService
        if the named service does not exist.
        """
        for service in self.services:
            if service.name == name:
                return service

        raise NoSuchService(name)

    def get_services(self, service_names=None):
        """
        Returns a list of this project's services filtered
        by the provided list of names, or all services if
        service_names is None or [].

        Preserves the original order of self.services.

        Raises NoSuchService if any of the named services
        do not exist.
        """
        if service_names is None or len(service_names) == 0:
            return self.services
        else:
            unsorted = [self.get_service(name) for name in service_names]
            return [s for s in self.services if s in unsorted]

    def recreate_containers(self, service_names=None):
        """
        For each service, create or recreate their containers.
        Returns a tuple with two lists. The first is a list of
        (service, old_container) tuples; the second is a list
        of (service, new_container) tuples.
        """
        old = []
        new = []

        for service in self.get_services(service_names):
            (s_old, s_new) = service.recreate_containers()
            old += [(service, container) for container in s_old]
            new += [(service, container) for container in s_new]

        return (old, new)

    def start(self, service_names=None, **options):
        for service in self.get_services(service_names):
            service.start(**options)

    def stop(self, service_names=None, **options):
        for service in reversed(self.get_services(service_names)):
            service.stop(**options)

    def kill(self, service_names=None, **options):
        for service in reversed(self.get_services(service_names)):
            service.kill(**options)

    def build(self, service_names=None, **options):
        for service in self.get_services(service_names):
            if service.can_be_built():
                service.build(**options)
            else:
                log.info('%s uses an image, skipping' % service.name)

    def remove_stopped(self, service_names=None, **options):
        for service in self.get_services(service_names):
            service.remove_stopped(**options)

    def containers(self, service_names=None, *args, **kwargs):
        l = []
        for service in self.get_services(service_names):
            for container in service.containers(*args, **kwargs):
                l.append(container)
        return l

class NoSuchService(Exception):
    def __init__(self, name):
        self.name = name
        self.msg = "No such service: %s" % self.name

    def __str__(self):
        return self.msg

class DependencyError(Exception):
    def __init__(self, msg):
        self.msg = msg

    def __str__(self):
        return self.msg
<|endoftext|>


<commit_before># Copyright 2016 Alcatel-Lucent USA Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from oslo_log import helpers as log_helpers

from neutron.callbacks import resources
from neutron import policy

from nuage_neutron.plugins.common import constants
from nuage_neutron.plugins.common import exceptions
from nuage_neutron.plugins.common.extensions.nuagefloatingip \
    import NUAGE_FLOATINGIP
from nuage_neutron.plugins.common import nuagedb
from nuage_neutron.plugins.common.service_plugins \
    import vsd_passthrough_resource
from nuage_neutron.plugins.common import utils as nuage_utils
from nuage_neutron.plugins.common.validation import require
from nuagenetlib.restproxy import ResourceNotFoundException

class NuageFloatingip(vsd_passthrough_resource.VsdPassthroughResource):
    vsd_to_os = {
        'ID': 'id',
        'address': 'floating_ip_address',
        'assigned': 'assigned'
    }
    os_to_vsd = {
        'id': 'ID',
        'floating_ip_address': 'address',
        'assigned': 'assigned'
    }
    vsd_filterables = ['id', 'floating_ip_address', 'assigned']
    extra_filters = ['for_port', 'for_subnet']

    def __init__(self):
        super(NuageFloatingip, self).__init__()
        self.nuage_callbacks.subscribe(self.post_port_update,
                                       resources.PORT, constants.AFTER_UPDATE)
        self.nuage_callbacks.subscribe(self.post_port_create,
                                       resources.PORT, constants.AFTER_CREATE)
        self.nuage_callbacks.subscribe(self._post_port_show,
                                       resources.PORT, constants.AFTER_SHOW)

    @nuage_utils.handle_nuage_api_errorcode
    @log_helpers.log_method_call
    def get_nuage_floatingip(self, context, id, fields=None):
        try:
            floatingip = self.nuageclient.get_nuage_floatingip(id,
                                                               externalID=None)
            if not floatingip:
                raise exceptions.NuageNotFound(resource="nuage-floatingip",
                                               resource_id=id)
            return self.map_vsd_to_os(floatingip, fields=fields)
        except ResourceNotFoundException:
            raise exceptions.NuageNotFound(resource="nuage_floatingip",
                                           resource_id=id)

    @nuage_utils.handle_nuage_api_errorcode
    @log_helpers.log_method_call
    def get_nuage_floatingips(self, context, filters=None, fields=None):
        if 'for_port' in filters and 'for_subnet' in filters:
            msg = _("Can't combine both 'for_port' and 'for_subnet' filter")
            raise exceptions.NuageBadRequest(msg=msg)

        if 'for_port' in filters:
            getter = self.get_port_available_nuage_floatingips
        elif 'for_subnet' in filters:
            getter = self.get_subnet_available_nuage_floatingips
        else:
            policy.enforce(context, 'get_nuage_floatingip_all', None)
            getter = self.get_all_nuage_floatingips
        floatingips = getter(context, filters=filters)
        return [self.map_vsd_to_os(floatingip, fields=fields)
                for floatingip in floatingips]

    def get_port_available_nuage_floatingips(self, context, filters=None):
        port_id = filters.pop('for_port')[0]
        vsd_mapping = nuagedb.get_subnet_l2dom_by_port_id(context.session,
                                                          port_id)
        return self._get_available_nuage_floatingips(vsd_mapping, filters)

    def get_subnet_available_nuage_floatingips(self, context, filters=None):
        subnet_id = filters.pop('for_subnet')[0]
        vsd_mapping = nuagedb.get_subnet_l2dom_by_id(context.session,
                                                     subnet_id)
        require(vsd_mapping, 'vsd subnet mapping for subnet', subnet_id)
        return self._get_available_nuage_floatingips(vsd_mapping, filters)

    def get_all_nuage_floatingips(self, context, filters=None):
        vsd_filters = self.osfilters_to_vsdfilters(filters)
        return self.nuageclient.get_nuage_floatingips(externalID=None,
                                                      **vsd_filters)

    def _get_available_nuage_floatingips(self, vsd_mapping, filters):
        vsd_filters = self.osfilters_to_vsdfilters(filters)
        vsd_id = vsd_mapping['nuage_subnet_id']
        vsd_subnet = self.nuageclient.get_subnet_or_domain_subnet_by_id(vsd_id)
        if not vsd_subnet:
            raise exceptions.VsdSubnetNotFound(id=vsd_id)
        if vsd_subnet['type'] == constants.L2DOMAIN:
            return []

        domain_id = self.nuageclient.get_router_by_domain_subnet_id(
            vsd_subnet['subnet_id'])
        return self.nuageclient.get_nuage_domain_floatingips(
            domain_id, assigned=False, **vsd_filters)

    def post_port_update(self, resource, event, trigger, **kwargs):
        self.process_port_nuage_floatingip(resource, event, trigger, **kwargs)

    def post_port_create(self, resource, event, trigger, **kwargs):
        self.process_port_nuage_floatingip(resource, event, trigger, **kwargs)
        if kwargs['vport']:
            kwargs['port'][NUAGE_FLOATINGIP] = kwargs['request_port'].get(
                NUAGE_FLOATINGIP)
        else:
            kwargs['port'][NUAGE_FLOATINGIP] = None

    def process_port_nuage_floatingip(self, resource, event,
                                      trigger, **kwargs):
        request_port = kwargs['request_port']
        vport = kwargs['vport']
        if not vport or NUAGE_FLOATINGIP not in request_port:
            return
        self._process_port_nuage_floatingip(
            event, request_port, kwargs['rollbacks'], vport)

    @nuage_utils.handle_nuage_api_errorcode
    def _process_port_nuage_floatingip(self, event, request_port, rollbacks,
                                       vport):
        request_fip = request_port[NUAGE_FLOATINGIP] or {}
        if request_fip:
            floatingip = self.nuageclient.get_nuage_floatingip(
                request_fip.get('id'), required=True)
            if floatingip['externalID']:
                msg = _("Floatingip %s has externalID, it can't be used with "
                        "this API.") % floatingip['ID']
                raise exceptions.NuageBadRequest(msg=msg)
        if event == constants.AFTER_UPDATE:
            rollbacks.append(
                (self.nuageclient.update_vport,
                 [vport['nuage_vport_id'],
                  {'associatedFloatingIPID': vport['nuage_floating_ip']}],
                 [])
            )
        self.nuageclient.update_vport(
            vport['nuage_vport_id'],
            {'associatedFloatingIPID': request_fip.get('id')})

    def _post_port_show(self, resource, event, trigger, **kwargs):
        port = kwargs.get('port')
        vport = kwargs.get('vport')
        fields = kwargs.get('fields')
        if fields and NUAGE_FLOATINGIP not in fields:
            return
        if not vport:
            port[NUAGE_FLOATINGIP] = None
            return

        floatingip = self.nuageclient.get_nuage_floatingip(
            vport['nuage_floating_ip'])
        if floatingip:
            port[NUAGE_FLOATINGIP] = {'id': floatingip['ID'],
                                      'ip_address': floatingip['address']}
        else:
            port[NUAGE_FLOATINGIP] = None
<commit_msg>fix for OPENSTACK-1340<commit_after># Copyright 2016 Alcatel-Lucent USA Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from oslo_log import helpers as log_helpers

from neutron.callbacks import resources
from neutron import policy

from nuage_neutron.plugins.common import constants
from nuage_neutron.plugins.common import exceptions
from nuage_neutron.plugins.common.extensions.nuagefloatingip \
    import NUAGE_FLOATINGIP
from nuage_neutron.plugins.common import nuagedb
from nuage_neutron.plugins.common.service_plugins \
    import vsd_passthrough_resource
from nuage_neutron.plugins.common import utils as nuage_utils
from nuage_neutron.plugins.common.validation import require
from nuagenetlib.restproxy import ResourceNotFoundException

class NuageFloatingip(vsd_passthrough_resource.VsdPassthroughResource):
    vsd_to_os = {
        'ID': 'id',
        'address': 'floating_ip_address',
        'assigned': 'assigned'
    }
    os_to_vsd = {
        'id': 'ID',
        'floating_ip_address': 'address',
        'assigned': 'assigned'
    }
    vsd_filterables = ['id', 'floating_ip_address', 'assigned']
    extra_filters = ['for_port', 'for_subnet']

    def __init__(self):
        super(NuageFloatingip, self).__init__()
        self.nuage_callbacks.subscribe(self.post_port_update,
                                       resources.PORT, constants.AFTER_UPDATE)
        self.nuage_callbacks.subscribe(self.post_port_create,
                                       resources.PORT, constants.AFTER_CREATE)
        self.nuage_callbacks.subscribe(self._post_port_show,
                                       resources.PORT, constants.AFTER_SHOW)

    @nuage_utils.handle_nuage_api_errorcode
    @log_helpers.log_method_call
    def get_nuage_floatingip(self, context, id, fields=None):
        try:
            floatingip = self.nuageclient.get_nuage_floatingip(id,
                                                               externalID=None)
            if not floatingip:
                raise exceptions.NuageNotFound(resource="nuage-floatingip",
                                               resource_id=id)
            return self.map_vsd_to_os(floatingip, fields=fields)
        except ResourceNotFoundException:
            raise exceptions.NuageNotFound(resource="nuage_floatingip",
                                           resource_id=id)

    @nuage_utils.handle_nuage_api_errorcode
    @log_helpers.log_method_call
    def get_nuage_floatingips(self, context, filters=None, fields=None):
        if 'for_port' in filters and 'for_subnet' in filters:
            msg = _("Can't combine both 'for_port' and 'for_subnet' filter")
            raise exceptions.NuageBadRequest(msg=msg)

        if 'for_port' in filters:
            getter = self.get_port_available_nuage_floatingips
        elif 'for_subnet' in filters:
            getter = self.get_subnet_available_nuage_floatingips
        else:
            policy.enforce(context, 'get_nuage_floatingip_all', None)
            getter = self.get_all_nuage_floatingips
        floatingips = getter(context, filters=filters)
        return [self.map_vsd_to_os(floatingip, fields=fields)
                for floatingip in floatingips]

    def get_port_available_nuage_floatingips(self, context, filters=None):
        port_id = filters.pop('for_port')[0]
        vsd_mapping = nuagedb.get_subnet_l2dom_by_port_id(context.session,
                                                          port_id)
        return self._get_available_nuage_floatingips(vsd_mapping, filters)

    def get_subnet_available_nuage_floatingips(self, context, filters=None):
        subnet_id = filters.pop('for_subnet')[0]
        vsd_mapping = nuagedb.get_subnet_l2dom_by_id(context.session,
                                                     subnet_id)
        require(vsd_mapping, 'vsd subnet mapping for subnet', subnet_id)
        return self._get_available_nuage_floatingips(vsd_mapping, filters)

    def get_all_nuage_floatingips(self, context, filters=None):
        vsd_filters = self.osfilters_to_vsdfilters(filters)
        return self.nuageclient.get_nuage_floatingips(externalID=None,
                                                      **vsd_filters)

    def _get_available_nuage_floatingips(self, vsd_mapping, filters):
        vsd_filters = self.osfilters_to_vsdfilters(filters)
        vsd_id = vsd_mapping['nuage_subnet_id']
        vsd_subnet = self.nuageclient.get_subnet_or_domain_subnet_by_id(vsd_id)
        if not vsd_subnet:
            raise exceptions.VsdSubnetNotFound(id=vsd_id)
        if vsd_subnet['type'] == constants.L2DOMAIN:
            return []

        domain_id = self.nuageclient.get_router_by_domain_subnet_id(
            vsd_subnet['subnet_id'])
        return self.nuageclient.get_nuage_domain_floatingips(
            domain_id, assigned=False, **vsd_filters)

    def post_port_update(self, resource, event, trigger, **kwargs):
        self.process_port_nuage_floatingip(resource, event, trigger, **kwargs)

    def post_port_create(self, resource, event, trigger, **kwargs):
        self.process_port_nuage_floatingip(resource, event, trigger, **kwargs)
        if kwargs['vport']:
            kwargs['port'][NUAGE_FLOATINGIP] = kwargs['request_port'].get(
                NUAGE_FLOATINGIP)
        else:
            kwargs['port'][NUAGE_FLOATINGIP] = None

    def process_port_nuage_floatingip(self, resource, event,
                                      trigger, **kwargs):
        request_port = kwargs['request_port']
        vport = kwargs['vport']
        if not vport or NUAGE_FLOATINGIP not in request_port:
            return
        self._process_port_nuage_floatingip(
            event, request_port, kwargs['rollbacks'], vport)

    @nuage_utils.handle_nuage_api_errorcode
    def _process_port_nuage_floatingip(self, event, request_port, rollbacks,
                                       vport):
        request_fip = request_port[NUAGE_FLOATINGIP] or {}
        if request_fip:
            floatingip = self.nuageclient.get_nuage_floatingip(
                request_fip.get('id'), required=True)
            if floatingip['externalID']:
                msg = _("Floatingip %s has externalID, it can't be used with "
                        "this API.") % floatingip['ID']
                raise exceptions.NuageBadRequest(msg=msg)
        if event == constants.AFTER_UPDATE:
            rollbacks.append(
                (self.nuageclient.update_vport,
                 [vport['nuage_vport_id'],
                  {'associatedFloatingIPID': vport['nuage_floating_ip']}],
                 {})
            )
        self.nuageclient.update_vport(
            vport['nuage_vport_id'],
            {'associatedFloatingIPID': request_fip.get('id')})

    def _post_port_show(self, resource, event, trigger, **kwargs):
        port = kwargs.get('port')
        vport = kwargs.get('vport')
        fields = kwargs.get('fields')
        if fields and NUAGE_FLOATINGIP not in fields:
            return
        if not vport:
            port[NUAGE_FLOATINGIP] = None
            return

        floatingip = self.nuageclient.get_nuage_floatingip(
            vport['nuage_floating_ip'])
        if floatingip:
            port[NUAGE_FLOATINGIP] = {'id': floatingip['ID'],
                                      'ip_address': floatingip['address']}
        else:
            port[NUAGE_FLOATINGIP] = None
<|endoftext|>


<commit_before>import fnmatch
import os
import re
import sublime
import sublime_plugin
import subprocess
from threading import Thread
import time

from sublime_haskell_common import get_cabal_project_dir_of_view, call_and_wait, log, are_paths_equal, get_setting
from autobuild import attach_sandbox, wait_for_build_to_complete, run_build_thread, run_chain_build_thread

class SublimeHaskellCabalBuild(sublime_plugin.WindowCommand):
	def run(self):
		run_build_command_with(
			'Cabal: Building Haskell',
			['cabal', 'build'])

	def is_enabled(self):
		return is_enabled_build_command()

class SublimeHaskellCabalClean(sublime_plugin.WindowCommand):
	def run(self):
		run_build_command_with(
			'Cabal: Cleaning Haskell',
			['cabal', 'clean'])

	def is_enabled(self):
		return is_enabled_build_command()

class SublimeHaskellCabalConfigure(sublime_plugin.WindowCommand):
	def run(self):
		run_build_command_with(
			'Cabal: Configuring Haskell',
			['cabal', 'configure'])

	def is_enabled(self):
		return is_enabled_build_command()

class SublimeHaskellCabalRebuild(sublime_plugin.WindowCommand):
	def run(self):
		run_build_commands_with(
			'Cabal: Rebuilding Haskell',
			[['cabal', 'clean'], ['cabal', 'configure'], ['cabal', 'build']])

	def is_enabled(self):
		return is_enabled_build_command()

class SublimeHaskellCabalDevBuild(sublime_plugin.WindowCommand):
	def run(self):
		run_build_command_with(
			'Cabal-Dev: Building Haskell',
			attach_sandbox(['cabal', 'build']))

	def is_enabled(self):
		return is_enabled_build_command()

class SublimeHaskellCabalDevClean(sublime_plugin.WindowCommand):
	def run(self):
		run_build_command_with(
			'Cabal-Dev: Cleaning Haskell',
			attach_sandbox(['cabal', 'clean']))

	def is_enabled(self):
		return is_enabled_build_command()

class SublimeHaskellCabalDevConfigure(sublime_plugin.WindowCommand):
	def run(self):
		run_build_command_with(
			'Cabal-Dev: Configuring Haskell',
			attach_sandbox(['cabal', 'configure']))

	def is_enabled(self):
		return is_enabled_build_command()

class SublimeHaskellCabalDevRebuild(sublime_plugin.WindowCommand):
	def run(self):
		run_build_commands_with(
			'Cabal-Dev: Rebuilding Haskell',
			[attach_sandbox(s) for s in [['cabal-dev', 'clean'], ['cabal-dev', 'configure'], ['cabal-dev', 'build']]])

	def is_enabled(self):
		return is_enabled_build_command()

def is_enabled_haskell_command():
	window = sublime.active_window()
	if not window:
		return False
	view = window.active_view()
	if view is None:
		return False
	file_shown_in_view = view.file_name()
	if file_shown_in_view is None:
		return False
	syntax_file_for_view = view.settings().get('syntax').lower()
	if 'haskell' not in syntax_file_for_view:
		return False
	return True

def is_enabled_build_command():
	window = sublime.active_window()
	if not window:
		return False
	view = window.active_view()
	if view is None:
		return False
	file_shown_in_view = view.file_name()
	if file_shown_in_view is None:
		return False
	syntax_file_for_view = view.settings().get('syntax').lower()
	if 'haskell' not in syntax_file_for_view:
		return False
	cabal_project_dir = get_cabal_project_dir_of_view(view)
	if cabal_project_dir is None:
		return False
	return True

def run_build_commands_with(msg, cmds):
	"""Run general build commands"""
	window = sublime.active_window()
	if not window:
		return
	view = window.active_view()
	if view is None:
		return
	file_shown_in_view = view.file_name()
	if file_shown_in_view is None:
		return
	syntax_file_for_view = view.settings().get('syntax').lower()
	if 'haskell' not in syntax_file_for_view:
		return
	cabal_project_dir = get_cabal_project_dir_of_view(view)
	if cabal_project_dir is None:
		return

	run_chain_build_thread(view, cabal_project_dir, msg, cmds)

def run_build_command_with(msg, cmd):
	"""Run one command"""
	run_build_commands_with(msg, [cmd])
<commit_msg>Fixed command names<commit_after>import fnmatch
import os
import re
import sublime
import sublime_plugin
import subprocess
from threading import Thread
import time

from sublime_haskell_common import get_cabal_project_dir_of_view, call_and_wait, log, are_paths_equal, get_setting
from autobuild import attach_sandbox, wait_for_build_to_complete, run_build_thread, run_chain_build_thread

class SublimeHaskellCabalBuild(sublime_plugin.WindowCommand):
	def run(self):
		run_build_command_with(
			'Cabal: Building Haskell',
			['cabal', 'build'])

	def is_enabled(self):
		return is_enabled_build_command()

class SublimeHaskellCabalClean(sublime_plugin.WindowCommand):
	def run(self):
		run_build_command_with(
			'Cabal: Cleaning Haskell',
			['cabal', 'clean'])

	def is_enabled(self):
		return is_enabled_build_command()

class SublimeHaskellCabalConfigure(sublime_plugin.WindowCommand):
	def run(self):
		run_build_command_with(
			'Cabal: Configuring Haskell',
			['cabal', 'configure'])

	def is_enabled(self):
		return is_enabled_build_command()

class SublimeHaskellCabalRebuild(sublime_plugin.WindowCommand):
	def run(self):
		run_build_commands_with(
			'Cabal: Rebuilding Haskell',
			[['cabal', 'clean'], ['cabal', 'configure'], ['cabal', 'build']])

	def is_enabled(self):
		return is_enabled_build_command()

class SublimeHaskellCabalDevBuild(sublime_plugin.WindowCommand):
	def run(self):
		run_build_command_with(
			'Cabal-Dev: Building Haskell',
			attach_sandbox(['cabal-dev', 'build']))

	def is_enabled(self):
		return is_enabled_build_command()

class SublimeHaskellCabalDevClean(sublime_plugin.WindowCommand):
	def run(self):
		run_build_command_with(
			'Cabal-Dev: Cleaning Haskell',
			attach_sandbox(['cabal-dev', 'clean']))

	def is_enabled(self):
		return is_enabled_build_command()

class SublimeHaskellCabalDevConfigure(sublime_plugin.WindowCommand):
	def run(self):
		run_build_command_with(
			'Cabal-Dev: Configuring Haskell',
			attach_sandbox(['cabal-dev', 'configure']))

	def is_enabled(self):
		return is_enabled_build_command()

class SublimeHaskellCabalDevRebuild(sublime_plugin.WindowCommand):
	def run(self):
		run_build_commands_with(
			'Cabal-Dev: Rebuilding Haskell',
			[attach_sandbox(s) for s in [['cabal-dev', 'clean'], ['cabal-dev', 'configure'], ['cabal-dev', 'build']]])

	def is_enabled(self):
		return is_enabled_build_command()

def is_enabled_haskell_command():
	window = sublime.active_window()
	if not window:
		return False
	view = window.active_view()
	if view is None:
		return False
	file_shown_in_view = view.file_name()
	if file_shown_in_view is None:
		return False
	syntax_file_for_view = view.settings().get('syntax').lower()
	if 'haskell' not in syntax_file_for_view:
		return False
	return True

def is_enabled_build_command():
	window = sublime.active_window()
	if not window:
		return False
	view = window.active_view()
	if view is None:
		return False
	file_shown_in_view = view.file_name()
	if file_shown_in_view is None:
		return False
	syntax_file_for_view = view.settings().get('syntax').lower()
	if 'haskell' not in syntax_file_for_view:
		return False
	cabal_project_dir = get_cabal_project_dir_of_view(view)
	if cabal_project_dir is None:
		return False
	return True

def run_build_commands_with(msg, cmds):
	"""Run general build commands"""
	window = sublime.active_window()
	if not window:
		return
	view = window.active_view()
	if view is None:
		return
	file_shown_in_view = view.file_name()
	if file_shown_in_view is None:
		return
	syntax_file_for_view = view.settings().get('syntax').lower()
	if 'haskell' not in syntax_file_for_view:
		return
	cabal_project_dir = get_cabal_project_dir_of_view(view)
	if cabal_project_dir is None:
		return

	run_chain_build_thread(view, cabal_project_dir, msg, cmds)

def run_build_command_with(msg, cmd):
	"""Run one command"""
	run_build_commands_with(msg, [cmd])
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
import os
import sys
import time
import argparse
import socket

import colorama

# No cactus imports here! There's no logging in place (or anything really).
# It's best to wait until we've ran setup_logging before running imports (some import soft errors log).

class CactusCli(object):
    """
    We wrap all imports into this object to control their order
    """
    def __init__(self, Site, bootstrap):

        self.Site = Site
        self.bootstrap = bootstrap

    def create(self, path, skeleton=None):
        """
        Creates a new project at the given path.

        :param path: The path where the new project should be created
        :param skeleton: An (optional) skeleton to use to create the project.
                         This could be a zip, tar archive (file path or URL)
        """

        if os.path.exists(path):
            new_path = '%s.%s.moved' % (path, int(time.time()))
            if raw_input('Path %s exists, move aside to %s? (y/n) ' % (path, new_path)) == 'y':
                os.rename(path, new_path)
            else:
                sys.exit()

        self.bootstrap(path, skeleton)

    def build(self, path, config):
        """Build a cactus project"""

        site = self.Site(path, config)
        site.build()

    def deploy(self, path, config):
        """Upload the project to S3"""
        site = self.Site(path, config)
        site.upload()

    def make_messages(self, path, config):
        """ Create the list of translation files for the site """
        site = self.Site(path, config)
        site.make_messages()

    def serve(self, path, config, port, browser):
        """Serve the project and watch changes"""
        site = self.Site(path, config)
        site.serve(port=port, browser=browser)

    def domain_setup(self, path, config):
        site = self.Site(path, config)
        site.domain_setup()

    def domain_list(self, path, config):
        site = self.Site(path, config)
        site.domain_list()

def main():
    # Basic UI and logging setup
    colorama.init()

    from cactus.logger import setup_logging
    setup_logging()

    # Network setup that we should presumably remove. Leaving it there for now:
    # it's a better place than cactus/__init__.py

    socket.setdefaulttimeout(5)

    # Cactus imports

    from cactus.site import Site
    from cactus.bootstrap import bootstrap

    cli = CactusCli(Site, bootstrap)

    # Actual CLI parsing

    parser = argparse.ArgumentParser(description = "Build and deploy static websites using Django templates.")

    subparsers = parser.add_subparsers(title = 'subcommands', description = 'Valid subcommands',
                                       help = 'Select a command to run.')

    parser_create = subparsers.add_parser('create', help='Create a new project')
    parser_create.add_argument('path', help='The path where the new project should be created')
    parser_create.add_argument('-s', '--skeleton', help='An archive to use as skeleton to create the new project')
    parser_create.set_defaults(target=cli.create)

    parser_build = subparsers.add_parser('build', help = 'Build the current project.')
    parser_build.set_defaults(target=cli.build)

    parser_deploy = subparsers.add_parser('deploy', help = 'Deploy the current project to S3.')
    parser_deploy.set_defaults(target=cli.deploy)

    parser_serve = subparsers.add_parser('serve', help = 'Serve the current project.')
    parser_serve.set_defaults(target=cli.serve)
    parser_serve.add_argument('-p', '--port', default = 8000, type = int, help = 'The port on which to serve the site.')
    parser_serve.add_argument('-b', '--browser', action = 'store_true',
                              help = 'Whether to open a browser for the site.')

    parser_make_messages = subparsers.add_parser('messages:make', help='Create translation files for the current project')
    parser_make_messages.set_defaults(target=cli.make_messages)

    parser_domain_setup = subparsers.add_parser('domain:setup', help='Setup records for a domain with route 53')
    parser_domain_setup.set_defaults(target=cli.domain_setup)

    parser_domain_list = subparsers.add_parser('domain:list', help='Setup records for a domain with route 53')
    parser_domain_list.set_defaults(target=cli.domain_list)

    for subparser in (parser_build, parser_deploy, parser_serve, parser_make_messages, parser_domain_setup, parser_domain_list):
        subparser.add_argument('-c', '--config', action="append",
                               help='Add a config file you want to use')

        subparser.set_defaults(path = os.getcwd())

    args = parser.parse_args()

    # Small hack to provide a default value while not replacing what's
    # given by the user, if there is
    if hasattr(args, 'config') and args.config is None:  # We don't need config for create
        args.config = ["config.json"]

    args.target(**{k: v for k, v in vars(args).items() if k != 'target'})

if __name__ == "__main__":
    main()
<commit_msg>Fix version compatibility issues in CLI<commit_after>#!/usr/bin/env python
# encoding: utf-8
import os
import sys
import time
import argparse
import socket

from six.moves import input
import colorama

# No cactus imports here! There's no logging in place (or anything really).
# It's best to wait until we've ran setup_logging before running imports (some import soft errors log).

class CactusCli(object):
    """
    We wrap all imports into this object to control their order
    """
    def __init__(self, Site, bootstrap):

        self.Site = Site
        self.bootstrap = bootstrap

    def create(self, path, skeleton=None):
        """
        Creates a new project at the given path.

        :param path: The path where the new project should be created
        :param skeleton: An (optional) skeleton to use to create the project.
                         This could be a zip, tar archive (file path or URL)
        """
        if os.path.exists(path):
            new_path = '%s.%s.moved' % (path, int(time.time()))
            if input('Path %s exists, move aside to %s? (y/n) ' % (path, new_path)) == 'y':
                os.rename(path, new_path)
            else:
                sys.exit()

        self.bootstrap(path, skeleton)

    def build(self, path, config):
        """Build a cactus project"""
        site = self.Site(path, config)
        site.build()

    def deploy(self, path, config):
        """Upload the project to S3"""
        site = self.Site(path, config)
        site.upload()

    def make_messages(self, path, config):
        """ Create the list of translation files for the site """
        site = self.Site(path, config)
        site.make_messages()

    def serve(self, path, config, port, browser):
        """Serve the project and watch changes"""
        site = self.Site(path, config)
        site.serve(port=port, browser=browser)

    def domain_setup(self, path, config):
        site = self.Site(path, config)
        site.domain_setup()

    def domain_list(self, path, config):
        site = self.Site(path, config)
        site.domain_list()

def main():
    # Basic UI and logging setup
    colorama.init()

    from cactus.logger import setup_logging
    setup_logging()

    # Network setup that we should presumably remove. Leaving it there for now:
    # it's a better place than cactus/__init__.py

    socket.setdefaulttimeout(5)

    # Cactus imports

    from cactus.site import Site
    from cactus.bootstrap import bootstrap

    cli = CactusCli(Site, bootstrap)

    # Actual CLI parsing

    parser = argparse.ArgumentParser(description = "Build and deploy static websites using Django templates.")

    subparsers = parser.add_subparsers(title = 'subcommands', description = 'Valid subcommands',
                                       help = 'Select a command to run.')

    parser_create = subparsers.add_parser('create', help='Create a new project')
    parser_create.add_argument('path', help='The path where the new project should be created')
    parser_create.add_argument('-s', '--skeleton', help='An archive to use as skeleton to create the new project')
    parser_create.set_defaults(target=cli.create)

    parser_build = subparsers.add_parser('build', help = 'Build the current project.')
    parser_build.set_defaults(target=cli.build)

    parser_deploy = subparsers.add_parser('deploy', help = 'Deploy the current project to S3.')
    parser_deploy.set_defaults(target=cli.deploy)

    parser_serve = subparsers.add_parser('serve', help = 'Serve the current project.')
    parser_serve.set_defaults(target=cli.serve)
    parser_serve.add_argument('-p', '--port', default = 8000, type = int, help = 'The port on which to serve the site.')
    parser_serve.add_argument('-b', '--browser', action = 'store_true',
                              help = 'Whether to open a browser for the site.')

    parser_make_messages = subparsers.add_parser('messages:make', help='Create translation files for the current project')
    parser_make_messages.set_defaults(target=cli.make_messages)

    parser_domain_setup = subparsers.add_parser('domain:setup', help='Setup records for a domain with route 53')
    parser_domain_setup.set_defaults(target=cli.domain_setup)

    parser_domain_list = subparsers.add_parser('domain:list', help='Setup records for a domain with route 53')
    parser_domain_list.set_defaults(target=cli.domain_list)

    for subparser in (parser_build, parser_deploy, parser_serve, parser_make_messages, parser_domain_setup, parser_domain_list):
        subparser.add_argument('-c', '--config', action="append",
                               help='Add a config file you want to use')

        subparser.set_defaults(path = os.getcwd())

    ns = parser.parse_args(args)

    # Small hack to provide a default value while not replacing what's
    # given by the user, if there is
    if hasattr(args, 'config') and args.config is None:  # We don't need config for create
        args.config = ["config.json"]

    kwargs = dict((k, v) for k, v in vars(ns).items() if k not in ['target', 'verbose', 'quiet'])
    ns.target(**kwargs)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import json
import awsiot
import logging
import sys
import time
import platform
import datetime

try:
    import picamera
except ImportError:
    logging.error("Unable to import picamera")
    pass

def snapshot(filename):
    try:
        logging.info("snapshot: {}".format(filename))
        camera.capture(filename)
        return True
    except Exception as e:
        logging.error("snapshot failed {}".format(e.message))
        return False

def callback(client, user_data, message):
    try:
        msg = json.loads(message.payload)
    except ValueError:
        msg = None
    logging.debug("received {} {}".format(message.topic, msg))
    commands = filter(None, message.topic.replace(args.topic, '').split('/'))
    now = datetime.datetime.now()
    if len(commands) > 0:
        cmd = commands.pop(0)
        tags = {'created': awsiot.timestamp_string(now), 'source': args.source}
        if cmd == 'workspace':
            logging.debug("command: {}".format(cmd))
            filename = "{}-{}.jpg".format(args.source, awsiot.file_timestamp_string(now))
            if snapshot(filename) and args.bucket is not None:
                awsiot.mv_to_s3(filename, args.bucket, tags)
        elif cmd == 'snapshot':
            logging.debug("command: {}".format(cmd))
            filename = "{}.jpg".format(args.source)
            if snapshot(filename) and args.web_bucket is not None:
                awsiot.mv_to_s3(filename, args.web_bucket,tags)
        elif cmd == 'recording':
            logging.debug("command: {}".format(cmd))
        elif cmd == 'recognize':
            logging.debug("command: {}".format(cmd))
            filename = "{}-{}.jpg".format(args.source, awsiot.file_timestamp_string(now))
            if snapshot(filename) and args.bucket is not None:
                awsiot.mv_to_s3(filename, args.bucket)
                result = awsiot.recognize(filename, args.bucket)
                logging.info("recognize result: {}".format(result))
                if "Labels" in result:
                    tags['recognize'] = awsiot.tagify(result['Labels'], 'Name')
                    awsiot.s3_tag(filename, args.bucket, tags)
        else:
            logging.warning('Unrecognized command: {}'.format(cmd))
    else:
        logging.warning("No commands")

if __name__ == "__main__":
    parser = awsiot.iot_arg_parser()
    parser.add_argument("-x", "--width", help="camera resolution width", type=int, default=1920)
    parser.add_argument("-y", "--height", help="camera resolution height", type=int, default=1080)
    parser.add_argument("-z", "--rotation", help="camera rotation", type=int, default=0)
    parser.add_argument("-s", "--source", help="source name", default=platform.node().split('.')[0])
    parser.add_argument("-b", "--bucket", help="S3 bucket")
    parser.add_argument("-w", "--web_bucket", help="S3 bucket for web storage")
    args = parser.parse_args()

    logging.basicConfig(filename=awsiot.LOG_FILE, level=args.log_level, format=awsiot.LOG_FORMAT)

    subscriber = awsiot.Subscriber(args.endpoint, args.rootCA, args.cert, args.key, args.thing, args.groupCA)

    camera = picamera.PiCamera()
    camera.resolution = (args.width, args.height)
    camera.rotation = args.rotation

    subscriber.subscribe("{}/#".format(args.topic), callback)
    time.sleep(2)  # pause

    # Loop forever
    try:
        while True:
            time.sleep(0.5)  # sleep needed because CPU race
    except (KeyboardInterrupt, SystemExit):
        sys.exit()
<commit_msg>store recognize as tag to image<commit_after>#!/usr/bin/env python

import json
import awsiot
import logging
import sys
import time
import platform
import datetime

try:
    import picamera
except ImportError:
    logging.error("Unable to import picamera")
    pass

RECOGNIZE = 'recognize'

def snapshot(filename):
    try:
        logging.info("snapshot: {}".format(filename))
        camera.capture(filename)
        return True
    except Exception as e:
        logging.error("snapshot failed {}".format(e.message))
        return False

def callback(client, user_data, message):
    try:
        msg = json.loads(message.payload)
    except ValueError:
        msg = None
    logging.debug("received {} {}".format(message.topic, msg))
    commands = filter(None, message.topic.replace(args.topic, '').split('/'))
    now = datetime.datetime.now()
    if len(commands) > 0:
        cmd = commands.pop(0)
        tags = {'created': awsiot.timestamp_string(now), 'source': args.source}
        if cmd == 'workspace':
            logging.debug("command: {}".format(cmd))
            filename = "{}-{}.jpg".format(args.source, awsiot.file_timestamp_string(now))
            if snapshot(filename) and args.bucket is not None:
                awsiot.mv_to_s3(filename, args.bucket, tags)
        elif cmd == 'snapshot':
            logging.debug("command: {}".format(cmd))
            filename = "{}.jpg".format(args.source)
            if snapshot(filename) and args.web_bucket is not None:
                awsiot.mv_to_s3(filename, args.web_bucket, tags)
        elif cmd == 'recording':
            logging.debug("command: {}".format(cmd))
        elif cmd == RECOGNIZE:
            logging.debug("command: {}".format(cmd))
            filename = "{}-{}.jpg".format(args.source, awsiot.file_timestamp_string(now))
            if snapshot(filename) and args.bucket is not None:
                awsiot.mv_to_s3(filename, args.bucket)
                result = awsiot.recognize(filename, args.bucket)
                logging.info("{}: {}".format(RECOGNIZE, result))
                if "Labels" in result:
                    tags[RECOGNIZE] = awsiot.tagify(result['Labels'], 'Name')
                    awsiot.s3_tag(filename, args.bucket, tags)
        else:
            logging.warning('Unrecognized command: {}'.format(cmd))
    else:
        logging.warning("No commands")

if __name__ == "__main__":
    parser = awsiot.iot_arg_parser()
    parser.add_argument("-x", "--width", help="camera resolution width", type=int, default=1920)
    parser.add_argument("-y", "--height", help="camera resolution height", type=int, default=1080)
    parser.add_argument("-z", "--rotation", help="camera rotation", type=int, default=0)
    parser.add_argument("-s", "--source", help="source name", default=platform.node().split('.')[0])
    parser.add_argument("-b", "--bucket", help="S3 bucket")
    parser.add_argument("-w", "--web_bucket", help="S3 bucket for web storage")
    args = parser.parse_args()

    logging.basicConfig(filename=awsiot.LOG_FILE, level=args.log_level, format=awsiot.LOG_FORMAT)

    subscriber = awsiot.Subscriber(args.endpoint, args.rootCA, args.cert, args.key, args.thing, args.groupCA)

    camera = picamera.PiCamera()
    camera.resolution = (args.width, args.height)
    camera.rotation = args.rotation

    subscriber.subscribe("{}/#".format(args.topic), callback)
    time.sleep(2)  # pause

    # Loop forever
    try:
        while True:
            time.sleep(0.5)  # sleep needed because CPU race
    except (KeyboardInterrupt, SystemExit):
        sys.exit()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full
# list see the documentation:
# http://www.sphinx-doc.org/en/master/config

# -- Path setup --------------------------------------------------------------

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
# import os
# import sys
# sys.path.insert(0, os.path.abspath('.'))

import sys
import os
import shlex
import subprocess

# -- Project information -----------------------------------------------------

# The master toctree document.
master_doc = 'index'

default_role = 'code'

project = 'OpenImageIO'
copyright = '2008-present, Contributors to OpenImageIO'
author = 'Larry Gritz'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.

# LG addition: we search for it in the CMakeLists.txt so we don't need to
# keep modifying this file:
version = '0.0'
release = '0.0.0'
import re
version_regex = re.compile(r'project .* VERSION ((\d+\.\d+)\.\d+)')
f = open('../../CMakeLists.txt')
for l in f:
    aa=re.search(version_regex, l)
    if aa is not None:
       release = aa.group(1)
       version = aa.group(2)
       break
f.close()
print ("OIIO docs version = {}, release = {}".format(version, release))

# -- General configuration ---------------------------------------------------

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.

# sphinxtr extensions:
# https://github.com/jterrace/sphinxtr
# Copyright (c) 2012, Jeff Terrace
# BSD 2-clause license.

# add custom extensions directory to python path
sys.path.insert(0, os.path.join(os.path.abspath(os.path.dirname(__file__)), 'extensions/sphinxtr'))
#import html_mods
#import latex_mods

extensions = [
              'breathe',

              # sphinxtr extensions:
              #'fix_equation_ref',
              #'sphinx.ext.mathjax',
              #'sphinx.ext.ifconfig',
              #'subfig',
              #'numfig',
              'numsec',
              #'natbib',
              #'figtable',
              #'singlehtml_toc',
              #'singletext',
 ]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['templates']

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = []

# -- Options for HTML output -------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
#
#html_theme = 'alabaster'
#html_theme = 'astropy-sphinx-theme'
html_theme = 'sphinx_rtd_theme'

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Breathe options
breathe_projects = { "oiio": "../../build/doxygen/xml" }
breathe_default_project = "oiio"
breathe_domain_by_extension = {'h': 'cpp'}
breathe_default_members = ()
primary_domain = 'cpp'
highlight_language = 'cpp'

read_the_docs_build = os.environ.get('READTHEDOCS', None) == 'True'

if read_the_docs_build:
    if not os.path.exists('../../build/doxygen') :
        os.mkdir ('../../build/doxygen')
    print ("cwd =", os.getcwd())
    print ("checkpoint -- rtd build")
    subprocess.call('echo "Calling Doxygen"', shell=True)
    subprocess.call(['doxygen'], shell=True)
    subprocess.call('echo "Ran Doxygen"', shell=True)
<commit_msg>In docs generation on rtd, need os.makedirs instead of os.mkdir.<commit_after>#!/usr/bin/env python3

# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full
# list see the documentation:
# http://www.sphinx-doc.org/en/master/config

# -- Path setup --------------------------------------------------------------

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
# import os
# import sys
# sys.path.insert(0, os.path.abspath('.'))

import sys
import os
import shlex
import subprocess

# -- Project information -----------------------------------------------------

# The master toctree document.
master_doc = 'index'

default_role = 'code'

project = 'OpenImageIO'
copyright = '2008-present, Contributors to OpenImageIO'
author = 'Larry Gritz'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.

# LG addition: we search for it in the CMakeLists.txt so we don't need to
# keep modifying this file:
version = '0.0'
release = '0.0.0'
import re
version_regex = re.compile(r'project .* VERSION ((\d+\.\d+)\.\d+)')
f = open('../../CMakeLists.txt')
for l in f:
    aa=re.search(version_regex, l)
    if aa is not None:
       release = aa.group(1)
       version = aa.group(2)
       break
f.close()
print ("OIIO docs version = {}, release = {}".format(version, release))

# -- General configuration ---------------------------------------------------

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.

# sphinxtr extensions:
# https://github.com/jterrace/sphinxtr
# Copyright (c) 2012, Jeff Terrace
# BSD 2-clause license.

# add custom extensions directory to python path
sys.path.insert(0, os.path.join(os.path.abspath(os.path.dirname(__file__)), 'extensions/sphinxtr'))
#import html_mods
#import latex_mods

extensions = [
              'breathe',

              # sphinxtr extensions:
              #'fix_equation_ref',
              #'sphinx.ext.mathjax',
              #'sphinx.ext.ifconfig',
              #'subfig',
              #'numfig',
              'numsec',
              #'natbib',
              #'figtable',
              #'singlehtml_toc',
              #'singletext',
 ]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['templates']

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = []

# -- Options for HTML output -------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
#
#html_theme = 'alabaster'
#html_theme = 'astropy-sphinx-theme'
html_theme = 'sphinx_rtd_theme'

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Breathe options
breathe_projects = { "oiio": "../../build/doxygen/xml" }
breathe_default_project = "oiio"
breathe_domain_by_extension = {'h': 'cpp'}
breathe_default_members = ()
primary_domain = 'cpp'
highlight_language = 'cpp'

read_the_docs_build = os.environ.get('READTHEDOCS', None) == 'True'

if read_the_docs_build:
    print ("cwd =", os.getcwd())
    print ("checkpoint -- rtd build")
    if not os.path.exists('../../build/doxygen') :
        os.makedirs ('../../build/doxygen')
    print ("checkpoint 2 -- rtd build")
    subprocess.call('echo "Calling Doxygen"', shell=True)
    subprocess.call(['doxygen'], shell=True)
    subprocess.call('echo "Ran Doxygen"', shell=True)
<|endoftext|>


<commit_before>"""
Copyright (c) 2015 Red Hat, Inc
All rights reserved.

This software may be modified and distributed under the terms
of the MIT license. See the LICENSE file for details.
"""
import hashlib
import imp
import inspect
import glob
import logging
import os
import re
import shlex
import string
import subprocess
import yaml

from pkg_resources import resource_string, resource_filename

from cct.errors import CCTError
from cct.lib.git import clone_repo

try:
    import urllib.request as urlrequest
except ImportError:
    import urllib as urlrequest

logger = logging.getLogger('cct')

class ModuleManager(object):
    modules = {}
    artifacts_dir = ""

    def __init__(self, directory):
        self.directory = directory
        self.artifacts_dir = os.path.abspath(os.path.join(directory, '..', 'artifacts'))
        self.modules['base.Dummy'] = Dummy('base.Dummy', None, None)

    def discover_modules(self, directory=None):
        directory = directory if directory is not None else self.directory
        module_dirs = []
        for root, _, files in os.walk(directory):
            if 'module.yaml' in files:
                module_dirs.append(root)

        for mod_dir in module_dirs:
            try:
                with open(os.path.join(mod_dir, "module.yaml")) as stream:
                    config = yaml.load(stream)
                    if 'dependencies' in config:
                        self.process_module_deps(config['dependencies'])
                    self.find_modules(mod_dir, config['language'])
            except Exception as ex:
                logger.error("Cannot setup module: %s" % ex)
                raise ex

    def install_module(self, url, version):
        repo_dir = "%s/%s" % (self.directory, os.path.basename(url))
        if repo_dir.endswith('git'):
            repo_dir = repo_dir[:-4]
        logger.info("Cloning module into %s" % repo_dir)
        clone_repo(url, repo_dir, version)
        self.discover_modules(repo_dir)

    def process_module_deps(self, deps):
        for dep in deps:
            logger.debug("Fetching module from %s" % dep['url'])
            self.install_module(dep['url'], dep['version'] if 'version' in deps else None)

    def find_modules(self, directory, language):
        """
        Finds all modules in the subdirs of directory
        """
        if not os.path.exists(directory):
            return {}

        logger.debug("discovering modules in %s" % directory)

        if 'bash' in language:
            pattern = os.path.join(os.path.abspath(directory), '*.sh')
            for candidate in glob.glob(pattern):
                self.check_module_sh(candidate)
        else:
            pattern = os.path.join(os.path.abspath(directory), '*.py')
            for candidate in glob.glob(pattern):
                self.check_module_py(candidate)

    def check_module_py(self, candidate):
        module_name = "cct.module." + os.path.dirname(candidate).split('/')[-1]
        logger.debug("importing module %s to %s" % (os.path.abspath(candidate), module_name))
        module = imp.load_source(module_name, os.path.abspath(candidate))
        # Get all classes from our module
        for name, clazz in inspect.getmembers(module, inspect.isclass):
            # Check that class is from our namespace
            if module_name == clazz.__module__:
                # Instantiate class
                cls = getattr(module, name)
                if issubclass(cls, Module):
                    self.modules[module_name.split('.')[-1] + "." + cls.__name__] = cls(module_name.split('.')[-1] + "." + cls.__name__, os.path.dirname(candidate), os.path.join(self.artifacts_dir, name.split('.')[0]))

    def check_module_sh(self, candidate):
        module_name = "cct.module." + os.path.dirname(candidate).split('/')[-1]
        logger.debug("importing module %s to %s" % (os.path.abspath(candidate), module_name))
        name = module_name.split('.')[-1] + "." + os.path.basename(candidate)[:-3]
        self.modules[name] = ShellModule(name, os.path.dirname(candidate),
                                         os.path.join(self.artifacts_dir, name.split('.')[0]), candidate)

    def list(self):
        print("available cct modules:")
        for module, _ in self.modules.iteritems():
            print("  %s" % module)

    def list_module_oper(self, name):
        module = None
        if name not in self.modules:
            print("Module %s cannot be found!" % name)
            exit(1)
        else:
            module = self.modules[name]
        print("Module %s contains commands: " % name)

        if getattr(module, "setup").__doc__:
            print("  setup: %s " % getattr(module, "setup").__doc__)

        for method in dir(module):
            if callable(getattr(module, method)):
                if method[0] in string.ascii_lowercase and method not in ['run', 'setup', 'teardown']:
                    print("  %s: %s" % (method, getattr(module, method).__doc__))

        if getattr(module, "teardown").__doc__:
            print("  teardown: %s " % getattr(module, "teardown").__doc__)

class ModuleRunner(object):
    def __init__(self, module):
        self.module = module
        self.state = "Processing"

    def run(self):
        self.module.instance.setup()
        for operation in self.module.operations:
            if operation.command in ['setup', 'run', 'teardown']:
                continue
            self.module.instance._process_environment(operation)
            try:
                logger.debug("executing module %s operation %s with args %s" % (self.module.name, operation.command, operation.args))
                self.module.instance.run(operation)
                self.state = "Passed"
            except Exception as e:
                self.state = "Error"
                logger.error("module %s cannot execute %s with args %s" % (self.module.name, operation.command, operation.args))
                raise e
        self.module.instance.teardown()
        self.state = "Passed"

class Module(object):
    artifacts = {}

    def __init__(self, name, directory, artifacts_dir):
        self.name = name
        self.environment = {}
        self.deps = []
        self.operations = []
        self.instance = None
        self.state = "NotRun"
        self.logger = logger
        if not directory:
            return
        with open(os.path.join(directory, "module.yaml")) as stream:
            config = yaml.load(stream)
            if 'artifacts' in config:
                self._get_artifacts(config['artifacts'], artifacts_dir)

    def getenv(self, name, default=None):
        if os.environ.get(name):
            return os.environ.get(name)
        if name in self.environment:
            return self.environment[name]
        return default

    def _update_env(self, env):
        self.environment.update(env)

    def _process_operations(self, ops):
        for op in ops:
            for name, args in op.items():
                logger.debug("processing operation %s with args '%s'" % (name, args))
                if name == "environment":
                    self._update_env(self._create_env_dict(args))
                else:
                    self._add_operation(name, args)

    def _add_operation(self, name, args):
        operation = Operation(name, shlex.split(str(args)) if args else None)
        self.operations.append(operation)

    def _replace_variables(self, string):
        result = ""
        for token in string.split(" "):
            logger.debug("processing token %s", token)
            if token.startswith("$"):
                var_name = token[1:]
                # set value from environment
                if os.environ.get(var_name):
                    logger.debug("Using host variable %s" % token)
                    token = os.environ[var_name]
                elif self.environment.get(var_name):
                    logger.debug("Using yaml file variable %s" % token)
                    token = self.environment[var_name]
            result += token + " "
        return result

    def _get_artifacts(self, artifacts, destination):
        for artifact in artifacts:
            cct_artifact = CctArtifact(**artifact)
            cct_artifact.fetch(destination)
            self.artifacts[cct_artifact.name] = cct_artifact

    def setup(self):
        pass

    def teardown(self):
        pass

    def _get_resource(self, resource):
        return resource_string(inspect.getmodule(self.__class__).__name__, resource)

    def _get_resource_path(self, resource):
        return resource_filename(inspect.getmodule(self.__class__).__name__, resource)

    def _process_environment(self, operation):
        if '$' in operation.command:
            operation.command = self._replace_variables(operation.command)
        for i in range(len(operation.args)):
            if '$' in operation.args[i]:
                operation.args[i] = self._replace_variables(operation.args[i])

    def run(self, operation):
        try:
            logger.debug("invoking command %s", operation.command)
            method = getattr(self, operation.command)
            method_params = inspect.getargspec(method)
            args = []
            kwargs = {}
            for arg in operation.args:
                if '=' in arg:
                    key, value = arg.split('=', 1)
                    if key in method_params.args:
                        kwargs[key] = value
                    else:
                        args.append(arg.strip())
                else:
                    args.append(arg.strip())
            method(*args, **kwargs)
            logger.debug("operation '%s' Passed" % operation.command)
            operation.state = "Passed"
        except Exception as e:
            logger.error("%s operation failed with: %s", operation.command, e, exc_info=True)
            operation.state = "Error"
            self.state = "Error"
            raise e

class CctArtifact(object):
    """
    Object representing artifact file for changes
    name - name of the file
    md5sum - md5sum
    """
    def __init__(self, name, chksum, artifact="", hint=""):
        self.name = name
        self.chksum = chksum
        self.alg = chksum.split(':')[0]
        self.hash = chksum.split(':')[1]
        self.artifact = self.replace_variables(artifact) if '$' in artifact else artifact
        self.filename = os.path.basename(artifact)
        self.path = None
        self.hint = hint

    def fetch(self, directory):
        if not os.path.exists(directory):
            os.makedirs(directory)

        self.path = os.path.join(directory, self.filename)

        url = self.artifact
        if 'CCT_ARTIFACT_CACHE' in os.environ:
            cache = os.environ['CCT_ARTIFACT_CACHE']
            logger.info('Using CCT_ARTIFACT_CACHE=%s to fetch artifact' % cache)
            for var in [v for v in dir(self) if not callable(getattr(self, v))]:
                if var.startswith('_'):
                    continue
                token = '#%s#' % var
                cache = cache.replace(token, getattr(self, var))
            url = cache

        if self.check_sum():
            logger.info("Using cached artifact for %s" % self.filename)
            return

        logger.info("Fetching %s from  %s." % (self.filename, url))

        try:
            urlrequest.urlretrieve(url, self.path)
        except Exception as ex:
            if self.hint:
                raise CCTError('artifact: "%s" was not found. %s' % (self.path, self.hint))
            else:
                raise CCTError("cannot download artifact from url %s, error: %s" % (url, ex))

        if not self.check_sum():
            if self.hint:
                raise CCTError('hash is not correct for artifact: "%s". %s' % (self.path, self.hint))
            else:
                raise CCTError("artifact from %s doesn't match required chksum %s" % (url, self.chksum))

    def check_sum(self):
        if not os.path.exists(self.path):
            return False
        hash = getattr(hashlib, self.chksum[:self.chksum.index(':')])()
        with open(self.path, "rb") as f:
            for block in iter(lambda: f.read(65536), b""):
                hash.update(block)
        if self.chksum[self.chksum.index(':') + 1:] == hash.hexdigest():
            return True
        return False

    def replace_variables(self, string):
        var_regex = re.compile('\$\{.*\}')
        variable = var_regex.search(string).group(0)[2:-1]
        if os.environ.get(variable):
            logger.debug("Using host variable %s" % variable)
            string = var_regex.sub(os.environ[variable], string)
        return string

class Operation(object):
    """
    Object representing single operation
    """
    command = None
    args = []
    state = "NotRun"

    def __init__(self, command, args):
        self.command = command
        self.args = []
        if args:
            for arg in args:
                self.args.append(arg.rstrip())

class ShellModule(Module):
    modules_dirs = {}

    def __init__(self, name, directory, artifacts_dir, path):
        Module.__init__(self, name, directory, artifacts_dir)
        self.script = path
        self.modules_dirs[os.path.splitext(name)[0]] = os.path.dirname(path)
        self._discover()

    def __getattr__(self, name):
        def wrapper(*args, **kwargs):
            if name in self.names:
                self._run_function(name, *args)
            else:
                raise Exception("no such method")
        return wrapper

    def _discover(self):
        self.names = {}
        comment = ""
        param = re.compile("\$\d+")
        with open(self.script, "r") as f:
            name = ""
            for line in f:
                if line.startswith('#'):
                    comment += line[1:]
                elif 'function' in line:
                    name = line.split()[1][:-2]
                    function = {"name": name,
                                "comment": comment,
                                "params": {}}
                    self.names[name] = function
                elif param.search(line):
                    param_name = line.split('=')[0].split()[-1]
                    self.logger.debug("function: %s, param_id: %s, param_name: %s"
                                      % (name, param_name, param.search(line).group(0)))
                    self.names[name]["params"][param_name] = param.search(line).group(0)
                else:
                    comment = ""

    def _run_function(self, name, *args):
        cmd = '/bin/bash -c " source %s ; %s %s"' % (self.script, name, " ".join(args))
        try:
            env = dict(os.environ)
            env['CCT_MODULE_PATH'] = os.path.dirname(self.script)

            for name, mod_dir in self.modules_dirs.items():
                var_name = 'CCT_MODULE_PATH_%s' % name.upper()
                env[var_name] = mod_dir
                logger.info('Created %s environment variable for module %s.' % (var_name, mod_dir))

            for name, artifact in self.artifacts.items():
                var_name = 'CCT_ARTIFACT_PATH_' + name.upper()
                logger.info('Created %s environment variable pointing to %s.' % (var_name, artifact.path))
                env['CCT_ARTIFACT_PATH_' + name.upper()] = artifact.path
            out = subprocess.check_output(cmd, stderr=subprocess.STDOUT, env=env, shell=True)
            self.logger.debug("Step ended with output: %s" % out)
        except subprocess.CalledProcessError as e:
            raise CCTError(e.output)

class Dummy(Module):
    def dump(self, *args):
        """
        Dumps arguments to a logfile.

        Args:
         *args: Will be dumped :).
        """
        logger.info("dummy module performed dump with args %s and environment: %s" % (args, self.environment))
<commit_msg>log better message when artifact is referenced by filename only.<commit_after>"""
Copyright (c) 2015 Red Hat, Inc
All rights reserved.

This software may be modified and distributed under the terms
of the MIT license. See the LICENSE file for details.
"""
import hashlib
import imp
import inspect
import glob
import logging
import os
import re
import shlex
import string
import subprocess
import yaml

from pkg_resources import resource_string, resource_filename

from cct.errors import CCTError
from cct.lib.git import clone_repo

try:
    import urllib.request as urlrequest
except ImportError:
    import urllib as urlrequest

logger = logging.getLogger('cct')

class ModuleManager(object):
    modules = {}
    artifacts_dir = ""

    def __init__(self, directory):
        self.directory = directory
        self.artifacts_dir = os.path.abspath(os.path.join(directory, '..', 'artifacts'))
        self.modules['base.Dummy'] = Dummy('base.Dummy', None, None)

    def discover_modules(self, directory=None):
        directory = directory if directory is not None else self.directory
        module_dirs = []
        for root, _, files in os.walk(directory):
            if 'module.yaml' in files:
                module_dirs.append(root)

        for mod_dir in module_dirs:
            try:
                with open(os.path.join(mod_dir, "module.yaml")) as stream:
                    config = yaml.load(stream)
                    if 'dependencies' in config:
                        self.process_module_deps(config['dependencies'])
                    self.find_modules(mod_dir, config['language'])
            except Exception as ex:
                logger.error("Cannot setup module: %s" % ex)
                raise ex

    def install_module(self, url, version):
        repo_dir = "%s/%s" % (self.directory, os.path.basename(url))
        if repo_dir.endswith('git'):
            repo_dir = repo_dir[:-4]
        logger.info("Cloning module into %s" % repo_dir)
        clone_repo(url, repo_dir, version)
        self.discover_modules(repo_dir)

    def process_module_deps(self, deps):
        for dep in deps:
            logger.debug("Fetching module from %s" % dep['url'])
            self.install_module(dep['url'], dep['version'] if 'version' in deps else None)

    def find_modules(self, directory, language):
        """
        Finds all modules in the subdirs of directory
        """
        if not os.path.exists(directory):
            return {}

        logger.debug("discovering modules in %s" % directory)

        if 'bash' in language:
            pattern = os.path.join(os.path.abspath(directory), '*.sh')
            for candidate in glob.glob(pattern):
                self.check_module_sh(candidate)
        else:
            pattern = os.path.join(os.path.abspath(directory), '*.py')
            for candidate in glob.glob(pattern):
                self.check_module_py(candidate)

    def check_module_py(self, candidate):
        module_name = "cct.module." + os.path.dirname(candidate).split('/')[-1]
        logger.debug("importing module %s to %s" % (os.path.abspath(candidate), module_name))
        module = imp.load_source(module_name, os.path.abspath(candidate))
        # Get all classes from our module
        for name, clazz in inspect.getmembers(module, inspect.isclass):
            # Check that class is from our namespace
            if module_name == clazz.__module__:
                # Instantiate class
                cls = getattr(module, name)
                if issubclass(cls, Module):
                    self.modules[module_name.split('.')[-1] + "." + cls.__name__] = cls(module_name.split('.')[-1] + "." + cls.__name__, os.path.dirname(candidate), os.path.join(self.artifacts_dir, name.split('.')[0]))

    def check_module_sh(self, candidate):
        module_name = "cct.module." + os.path.dirname(candidate).split('/')[-1]
        logger.debug("importing module %s to %s" % (os.path.abspath(candidate), module_name))
        name = module_name.split('.')[-1] + "." + os.path.basename(candidate)[:-3]
        self.modules[name] = ShellModule(name, os.path.dirname(candidate),
                                         os.path.join(self.artifacts_dir, name.split('.')[0]), candidate)

    def list(self):
        print("available cct modules:")
        for module, _ in self.modules.iteritems():
            print("  %s" % module)

    def list_module_oper(self, name):
        module = None
        if name not in self.modules:
            print("Module %s cannot be found!" % name)
            exit(1)
        else:
            module = self.modules[name]
        print("Module %s contains commands: " % name)

        if getattr(module, "setup").__doc__:
            print("  setup: %s " % getattr(module, "setup").__doc__)

        for method in dir(module):
            if callable(getattr(module, method)):
                if method[0] in string.ascii_lowercase and method not in ['run', 'setup', 'teardown']:
                    print("  %s: %s" % (method, getattr(module, method).__doc__))

        if getattr(module, "teardown").__doc__:
            print("  teardown: %s " % getattr(module, "teardown").__doc__)

class ModuleRunner(object):
    def __init__(self, module):
        self.module = module
        self.state = "Processing"

    def run(self):
        self.module.instance.setup()
        for operation in self.module.operations:
            if operation.command in ['setup', 'run', 'teardown']:
                continue
            self.module.instance._process_environment(operation)
            try:
                logger.debug("executing module %s operation %s with args %s" % (self.module.name, operation.command, operation.args))
                self.module.instance.run(operation)
                self.state = "Passed"
            except Exception as e:
                self.state = "Error"
                logger.error("module %s cannot execute %s with args %s" % (self.module.name, operation.command, operation.args))
                raise e
        self.module.instance.teardown()
        self.state = "Passed"

class Module(object):
    artifacts = {}

    def __init__(self, name, directory, artifacts_dir):
        self.name = name
        self.environment = {}
        self.deps = []
        self.operations = []
        self.instance = None
        self.state = "NotRun"
        self.logger = logger
        if not directory:
            return
        with open(os.path.join(directory, "module.yaml")) as stream:
            config = yaml.load(stream)
            if 'artifacts' in config:
                self._get_artifacts(config['artifacts'], artifacts_dir)

    def getenv(self, name, default=None):
        if os.environ.get(name):
            return os.environ.get(name)
        if name in self.environment:
            return self.environment[name]
        return default

    def _update_env(self, env):
        self.environment.update(env)

    def _process_operations(self, ops):
        for op in ops:
            for name, args in op.items():
                logger.debug("processing operation %s with args '%s'" % (name, args))
                if name == "environment":
                    self._update_env(self._create_env_dict(args))
                else:
                    self._add_operation(name, args)

    def _add_operation(self, name, args):
        operation = Operation(name, shlex.split(str(args)) if args else None)
        self.operations.append(operation)

    def _replace_variables(self, string):
        result = ""
        for token in string.split(" "):
            logger.debug("processing token %s", token)
            if token.startswith("$"):
                var_name = token[1:]
                # set value from environment
                if os.environ.get(var_name):
                    logger.debug("Using host variable %s" % token)
                    token = os.environ[var_name]
                elif self.environment.get(var_name):
                    logger.debug("Using yaml file variable %s" % token)
                    token = self.environment[var_name]
            result += token + " "
        return result

    def _get_artifacts(self, artifacts, destination):
        for artifact in artifacts:
            cct_artifact = CctArtifact(**artifact)
            cct_artifact.fetch(destination)
            self.artifacts[cct_artifact.name] = cct_artifact

    def setup(self):
        pass

    def teardown(self):
        pass

    def _get_resource(self, resource):
        return resource_string(inspect.getmodule(self.__class__).__name__, resource)

    def _get_resource_path(self, resource):
        return resource_filename(inspect.getmodule(self.__class__).__name__, resource)

    def _process_environment(self, operation):
        if '$' in operation.command:
            operation.command = self._replace_variables(operation.command)
        for i in range(len(operation.args)):
            if '$' in operation.args[i]:
                operation.args[i] = self._replace_variables(operation.args[i])

    def run(self, operation):
        try:
            logger.debug("invoking command %s", operation.command)
            method = getattr(self, operation.command)
            method_params = inspect.getargspec(method)
            args = []
            kwargs = {}
            for arg in operation.args:
                if '=' in arg:
                    key, value = arg.split('=', 1)
                    if key in method_params.args:
                        kwargs[key] = value
                    else:
                        args.append(arg.strip())
                else:
                    args.append(arg.strip())
            method(*args, **kwargs)
            logger.debug("operation '%s' Passed" % operation.command)
            operation.state = "Passed"
        except Exception as e:
            logger.error("%s operation failed with: %s", operation.command, e, exc_info=True)
            operation.state = "Error"
            self.state = "Error"
            raise e

class CctArtifact(object):
    """
    Object representing artifact file for changes
    name - name of the file
    md5sum - md5sum
    """
    def __init__(self, name, chksum, artifact="", hint=""):
        self.name = name
        self.chksum = chksum
        self.alg = chksum.split(':')[0]
        self.hash = chksum.split(':')[1]
        self.artifact = self.replace_variables(artifact) if '$' in artifact else artifact
        self.filename = os.path.basename(artifact)
        self.path = None
        self.hint = hint

    def fetch(self, directory):
        if not os.path.exists(directory):
            os.makedirs(directory)

        self.path = os.path.join(directory, self.filename)

        url = self.artifact
        if 'CCT_ARTIFACT_CACHE' in os.environ:
            cache = os.environ['CCT_ARTIFACT_CACHE']
            logger.info('Using CCT_ARTIFACT_CACHE=%s to fetch artifact' % cache)
            for var in [v for v in dir(self) if not callable(getattr(self, v))]:
                if var.startswith('_'):
                    continue
                token = '#%s#' % var
                cache = cache.replace(token, getattr(self, var))
            url = cache

        if self.check_sum():
            logger.info("Using cached artifact for %s" % self.filename)
            return

        logger.info("Fetching %s from  %s." % (self.filename, url))

        try:
            if os.path.basename(url) == url:
                raise CCTError("Artifact is referenced by filename - can't download it.")
            urlrequest.urlretrieve(url, self.path)
        except Exception as ex:
            if self.hint:
                raise CCTError('artifact: "%s" was not found. %s' % (self.path, self.hint))
            else:
                raise CCTError("cannot download artifact from url %s, error: %s" % (url, ex))

        if not self.check_sum():
            if self.hint:
                raise CCTError('hash is not correct for artifact: "%s". %s' % (self.path, self.hint))
            else:
                raise CCTError("artifact from %s doesn't match required chksum %s" % (url, self.chksum))

    def check_sum(self):
        if not os.path.exists(self.path):
            return False
        hash = getattr(hashlib, self.chksum[:self.chksum.index(':')])()
        with open(self.path, "rb") as f:
            for block in iter(lambda: f.read(65536), b""):
                hash.update(block)
        if self.chksum[self.chksum.index(':') + 1:] == hash.hexdigest():
            return True
        return False

    def replace_variables(self, string):
        var_regex = re.compile('\$\{.*\}')
        variable = var_regex.search(string).group(0)[2:-1]
        if os.environ.get(variable):
            logger.debug("Using host variable %s" % variable)
            string = var_regex.sub(os.environ[variable], string)
        return string

class Operation(object):
    """
    Object representing single operation
    """
    command = None
    args = []
    state = "NotRun"

    def __init__(self, command, args):
        self.command = command
        self.args = []
        if args:
            for arg in args:
                self.args.append(arg.rstrip())

class ShellModule(Module):
    modules_dirs = {}

    def __init__(self, name, directory, artifacts_dir, path):
        Module.__init__(self, name, directory, artifacts_dir)
        self.script = path
        self.modules_dirs[os.path.splitext(name)[0]] = os.path.dirname(path)
        self._discover()

    def __getattr__(self, name):
        def wrapper(*args, **kwargs):
            if name in self.names:
                self._run_function(name, *args)
            else:
                raise Exception("no such method")
        return wrapper

    def _discover(self):
        self.names = {}
        comment = ""
        param = re.compile("\$\d+")
        with open(self.script, "r") as f:
            name = ""
            for line in f:
                if line.startswith('#'):
                    comment += line[1:]
                elif 'function' in line:
                    name = line.split()[1][:-2]
                    function = {"name": name,
                                "comment": comment,
                                "params": {}}
                    self.names[name] = function
                elif param.search(line):
                    param_name = line.split('=')[0].split()[-1]
                    self.logger.debug("function: %s, param_id: %s, param_name: %s"
                                      % (name, param_name, param.search(line).group(0)))
                    self.names[name]["params"][param_name] = param.search(line).group(0)
                else:
                    comment = ""

    def _run_function(self, name, *args):
        cmd = '/bin/bash -c " source %s ; %s %s"' % (self.script, name, " ".join(args))
        try:
            env = dict(os.environ)
            env['CCT_MODULE_PATH'] = os.path.dirname(self.script)

            for name, mod_dir in self.modules_dirs.items():
                var_name = 'CCT_MODULE_PATH_%s' % name.upper()
                env[var_name] = mod_dir
                logger.info('Created %s environment variable for module %s.' % (var_name, mod_dir))

            for name, artifact in self.artifacts.items():
                var_name = 'CCT_ARTIFACT_PATH_' + name.upper()
                logger.info('Created %s environment variable pointing to %s.' % (var_name, artifact.path))
                env['CCT_ARTIFACT_PATH_' + name.upper()] = artifact.path
            out = subprocess.check_output(cmd, stderr=subprocess.STDOUT, env=env, shell=True)
            self.logger.debug("Step ended with output: %s" % out)
        except subprocess.CalledProcessError as e:
            raise CCTError(e.output)

class Dummy(Module):
    def dump(self, *args):
        """
        Dumps arguments to a logfile.

        Args:
         *args: Will be dumped :).
        """
        logger.info("dummy module performed dump with args %s and environment: %s" % (args, self.environment))
<|endoftext|>


<commit_before>import re

def _scan(first_char, chars, allowed):
    ret = first_char
    p = chars.next
    while p is not None and re.match(allowed, p):
        ret += chars.move_next()
        p = chars.next
    return ret

def _scan_string(delim, chars):
    ret = ""
    while chars.next != delim:
        c = chars.move_next()
        if c is None:
            raise Exception("A string ran off the end of the program.")
        ret += c
    chars.move_next()
    return ret

class PeekableStream:

    def __init__(self, iterator):
        self.iterator = iter(iterator)
        self._fill()

    def _fill(self):
        try:
            self.next = next(self.iterator)
        except StopIteration:
            self.next = None

    def move_next(self):
        ret = self.next
        self._fill()
        return ret

_midsym = "[_a-zA-Z0-9]"
_stsym = "[_a-zA-Z]"

def lex(chars_iter):
    chars = PeekableStream(chars_iter)
    while chars.next is not None:
        c = chars.move_next()
        if c in " \n":              pass           # Ignore white space
        elif c in "(){},;=:":       yield (c, "")  # Punctuation marks
        elif c in "+-*/":           yield ("arithmetic", c)
        elif c in ("'", '"'):       yield ("string", _scan_string(c, chars))
        elif re.match("[.0-9]", c): yield ("number", _scan(c, chars, "[.0-9]"))
        elif re.match(_stsym, c):   yield ("symbol", _scan(c, chars, _midsym))
        elif c == "\t":
            raise Exception("Tab characters are not allowed in Cell.")
        else:
            raise Exception("Unrecognised character: '" + c + "'.")
<commit_msg>Update comment<commit_after>import re

def _scan(first_char, chars, allowed):
    ret = first_char
    p = chars.next
    while p is not None and re.match(allowed, p):
        ret += chars.move_next()
        p = chars.next
    return ret

def _scan_string(delim, chars):
    ret = ""
    while chars.next != delim:
        c = chars.move_next()
        if c is None:
            raise Exception("A string ran off the end of the program.")
        ret += c
    chars.move_next()
    return ret

class PeekableStream:

    def __init__(self, iterator):
        self.iterator = iter(iterator)
        self._fill()

    def _fill(self):
        try:
            self.next = next(self.iterator)
        except StopIteration:
            self.next = None

    def move_next(self):
        ret = self.next
        self._fill()
        return ret

_midsym = "[_a-zA-Z0-9]"
_stsym = "[_a-zA-Z]"

def lex(chars_iter):
    chars = PeekableStream(chars_iter)
    while chars.next is not None:
        c = chars.move_next()
        if c in " \n":              pass           # Ignore white space
        elif c in "(){},;=:":       yield (c, "")  # Special characters
        elif c in "+-*/":           yield ("arithmetic", c)
        elif c in ("'", '"'):       yield ("string", _scan_string(c, chars))
        elif re.match("[.0-9]", c): yield ("number", _scan(c, chars, "[.0-9]"))
        elif re.match(_stsym, c):   yield ("symbol", _scan(c, chars, _midsym))
        elif c == "\t":
            raise Exception("Tab characters are not allowed in Cell.")
        else:
            raise Exception("Unrecognised character: '" + c + "'.")
<|endoftext|>