<commit_before>#!/usr/bin/env python
# -- Content-Encoding: UTF-8 --
"""
Pelix remote services: Imported end points registry

:author: Thomas Calmant
:copyright: Copyright 2013, isandlaTech
:license: Apache License 2.0
:version: 0.1.1
:status: Beta

..

    Copyright 2013 isandlaTech

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

        http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
"""

# Module version
__version_info__ = (0, 1, 1)
__version__ = ".".join(str(x) for x in __version_info__)

# Documentation strings format
__docformat__ = "restructuredtext en"

# ------------------------------------------------------------------------------

# Remote Services constants
import pelix.remote

# iPOPO decorators
from pelix.ipopo.decorators import ComponentFactory, Requires, Provides, \
    Instantiate, Invalidate, Validate, BindField

# Standard library
import logging
import threading

# ------------------------------------------------------------------------------

_logger = logging.getLogger(__name__)

# ------------------------------------------------------------------------------

@ComponentFactory('pelix-remote-imports-registry-factory')
@Provides(pelix.remote.SERVICE_REGISTRY)
@Requires('_listeners', pelix.remote.SERVICE_IMPORT_ENDPOINT_LISTENER,
          aggregate=True, optional=True)
@Instantiate('pelix-remote-imports-registry')
class ImportsRegistry(object):
    """
    Registry of discovered end points. End points are identified by their UID
    """
    def __init__(self):
        """
        Sets up the component
        """
        # Listeners (injected)
        self._listeners = []

        # Framework UID
        self._fw_uid = None

        # Framework UID -> [Endpoints]
        self._frameworks = {}

        # End point UID -> Endpoint
        self._registry = {}

        # Lock
        self.__lock = threading.Lock()

        # Validation flag
        self.__validated = False

    @BindField('_listeners')
    def _bind_listener(self, field, listener, svc_ref):
        """
        New listener bound
        """
        with self.__lock:
            if self.__validated:
                # Late listener
                for endpoint in self._registry.values():
                    try:
                        listener.endpoint_added(endpoint)

                    except Exception as ex:
                        _logger.exception("Error calling listener: %s", ex)

    def add(self, endpoint):
        """
        Registers an end point and notifies listeners. Does nothing if the
        endpoint UID was already known.

        :param endpoint: An ImportedEndpoint object
        :return: True if the end point has been added
        """
        with self.__lock:
            # Check framework UID (avoid to import our own services)
            if endpoint.framework == self._fw_uid:
                _logger.debug("Same framework UID")
                return False

            # Check if the end point already exists
            if endpoint.uid in self._registry:
                # Already known end point: do nothing
                _logger.debug("Already known endpoint")
                return False

            # Store the end point
            self._registry[endpoint.uid] = endpoint
            if endpoint.framework:
                self._frameworks.setdefault(endpoint.framework, []) \
                                                            .append(endpoint)

        # Notify listeners (out of lock)
        if self._listeners:
            for listener in self._listeners[:]:
                try:
                    listener.endpoint_added(endpoint)

                except Exception as ex:
                        _logger.exception("Error calling listener: %s", ex)

        return True

    def update(self, uid, new_properties):
        """
        Updates an end point and notifies listeners

        :param uid: The UID of the end point
        :param new_properties: The new properties of the end point
        """
        try:
            with self.__lock:
                # Update the stored end point
                stored_endpoint = self._registry[uid]

                # Replace the stored properties
                old_properties = stored_endpoint.properties.copy()
                stored_endpoint.properties = new_properties.copy()

        except KeyError:
            # Unknown end point: ignore it
            return

        else:
            # Notify listeners
            if self._listeners:
                for listener in self._listeners[:]:
                    try:
                        listener.endpoint_updated(stored_endpoint,
                                                  old_properties)

                    except Exception as ex:
                        _logger.exception("Error calling listener: %s", ex)

    def remove(self, uid):
        """
        Unregisters an end point and notifies listeners

        :param uid: The UID of the end point to unregister
        """
        # Remove the end point from the individual storage
        try:
            endpoint = self._registry.pop(uid)

        except KeyError:
            # Unknown end point
            _logger.warning("Unknown end point UID: %s", uid)
            return

        # Remove it from its framework storage, if any
        try:
            framework_endpoints = self._frameworks[endpoint.framework]
            if endpoint in framework_endpoints:
                framework_endpoints.remove(endpoint)

                if not framework_endpoints:
                    # Remove framework entry if there is no more endpoint
                    # from it
                    del self._frameworks[endpoint.framework]

        except (KeyError, ValueError):
            # Ignore the absence of reference in the framework storage
            pass

        # Notify listeners
        if self._listeners:
            for listener in self._listeners[:]:
                try:
                    listener.endpoint_removed(endpoint)

                except Exception as ex:
                    _logger.exception("Error calling listener: %s", ex)

    def lost_framework(self, uid):
        """
        Unregisters all the end points associated to the given framework UID

        :param uid: The UID of a framework
        """
        # Get the end points of this framework
        endpoints = self._frameworks.pop(uid, None)
        if endpoints:
            for endpoint in endpoints:
                with self.__lock:
                    # Remove endpoint from registry
                    try:
                        del self._registry[endpoint.uid]

                    except KeyError:
                        # The endpoint may have been removed by a listener
                        pass

                # Notify listeners
                if self._listeners:
                    for listener in self._listeners[:]:
                        try:
                            listener.endpoint_removed(endpoint)

                        except Exception as ex:
                            _logger.exception("Error calling listener: %s", ex)

    @Validate
    def validate(self, context):
        """
        Component validated
        """
        # Get the framework UID
        self._fw_uid = context.get_property(pelix.framework.FRAMEWORK_UID)

        # We are now validated
        self.__validated = True

    @Invalidate
    def invalidate(self, context):
        """
        Component invalidated: clean up storage
        """
        # Update the validation flag
        self.__validated = False

        # Clean up
        self._fw_uid = None
        self._frameworks.clear()
        self._registry.clear()
<commit_msg>Do not log the endpoints ignored due to their framework UID<commit_after>#!/usr/bin/env python
# -- Content-Encoding: UTF-8 --
"""
Pelix remote services: Imported end points registry

:author: Thomas Calmant
:copyright: Copyright 2013, isandlaTech
:license: Apache License 2.0
:version: 0.1.1
:status: Beta

..

    Copyright 2013 isandlaTech

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

        http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
"""

# Module version
__version_info__ = (0, 1, 1)
__version__ = ".".join(str(x) for x in __version_info__)

# Documentation strings format
__docformat__ = "restructuredtext en"

# ------------------------------------------------------------------------------

# Remote Services constants
import pelix.remote

# iPOPO decorators
from pelix.ipopo.decorators import ComponentFactory, Requires, Provides, \
    Instantiate, Invalidate, Validate, BindField

# Standard library
import logging
import threading

# ------------------------------------------------------------------------------

_logger = logging.getLogger(__name__)

# ------------------------------------------------------------------------------

@ComponentFactory('pelix-remote-imports-registry-factory')
@Provides(pelix.remote.SERVICE_REGISTRY)
@Requires('_listeners', pelix.remote.SERVICE_IMPORT_ENDPOINT_LISTENER,
          aggregate=True, optional=True)
@Instantiate('pelix-remote-imports-registry')
class ImportsRegistry(object):
    """
    Registry of discovered end points. End points are identified by their UID
    """
    def __init__(self):
        """
        Sets up the component
        """
        # Listeners (injected)
        self._listeners = []

        # Framework UID
        self._fw_uid = None

        # Framework UID -> [Endpoints]
        self._frameworks = {}

        # End point UID -> Endpoint
        self._registry = {}

        # Lock
        self.__lock = threading.Lock()

        # Validation flag
        self.__validated = False

    @BindField('_listeners')
    def _bind_listener(self, field, listener, svc_ref):
        """
        New listener bound
        """
        with self.__lock:
            if self.__validated:
                # Late listener
                for endpoint in self._registry.values():
                    try:
                        listener.endpoint_added(endpoint)

                    except Exception as ex:
                        _logger.exception("Error calling listener: %s", ex)

    def add(self, endpoint):
        """
        Registers an end point and notifies listeners. Does nothing if the
        endpoint UID was already known.

        :param endpoint: An ImportedEndpoint object
        :return: True if the end point has been added
        """
        with self.__lock:
            # Check framework UID (avoid to import our own services)
            if endpoint.framework == self._fw_uid:
                return False

            # Check if the end point already exists
            if endpoint.uid in self._registry:
                # Already known end point: do nothing
                _logger.debug("Already known endpoint")
                return False

            # Store the end point
            self._registry[endpoint.uid] = endpoint
            if endpoint.framework:
                self._frameworks.setdefault(endpoint.framework, []) \
                                                            .append(endpoint)

        # Notify listeners (out of lock)
        if self._listeners:
            for listener in self._listeners[:]:
                try:
                    listener.endpoint_added(endpoint)

                except Exception as ex:
                        _logger.exception("Error calling listener: %s", ex)

        return True

    def update(self, uid, new_properties):
        """
        Updates an end point and notifies listeners

        :param uid: The UID of the end point
        :param new_properties: The new properties of the end point
        """
        try:
            with self.__lock:
                # Update the stored end point
                stored_endpoint = self._registry[uid]

                # Replace the stored properties
                old_properties = stored_endpoint.properties.copy()
                stored_endpoint.properties = new_properties.copy()

        except KeyError:
            # Unknown end point: ignore it
            return

        else:
            # Notify listeners
            if self._listeners:
                for listener in self._listeners[:]:
                    try:
                        listener.endpoint_updated(stored_endpoint,
                                                  old_properties)

                    except Exception as ex:
                        _logger.exception("Error calling listener: %s", ex)

    def remove(self, uid):
        """
        Unregisters an end point and notifies listeners

        :param uid: The UID of the end point to unregister
        """
        # Remove the end point from the individual storage
        try:
            endpoint = self._registry.pop(uid)

        except KeyError:
            # Unknown end point
            _logger.warning("Unknown end point UID: %s", uid)
            return

        # Remove it from its framework storage, if any
        try:
            framework_endpoints = self._frameworks[endpoint.framework]
            if endpoint in framework_endpoints:
                framework_endpoints.remove(endpoint)

                if not framework_endpoints:
                    # Remove framework entry if there is no more endpoint
                    # from it
                    del self._frameworks[endpoint.framework]

        except (KeyError, ValueError):
            # Ignore the absence of reference in the framework storage
            pass

        # Notify listeners
        if self._listeners:
            for listener in self._listeners[:]:
                try:
                    listener.endpoint_removed(endpoint)

                except Exception as ex:
                    _logger.exception("Error calling listener: %s", ex)

    def lost_framework(self, uid):
        """
        Unregisters all the end points associated to the given framework UID

        :param uid: The UID of a framework
        """
        # Get the end points of this framework
        endpoints = self._frameworks.pop(uid, None)
        if endpoints:
            for endpoint in endpoints:
                with self.__lock:
                    # Remove endpoint from registry
                    try:
                        del self._registry[endpoint.uid]

                    except KeyError:
                        # The endpoint may have been removed by a listener
                        pass

                # Notify listeners
                if self._listeners:
                    for listener in self._listeners[:]:
                        try:
                            listener.endpoint_removed(endpoint)

                        except Exception as ex:
                            _logger.exception("Error calling listener: %s", ex)

    @Validate
    def validate(self, context):
        """
        Component validated
        """
        # Get the framework UID
        self._fw_uid = context.get_property(pelix.framework.FRAMEWORK_UID)

        # We are now validated
        self.__validated = True

    @Invalidate
    def invalidate(self, context):
        """
        Component invalidated: clean up storage
        """
        # Update the validation flag
        self.__validated = False

        # Clean up
        self._fw_uid = None
        self._frameworks.clear()
        self._registry.clear()
<|endoftext|>


<commit_before>from logger import logger
from perfrunner.helpers.cbmonitor import timeit, with_stats
from perfrunner.helpers.misc import pretty_dict
from perfrunner.helpers.profiler import with_profiles
from perfrunner.tests import PerfTest, TargetIterator

class N1QLTest(PerfTest):

    COLLECTORS = {
        'iostat': False,
        'memory': False,
        'n1ql_latency': True,
        'n1ql_stats': True,
        'secondary_stats': True,
    }

    def load(self, *args):
        """Create two data sets with different key prefixes.

        In order to run the N1QL tests we need to satisfy two contradicting
        requirements:
        * Fields should be changed so that the secondary indexes are being
        updated.
        * Fields remain the same (based on a deterministic random algorithm) so
        that we can query them.

        The following workaround was introduced:
        * 50% of documents are being randomly mutated. These documents are not
        used for queries.
        * 50% of documents remain unchanged. Only these documents are used for
        queries.
        """
        load_settings = self.test_config.load_settings
        load_settings.items //= 2

        iterator = TargetIterator(self.cluster_spec, self.test_config, 'n1ql')
        super().load(settings=load_settings, target_iterator=iterator)
        super().load(settings=load_settings)

    def access_bg(self, *args):
        access_settings = self.test_config.access_settings
        access_settings.items //= 2
        access_settings.n1ql_workers = 0

        super().access_bg(settings=access_settings)

    @with_stats
    @with_profiles
    def access(self, *args):
        self.download_certificate()

        access_settings = self.test_config.access_settings
        access_settings.items //= 2
        access_settings.workers = 0

        iterator = TargetIterator(self.cluster_spec, self.test_config, 'n1ql')

        super().access(settings=access_settings,
                       target_iterator=iterator)

    def store_plans(self):
        logger.info('Storing query plans')

        for i, query in enumerate(self.test_config.access_settings.n1ql_queries):
            plan = self.rest.explain_n1ql_statement(self.query_nodes[0],
                                                    query['statement'])
            with open('query_plan_{}.json'.format(i), 'w') as fh:
                fh.write(pretty_dict(plan))

    def run(self):
        self.load()
        self.wait_for_persistence()

        self.create_indexes()
        self.wait_for_indexing()

        self.store_plans()

        self.access_bg()
        self.access()

        self.report_kpi()

class N1QLLatencyTest(N1QLTest):

    def _report_kpi(self):
        self.reporter.post(
            *self.metrics.query_latency(percentile=90)
        )

class N1QLThroughputTest(N1QLTest):

    def _report_kpi(self):
        self.reporter.post(
            *self.metrics.avg_n1ql_throughput()
        )

class N1QLJoinTest(N1QLThroughputTest):

    ALL_BUCKETS = True

    def load_regular(self, load_settings, target):
        load_settings.items //= 2
        super(N1QLTest, self).load(settings=load_settings,
                                   target_iterator=(target, ))
        target.prefix = 'n1ql'
        super(N1QLTest, self).load(settings=load_settings,
                                   target_iterator=(target, ))

    def load_categories(self, load_settings, target):
        load_settings.items = load_settings.num_categories
        target.prefix = 'n1ql'
        super(N1QLTest, self).load(settings=load_settings,
                                   target_iterator=(target, ))

    def load(self, *args):
        doc_gens = self.test_config.load_settings.doc_gen.split(',')
        for doc_gen, target in zip(doc_gens, self.target_iterator):
            load_settings = self.test_config.load_settings
            load_settings.doc_gen = doc_gen

            if doc_gen == 'ref':
                self.load_categories(load_settings, target)
            else:
                self.load_regular(load_settings, target)

    def access_bg(self, *args):
        doc_gens = self.test_config.load_settings.doc_gen.split(',')
        for doc_gen, target in zip(doc_gens, self.target_iterator):
            if doc_gen == 'ref':
                continue

            access_settings = self.test_config.access_settings
            access_settings.doc_gen = doc_gen
            access_settings.items //= 2
            access_settings.buckets = self.test_config.buckets

            if doc_gen != access_settings.n1ql_gen:
                access_settings.n1ql_workers = 0

            super(N1QLTest, self).access_bg(settings=access_settings,
                                            target_iterator=(target, ))

    @ with_stats
    def access(self, *args):
        super().sleep()

        self.worker_manager.wait_for_workers()

    def _report_kpi(self):
        self.reporter.post(
            *self.metrics.avg_n1ql_throughput()
        )

class N1QLBulkTest(N1QLTest):

    @with_stats
    @timeit
    def access(self, *args):
        statement = self.test_config.access_settings.n1ql_queries[0]['statement']

        self.rest.exec_n1ql_statement(self.query_nodes[0], statement)

    def _report_kpi(self, time_elapsed):
        self.reporter.post(
            *self.metrics.bulk_n1ql_throughput(time_elapsed)
        )

    def run(self):
        self.load()
        self.wait_for_persistence()

        self.create_indexes()
        self.wait_for_indexing()

        self.store_plans()

        time_elapsed = self.access()

        self.report_kpi(time_elapsed)

class N1QLDGMTest(PerfTest):

    COLLECTORS = {
        'n1ql_latency': True,
        'n1ql_stats': True,
        'net': False,
        'secondary_stats': True,
        'secondary_storage_stats': True,
    }

    def load(self, *args):
        PerfTest.load(*args)

    def access_bg(self, *args):
        access_settings = self.test_config.access_settings
        access_settings.n1ql_workers = 0

        PerfTest.access_bg(self, settings=access_settings)

    @with_stats
    @with_profiles
    def access(self, *args):
        access_settings = self.test_config.access_settings
        access_settings.workers = 0

        PerfTest.access(self, settings=access_settings)

    def run(self):
        self.load()
        self.wait_for_persistence()

        self.create_indexes()
        self.wait_for_indexing()

        self.access_bg()
        self.access()

        self.report_kpi()

class N1QLDGMThroughputTest(N1QLDGMTest, N1QLThroughputTest):

    pass

class N1QLDGMLatencyTest(N1QLDGMTest, N1QLLatencyTest):

    pass

class N1QLXattrThroughputTest(N1QLThroughputTest):

    def xattr_load(self, *args, **kwargs):
        iterator = TargetIterator(self.cluster_spec, self.test_config, 'n1ql')
        super().xattr_load()
        super().xattr_load(target_iterator=iterator)

    def run(self):
        self.load()
        self.xattr_load()
        self.wait_for_persistence()

        self.create_indexes()
        self.wait_for_indexing()

        self.store_plans()

        self.access_bg()
        self.access()

        self.report_kpi()

class TpcDsTest(N1QLTest):

    COLLECTORS = {
        'iostat': False,
        'memory': False,
        'n1ql_latency': True,
        'n1ql_stats': True,
        'net': False,
        'secondary_debugstats_index': True,
    }

    def run(self):
        self.import_data()

        self.create_indexes()
        self.wait_for_indexing()

        self.store_plans()

        self.access()

        self.report_kpi()

class TpcDsLatencyTest(TpcDsTest, N1QLLatencyTest):

    pass

class TpcDsThroughputTest(TpcDsTest, N1QLThroughputTest):

    pass

class TpcDsIndexTest(TpcDsTest):

    COLLECTORS = {
        'memory': False,
        'net': False,
        'secondary_debugstats_index': True,
    }

    @with_stats
    @with_profiles
    def create_indexes(self):
        super().create_indexes()
        self.wait_for_indexing()

    def run(self):
        self.import_data()

        self.create_indexes()
<commit_msg>Fix missing required positional argument<commit_after>from logger import logger
from perfrunner.helpers.cbmonitor import timeit, with_stats
from perfrunner.helpers.misc import pretty_dict
from perfrunner.helpers.profiler import with_profiles
from perfrunner.tests import PerfTest, TargetIterator

class N1QLTest(PerfTest):

    COLLECTORS = {
        'iostat': False,
        'memory': False,
        'n1ql_latency': True,
        'n1ql_stats': True,
        'secondary_stats': True,
    }

    def load(self, *args):
        """Create two data sets with different key prefixes.

        In order to run the N1QL tests we need to satisfy two contradicting
        requirements:
        * Fields should be changed so that the secondary indexes are being
        updated.
        * Fields remain the same (based on a deterministic random algorithm) so
        that we can query them.

        The following workaround was introduced:
        * 50% of documents are being randomly mutated. These documents are not
        used for queries.
        * 50% of documents remain unchanged. Only these documents are used for
        queries.
        """
        load_settings = self.test_config.load_settings
        load_settings.items //= 2

        iterator = TargetIterator(self.cluster_spec, self.test_config, 'n1ql')
        super().load(settings=load_settings, target_iterator=iterator)
        super().load(settings=load_settings)

    def access_bg(self, *args):
        access_settings = self.test_config.access_settings
        access_settings.items //= 2
        access_settings.n1ql_workers = 0

        super().access_bg(settings=access_settings)

    @with_stats
    @with_profiles
    def access(self, *args):
        self.download_certificate()

        access_settings = self.test_config.access_settings
        access_settings.items //= 2
        access_settings.workers = 0

        iterator = TargetIterator(self.cluster_spec, self.test_config, 'n1ql')

        super().access(settings=access_settings,
                       target_iterator=iterator)

    def store_plans(self):
        logger.info('Storing query plans')

        for i, query in enumerate(self.test_config.access_settings.n1ql_queries):
            plan = self.rest.explain_n1ql_statement(self.query_nodes[0],
                                                    query['statement'])
            with open('query_plan_{}.json'.format(i), 'w') as fh:
                fh.write(pretty_dict(plan))

    def run(self):
        self.load()
        self.wait_for_persistence()

        self.create_indexes()
        self.wait_for_indexing()

        self.store_plans()

        self.access_bg()
        self.access()

        self.report_kpi()

class N1QLLatencyTest(N1QLTest):

    def _report_kpi(self):
        self.reporter.post(
            *self.metrics.query_latency(percentile=90)
        )

class N1QLThroughputTest(N1QLTest):

    def _report_kpi(self):
        self.reporter.post(
            *self.metrics.avg_n1ql_throughput()
        )

class N1QLJoinTest(N1QLThroughputTest):

    ALL_BUCKETS = True

    def load_regular(self, load_settings, target):
        load_settings.items //= 2
        super(N1QLTest, self).load(settings=load_settings,
                                   target_iterator=(target, ))
        target.prefix = 'n1ql'
        super(N1QLTest, self).load(settings=load_settings,
                                   target_iterator=(target, ))

    def load_categories(self, load_settings, target):
        load_settings.items = load_settings.num_categories
        target.prefix = 'n1ql'
        super(N1QLTest, self).load(settings=load_settings,
                                   target_iterator=(target, ))

    def load(self, *args):
        doc_gens = self.test_config.load_settings.doc_gen.split(',')
        for doc_gen, target in zip(doc_gens, self.target_iterator):
            load_settings = self.test_config.load_settings
            load_settings.doc_gen = doc_gen

            if doc_gen == 'ref':
                self.load_categories(load_settings, target)
            else:
                self.load_regular(load_settings, target)

    def access_bg(self, *args):
        doc_gens = self.test_config.load_settings.doc_gen.split(',')
        for doc_gen, target in zip(doc_gens, self.target_iterator):
            if doc_gen == 'ref':
                continue

            access_settings = self.test_config.access_settings
            access_settings.doc_gen = doc_gen
            access_settings.items //= 2
            access_settings.buckets = self.test_config.buckets

            if doc_gen != access_settings.n1ql_gen:
                access_settings.n1ql_workers = 0

            super(N1QLTest, self).access_bg(settings=access_settings,
                                            target_iterator=(target, ))

    @ with_stats
    def access(self, *args):
        super().sleep()

        self.worker_manager.wait_for_workers()

    def _report_kpi(self):
        self.reporter.post(
            *self.metrics.avg_n1ql_throughput()
        )

class N1QLBulkTest(N1QLTest):

    @with_stats
    @timeit
    def access(self, *args):
        statement = self.test_config.access_settings.n1ql_queries[0]['statement']

        self.rest.exec_n1ql_statement(self.query_nodes[0], statement)

    def _report_kpi(self, time_elapsed):
        self.reporter.post(
            *self.metrics.bulk_n1ql_throughput(time_elapsed)
        )

    def run(self):
        self.load()
        self.wait_for_persistence()

        self.create_indexes()
        self.wait_for_indexing()

        self.store_plans()

        time_elapsed = self.access()

        self.report_kpi(time_elapsed)

class N1QLDGMTest(PerfTest):

    COLLECTORS = {
        'n1ql_latency': True,
        'n1ql_stats': True,
        'net': False,
        'secondary_stats': True,
        'secondary_storage_stats': True,
    }

    def load(self, *args):
        PerfTest.load(self, *args)

    def access_bg(self, *args):
        access_settings = self.test_config.access_settings
        access_settings.n1ql_workers = 0

        PerfTest.access_bg(self, settings=access_settings)

    @with_stats
    @with_profiles
    def access(self, *args):
        access_settings = self.test_config.access_settings
        access_settings.workers = 0

        PerfTest.access(self, settings=access_settings)

    def run(self):
        self.load()
        self.wait_for_persistence()

        self.create_indexes()
        self.wait_for_indexing()

        self.access_bg()
        self.access()

        self.report_kpi()

class N1QLDGMThroughputTest(N1QLDGMTest, N1QLThroughputTest):

    pass

class N1QLDGMLatencyTest(N1QLDGMTest, N1QLLatencyTest):

    pass

class N1QLXattrThroughputTest(N1QLThroughputTest):

    def xattr_load(self, *args, **kwargs):
        iterator = TargetIterator(self.cluster_spec, self.test_config, 'n1ql')
        super().xattr_load()
        super().xattr_load(target_iterator=iterator)

    def run(self):
        self.load()
        self.xattr_load()
        self.wait_for_persistence()

        self.create_indexes()
        self.wait_for_indexing()

        self.store_plans()

        self.access_bg()
        self.access()

        self.report_kpi()

class TpcDsTest(N1QLTest):

    COLLECTORS = {
        'iostat': False,
        'memory': False,
        'n1ql_latency': True,
        'n1ql_stats': True,
        'net': False,
        'secondary_debugstats_index': True,
    }

    def run(self):
        self.import_data()

        self.create_indexes()
        self.wait_for_indexing()

        self.store_plans()

        self.access()

        self.report_kpi()

class TpcDsLatencyTest(TpcDsTest, N1QLLatencyTest):

    pass

class TpcDsThroughputTest(TpcDsTest, N1QLThroughputTest):

    pass

class TpcDsIndexTest(TpcDsTest):

    COLLECTORS = {
        'memory': False,
        'net': False,
        'secondary_debugstats_index': True,
    }

    @with_stats
    @with_profiles
    def create_indexes(self):
        super().create_indexes()
        self.wait_for_indexing()

    def run(self):
        self.import_data()

        self.create_indexes()
<|endoftext|>


<commit_before>#!/opt/local/bin/python
'''
Created on Sep 19, 2012

@author: steve

Tryin' to learn to recognize handwritten digits!

Tricky, for this to work properly, the number of sections needs to be
a perfect square and the root of it needs to be a divisor of the dimension.
Divisor so we fill each section with no leftovers, perf square because
we want to deal with a physical square.
'''

from __future__ import print_function
import neuralnet
import math
import time
import threading

'''
The data we have currently is for 28x28 samples, possible dimensions for
section numbers (and neural net size) are:
4, 16, 49, 196, 784 total sections.
'''
SECTIONS = 49

def _two_dimension(pixels):
    ''' Pixels must be a perfect square. Access by twod[y][x] '''
    assert math.sqrt(len(pixels)) % 1 == 0
    side = int(math.sqrt(len(pixels)))
    twod = []
    for y in range(side):
        row = []
        for x in range(side):
            row.append(pixels[y * side + x])
        twod.append(row)
    return twod

def _section_contains_ink(lst, section_num, sections):
    ''' Get list, section number in question, and total sections,
    return whether any pixel in that section is over the threshold. '''
    sections_per_side = math.sqrt(sections)
    section_dimension = len(lst) / sections_per_side
    assert section_dimension % 1 == 0 and sections_per_side % 1 == 0
    start_row = int(math.floor(section_num / sections_per_side) *
                    section_dimension)
    start_col = int(math.floor(section_num % sections_per_side) *
                    section_dimension)
    ret = []
    for y in range(start_row, int(start_row + section_dimension)):
        for x in range(start_col, int(start_col + section_dimension)):
            ret.append(lst[y][x])
    ret.sort()
    ret.reverse()
    # Change the threshold here
    if ret[0] > 245:
        return True
    return False

def sections_as_ink(values, sections):
    ''' This is how we want to represent the data. Given all the pixel values,
    think of them as sections and return a list of binary values. Only represent
    whether each section contained ink.

    Picture read from left to right, top to bottom, put in linear list. '''
    results = []
    for x in range(sections):
        if _section_contains_ink(values, x, sections):
            results.append(1)
        else:
            results.append(0)
    return results

def _learned(nn, num_sections, count_seen=None):
    ''' Determine if the network knows what's up yet. '''
    goal = .5
    total = 1000
    correct = 0
    with open('./data/test-no-header.csv', 'r') as fh:
        for i in range(total):
            line = fh.readline()
            pixels = [int(x) for x in line[2:].split(',')]
            inputs = sections_as_ink(_two_dimension(pixels), num_sections)
            res = nn.evaluate(inputs)
            print(res)
            # The results is just a list of outputs between -1 and 1.
            # Interpret as the index of the greatest output is the guess.
            n = max(res)
            ans = [i for i, j in enumerate(res) if j == n]
            #print('response:  %s\tlist: %s\nexpected: %s' %(ans, res, line[0]))
            if int(line[0]) == ans[0]:
                correct += 1
    percent = float(correct) / float(total)
    print('%d out of %d. %f percent.' %(correct, total, percent))
    return percent >= goal

def _draw_things(screen, pixels, inked):
    ''' Just give a visual to verify data makes sense. '''
    # Paint it black
    screen.fill((0,0,0))
    # Draw the pixels
    ink = [(p,p,p) for p in pixels]
    for y in range(28):
        for x in range(28):
            pygame.draw.line(screen, ink[y * 28 + x], (x, y), (x, y))

    # Draw the sections
    sections_per_side = int(math.sqrt(SECTIONS))
    section_dimensions = 28 / sections_per_side
    for y in range(sections_per_side):
        for x in range(sections_per_side):
            ink_color = [inked[y * sections_per_side + x] * 255] * 3
            start_x = section_dimensions * x +56
            start_y = section_dimensions * y
            pygame.draw.rect(screen,
                             ink_color,
                             (start_x, start_y,
                              section_dimensions, section_dimensions),
                             0)
    pygame.display.flip()

def train(nnet, num_sections, screen):
    ''' With chunks of data from the file, train the network. '''
    file_count = 0
    while not _learned(nnet, num_sections):
        print('\n***\nGone through the file %d times.\n***\n' %file_count)
        with open('./data/train-no-header.csv', 'r') as fh:
            data = []
            # Not at all resource friendly. If issues, use generator.
            for line_count, line in enumerate(fh.readlines()):
                # First character of each line is the answer digit.
                correct_index = int(line[0])
                ans = [1 if correct_index == i else -1 for i in range(10)]
                # Pixels are a 0-255 color rating, comma separated, for each
                # character in the rest of the line. 1 character -> 1 pixel.
                pixels = [int(x) for x in line[2:].split(',')]
                inputs = sections_as_ink(_two_dimension(pixels), num_sections)
                if screen and line_count % 100 == 0:
                    t = threading.Thread(target=_draw_things,
                                         args=(screen, pixels, inputs))
                    t.start()

                data.append((inputs, ans))
                # When we read in a good chunk of data, train.
                if line_count % 200 == 0:
                    nn.train_network(data,
                                     change_rate=0.2,
                                     momentum=0.1,
                                     iters=500)
                    if _learned(nnet,
                                num_sections,
                                (file_count * line_count + line_count)):
                        print('Made it!!')
                        import sys
                        sys.exit(0)
                    data = []
        file_count += 1

if __name__ == '__main__':
    visual_wanted = True
    if visual_wanted:
        try:
            import pygame
        except ImportError:
            print('Cannot do visualizations without PyGame.')
            screen = None
        else:
            pygame.init()
            screen = pygame.display.set_mode((84, 28))
    else:
        screen = None

    num_hidden = SECTIONS
    nn = neuralnet.NeuralNetwork(SECTIONS, num_hidden, 10)

    train(nn, SECTIONS, screen)
<commit_msg>Train higher to help results not all be so low.<commit_after>#!/opt/local/bin/python
'''
Created on Sep 19, 2012

@author: steve

Tryin' to learn to recognize handwritten digits!

Tricky, for this to work properly, the number of sections needs to be
a perfect square and the root of it needs to be a divisor of the dimension.
Divisor so we fill each section with no leftovers, perf square because
we want to deal with a physical square.
'''

from __future__ import print_function
import neuralnet
import math
import time
import threading

'''
The data we have currently is for 28x28 samples, possible dimensions for
section numbers (and neural net size) are:
4, 16, 49, 196, 784 total sections.
'''
SECTIONS = 49

def _two_dimension(pixels):
    ''' Pixels must be a perfect square. Access by twod[y][x] '''
    assert math.sqrt(len(pixels)) % 1 == 0
    side = int(math.sqrt(len(pixels)))
    twod = []
    for y in range(side):
        row = []
        for x in range(side):
            row.append(pixels[y * side + x])
        twod.append(row)
    return twod

def _section_contains_ink(lst, section_num, sections):
    ''' Get list, section number in question, and total sections,
    return whether any pixel in that section is over the threshold. '''
    sections_per_side = math.sqrt(sections)
    section_dimension = len(lst) / sections_per_side
    assert section_dimension % 1 == 0 and sections_per_side % 1 == 0
    start_row = int(math.floor(section_num / sections_per_side) *
                    section_dimension)
    start_col = int(math.floor(section_num % sections_per_side) *
                    section_dimension)
    ret = []
    for y in range(start_row, int(start_row + section_dimension)):
        for x in range(start_col, int(start_col + section_dimension)):
            ret.append(lst[y][x])
    ret.sort()
    ret.reverse()
    # Change the threshold here
    if ret[0] > 245:
        return True
    return False

def sections_as_ink(values, sections):
    ''' This is how we want to represent the data. Given all the pixel values,
    think of them as sections and return a list of binary values. Only represent
    whether each section contained ink.

    Picture read from left to right, top to bottom, put in linear list. '''
    results = []
    for x in range(sections):
        if _section_contains_ink(values, x, sections):
            results.append(1)
        else:
            results.append(0)
    return results

def _learned(nn, num_sections, count_seen=None):
    ''' Determine if the network knows what's up yet. '''
    goal = .5
    total = 1000
    correct = 0
    with open('./data/test-no-header.csv', 'r') as fh:
        for i in range(total):
            line = fh.readline()
            pixels = [int(x) for x in line[2:].split(',')]
            inputs = sections_as_ink(_two_dimension(pixels), num_sections)
            res = nn.evaluate(inputs)
            print(res)
            # The results is just a list of outputs between -1 and 1.
            # Interpret as the index of the greatest output is the guess.
            ans = res.index(max(res))
            if int(line[0]) == ans:
                correct += 1
    percent = float(correct) / float(total)
    print('%d out of %d. %f percent.' %(correct, total, percent))
    return percent >= goal

def _draw_things(screen, pixels, inked):
    ''' Just give a visual to verify data makes sense. '''
    # Paint it black
    screen.fill((0,0,0))
    # Draw the pixels
    ink = [(p,p,p) for p in pixels]
    for y in range(28):
        for x in range(28):
            pygame.draw.line(screen, ink[y * 28 + x], (x, y), (x, y))

    # Draw the sections
    sections_per_side = int(math.sqrt(SECTIONS))
    section_dimensions = 28 / sections_per_side
    for y in range(sections_per_side):
        for x in range(sections_per_side):
            ink_color = [inked[y * sections_per_side + x] * 255] * 3
            start_x = section_dimensions * x +56
            start_y = section_dimensions * y
            pygame.draw.rect(screen,
                             ink_color,
                             (start_x, start_y,
                              section_dimensions, section_dimensions),
                             0)
    pygame.display.flip()

def train(nnet, num_sections, screen):
    ''' With chunks of data from the file, train the network. '''
    file_count = 0
    while not _learned(nnet, num_sections):
        print('\n***\nGone through the file %d times.\n***\n' %file_count)
        with open('./data/train-no-header.csv', 'r') as fh:
            data = []
            # Not at all resource friendly. If issues, use generator.
            for line_count, line in enumerate(fh.readlines()):
                # First character of each line is the answer digit.
                correct_index = int(line[0])
                ans = [3 if correct_index == i else -1 for i in range(10)]
                # Pixels are a 0-255 color rating, comma separated, for each
                # character in the rest of the line. 1 character -> 1 pixel.
                pixels = [int(x) for x in line[2:].split(',')]
                inputs = sections_as_ink(_two_dimension(pixels), num_sections)
                if screen and line_count % 100 == 0:
                    t = threading.Thread(target=_draw_things,
                                         args=(screen, pixels, inputs))
                    t.start()

                data.append((inputs, ans))
                # When we read in a good chunk of data, train.
                if line_count % 200 == 0:
                    nn.train_network(data,
                                     change_rate=0.2,
                                     momentum=0.1,
                                     iters=500)
                    if _learned(nnet,
                                num_sections,
                                (file_count * line_count + line_count)):
                        print('Made it!!')
                        import sys
                        sys.exit(0)
                    data = []
        file_count += 1

if __name__ == '__main__':
    visual_wanted = True
    if visual_wanted:
        try:
            import pygame
        except ImportError:
            print('Cannot do visualizations without PyGame.')
            screen = None
        else:
            pygame.init()
            screen = pygame.display.set_mode((84, 28))
    else:
        screen = None

    num_hidden = SECTIONS
    nn = neuralnet.NeuralNetwork(SECTIONS, num_hidden, 10)

    train(nn, SECTIONS, screen)
<|endoftext|>


<commit_before># encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Author: Kyle Lahnakoski (kyle@lahnakoski.com)
#
from __future__ import absolute_import, division, unicode_literals

import os
import platform
import subprocess

from mo_dots import set_default, wrap, Null
from mo_logs import Log, strings
from mo_logs.exceptions import Except
from mo_threads.lock import Lock
from mo_threads.queues import Queue
from mo_threads.signals import Signal
from mo_threads.threads import THREAD_STOP, Thread
from mo_threads.till import Till
from mo_times import Timer

DEBUG = True

class Process(object):
    def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1):
        self.name = name
        self.service_stopped = Signal("stopped signal for " + strings.quote(name))
        self.stdin = Queue("stdin for process " + strings.quote(name), silent=True)
        self.stdout = Queue("stdout for process " + strings.quote(name), silent=True)
        self.stderr = Queue("stderr for process " + strings.quote(name), silent=True)

        try:
            if cwd == None:
                cwd = os.getcwd()
            else:
                cwd = str(cwd)

            self.debug = debug or DEBUG
            self.service = service = subprocess.Popen(
                [str(p) for p in params],
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                bufsize=bufsize,
                cwd=cwd,
                env={str(k): str(v) for k, v in set_default(env, os.environ).items()},
                shell=shell
            )

            self.please_stop = Signal()
            self.please_stop.then(self._kill)
            self.child_locker = Lock()
            self.children = [
                Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self),
                Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self),
                Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self),
                Thread.run(self.name + " waiter", self._monitor, parent_thread=self),
            ]
        except Exception as e:
            Log.error("Can not call", e)

        self.debug and Log.note("{{process}} START: {{command}}", process=self.name, command=" ".join(map(strings.quote, params)))

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.join(raise_on_error=True)

    def stop(self):
        self.stdin.add(THREAD_STOP)  # ONE MORE SEND
        self.please_stop.go()

    def join(self, raise_on_error=False):
        self.service_stopped.wait()
        with self.child_locker:
            child_threads, self.children = self.children, []
        for c in child_threads:
            c.join()
        if raise_on_error and self.returncode != 0:
            Log.error(
                "{{process}} FAIL: returncode={{code}}\n{{stderr}}",
                process=self.name,
                code=self.service.returncode,
                stderr=list(self.stderr)
            )
        return self

    def remove_child(self, child):
        with self.child_locker:
            try:
                self.children.remove(child)
            except Exception:
                pass

    @property
    def pid(self):
        return self.service.pid

    @property
    def returncode(self):
        return self.service.returncode

    def _monitor(self, please_stop):
        with Timer(self.name):
            self.service.wait()
            self.debug and Log.note("{{process}} STOP: returncode={{returncode}}", process=self.name, returncode=self.service.returncode)
            self.service_stopped.go()
            please_stop.go()

    def _reader(self, name, pipe, receive, please_stop):
        try:
            while not please_stop and self.service.returncode is None:
                line = to_text(pipe.readline().rstrip())
                if line:
                    receive.add(line)
                    self.debug and Log.note("{{process}} ({{name}}): {{line}}", name=name, process=self.name, line=line)
                else:
                    (Till(seconds=1) | please_stop).wait()

            # GRAB A FEW MORE LINES
            max = 100
            while max:
                try:
                    line = to_text(pipe.readline().rstrip())
                    if line:
                        max = 100
                        receive.add(line)
                        self.debug and Log.note("RESIDUE: {{process}} ({{name}}): {{line}}", name=name, process=self.name, line=line)
                    else:
                        max -= 1
                except Exception:
                    break
        finally:
            pipe.close()
            receive.add(THREAD_STOP)
        self.debug and Log.note("{{process}} ({{name}} is closed)", name=name, process=self.name)

        receive.add(THREAD_STOP)

    def _writer(self, pipe, send, please_stop):
        while not please_stop:
            line = send.pop(till=please_stop)
            if line is THREAD_STOP:
                please_stop.go()
                break
            elif line is None:
                continue

            self.debug and Log.note("{{process}} (stdin): {{line}}", process=self.name, line=line.rstrip())
            pipe.write(line.encode('utf8') + b"\n")
            pipe.flush()

    def _kill(self):
        try:
            self.service.kill()
            Log.note("Service was successfully terminated.")
        except Exception as e:
            ee = Except.wrap(e)
            if 'The operation completed successfully' in ee:
                return
            if 'No such process' in ee:
                return

            Log.warning("Failure to kill process {{process|quote}}", process=self.name, cause=ee)

WINDOWS_ESCAPE_DCT = {
    u"%": u"%%",
    u"&": u"^&",
    u"\\": u"^\\",
    u"<": u"^<",
    u">": u"^>",
    u"^": u"^^",
    u"|": u"^|",
    u"\t": u"^\t",
    u"\n": u"^\n",
    u"\r": u"^\r",
    u" ": u"^ ",
}

PROMPT = "READY_FOR_MORE"

if "windows" in platform.system().lower():
    # def cmd_escape(v):
    #     return "".join(WINDOWS_ESCAPE_DCT.get(c, c) for c in v)
    cmd_escape = strings.quote

    def set_prompt():
        return "prompt "+PROMPT+"$g"

    def cmd():
        return "%windir%\system32\cmd.exe"

    def to_text(value):
        return value.decode("latin1")

else:
    cmd_escape = strings.quote

    def set_prompt():
        return "set prompt="+cmd_escape(PROMPT+">")

    def cmd():
        return "bash"

    def to_text(value):
        return value.decode("latin1")

class Command(object):
    """
    FASTER Process CLASS - OPENS A COMMAND_LINE APP (CMD on windows) AND KEEPS IT OPEN FOR MULTIPLE COMMANDS
    EACH WORKING DIRECTORY WILL HAVE ITS OWN PROCESS, MULTIPLE PROCESSES WILL OPEN FOR THE SAME DIR IF MULTIPLE
    THREADS ARE REQUESTING Commands
    """

    available_locker = Lock("cmd lock")
    available_process = {}

    def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1):
        shell = True
        self.name=name
        self.key = (cwd, wrap(env), debug, shell)
        self.stdout = Queue("stdout for "+name)
        self.stderr = Queue("stderr for "+name)

        with Command.available_locker:
            avail = Command.available_process.setdefault(self.key, [])
            if not avail:
                self.process = Process("command shell", [cmd()], cwd, env, debug, shell, bufsize)
                self.process.stdin.add(set_prompt())
                self.process.stdin.add("echo %errorlevel%")
                _wait_for_start(self.process.stdout, Null)
            else:
                self.process = avail.pop()

        self.process.stdin.add(" ".join(cmd_escape(p) for p in params))
        self.process.stdin.add("echo %errorlevel%")
        self.stdout_thread = Thread.run("", self._stream_relay, self.process.stdout, self.stdout)
        self.stderr_thread = Thread.run("", self._stream_relay, self.process.stderr, self.stderr)
        self.returncode = None

    def join(self, raise_on_error=False, till=None):
        try:
            try:
                # WAIT FOR COMMAND LINE RESPONSE ON stdout
                self.stdout_thread.join()
            except Exception as e:
                Log.error("unexpected problem processing stdout", cause=e)

            try:
                self.stderr_thread.please_stop.go()
                self.stderr_thread.join()
            except Exception as e:
                Log.error("unexpected problem processing stderr", cause=e)

            if raise_on_error and self.returncode != 0:
                Log.error(
                    "{{process}} FAIL: returncode={{code}}\n{{stderr}}",
                    process=self.name,
                    code=self.returncode,
                    stderr=list(self.stderr)
                )
            return self
        finally:
            with Command.available_locker:
                Command.available_process[self.key].append(self.process)

    def _stream_relay(self, source, destination, please_stop=None):
        """
        :param source:
        :param destination:
        :param error: Throw error if line shows up
        :param please_stop:
        :return:
        """
        prompt_count = 0
        prompt = PROMPT + ">"
        line_count = 0

        while not please_stop:
            value = source.pop(till=please_stop)
            if value is None:
                destination.add(THREAD_STOP)
                return
            elif value is THREAD_STOP:
                destination.add(THREAD_STOP)
                return
            elif line_count==0 and "is not recognized as an internal or external command" in value:
                Log.error("Problem with command: {{desc}}", desc=value)
            elif value.startswith(prompt):
                if prompt_count:
                    # GET THE ERROR LEVEL
                    self.returncode = int(source.pop(till=please_stop))
                    destination.add(THREAD_STOP)
                    return
                else:
                    prompt_count += 1
            else:
                line_count += 1
                destination.add(value)

def _wait_for_start(source, destination):
    prompt = PROMPT + ">"

    while True:
        value = source.pop()
        if value.startswith(prompt):
            # GET THE ERROR LEVEL
            returncode = int(source.pop())
            destination.add(THREAD_STOP)
            return
        destination.add(value)
<commit_msg>fix logging<commit_after># encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Author: Kyle Lahnakoski (kyle@lahnakoski.com)
#
from __future__ import absolute_import, division, unicode_literals

import os
import platform
import subprocess

from mo_dots import set_default, wrap, Null
from mo_logs import Log, strings
from mo_logs.exceptions import Except
from mo_threads.lock import Lock
from mo_threads.queues import Queue
from mo_threads.signals import Signal
from mo_threads.threads import THREAD_STOP, Thread
from mo_threads.till import Till
from mo_times import Timer

DEBUG = True

class Process(object):
    def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1):
        self.name = name
        self.service_stopped = Signal("stopped signal for " + strings.quote(name))
        self.stdin = Queue("stdin for process " + strings.quote(name), silent=True)
        self.stdout = Queue("stdout for process " + strings.quote(name), silent=True)
        self.stderr = Queue("stderr for process " + strings.quote(name), silent=True)

        try:
            if cwd == None:
                cwd = os.getcwd()
            else:
                cwd = str(cwd)

            self.debug = debug or DEBUG
            self.service = service = subprocess.Popen(
                [str(p) for p in params],
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                bufsize=bufsize,
                cwd=cwd,
                env={str(k): str(v) for k, v in set_default(env, os.environ).items()},
                shell=shell
            )

            self.please_stop = Signal()
            self.please_stop.then(self._kill)
            self.child_locker = Lock()
            self.children = [
                Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self),
                Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self),
                Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self),
                Thread.run(self.name + " waiter", self._monitor, parent_thread=self),
            ]
        except Exception as e:
            Log.error("Can not call", e)

        self.debug and Log.note("{{process}} START: {{command}}", process=self.name, command=" ".join(map(strings.quote, params)))

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.join(raise_on_error=True)

    def stop(self):
        self.stdin.add(THREAD_STOP)  # ONE MORE SEND
        self.please_stop.go()

    def join(self, raise_on_error=False):
        self.service_stopped.wait()
        with self.child_locker:
            child_threads, self.children = self.children, []
        for c in child_threads:
            c.join()
        if raise_on_error and self.returncode != 0:
            Log.error(
                "{{process}} FAIL: returncode={{code}}\n{{stderr}}",
                process=self.name,
                code=self.service.returncode,
                stderr=list(self.stderr)
            )
        return self

    def remove_child(self, child):
        with self.child_locker:
            try:
                self.children.remove(child)
            except Exception:
                pass

    @property
    def pid(self):
        return self.service.pid

    @property
    def returncode(self):
        return self.service.returncode

    def _monitor(self, please_stop):
        with Timer(self.name):
            self.service.wait()
            self.debug and Log.note("{{process}} STOP: returncode={{returncode}}", process=self.name, returncode=self.service.returncode)
            self.service_stopped.go()
            please_stop.go()

    def _reader(self, name, pipe, receive, please_stop):
        try:
            while not please_stop and self.service.returncode is None:
                line = to_text(pipe.readline().rstrip())
                if line:
                    receive.add(line)
                    self.debug and Log.note("{{process}} ({{name}}): {{line}}", name=name, process=self.name, line=line)
                else:
                    (Till(seconds=1) | please_stop).wait()

            # GRAB A FEW MORE LINES
            max = 100
            while max:
                try:
                    line = to_text(pipe.readline().rstrip())
                    if line:
                        max = 100
                        receive.add(line)
                        self.debug and Log.note("{{process}} RESIDUE: ({{name}}): {{line}}", name=name, process=self.name, line=line)
                    else:
                        max -= 1
                except Exception:
                    break
        finally:
            pipe.close()
            receive.add(THREAD_STOP)
        self.debug and Log.note("{{process}} ({{name}} is closed)", name=name, process=self.name)

        receive.add(THREAD_STOP)

    def _writer(self, pipe, send, please_stop):
        while not please_stop:
            line = send.pop(till=please_stop)
            if line is THREAD_STOP:
                please_stop.go()
                break
            elif line is None:
                continue

            self.debug and Log.note("{{process}} (stdin): {{line}}", process=self.name, line=line.rstrip())
            pipe.write(line.encode('utf8') + b"\n")
            pipe.flush()

    def _kill(self):
        try:
            self.service.kill()
            Log.note("Service was successfully terminated.")
        except Exception as e:
            ee = Except.wrap(e)
            if 'The operation completed successfully' in ee:
                return
            if 'No such process' in ee:
                return

            Log.warning("Failure to kill process {{process|quote}}", process=self.name, cause=ee)

WINDOWS_ESCAPE_DCT = {
    u"%": u"%%",
    u"&": u"^&",
    u"\\": u"^\\",
    u"<": u"^<",
    u">": u"^>",
    u"^": u"^^",
    u"|": u"^|",
    u"\t": u"^\t",
    u"\n": u"^\n",
    u"\r": u"^\r",
    u" ": u"^ ",
}

PROMPT = "READY_FOR_MORE"

if "windows" in platform.system().lower():
    # def cmd_escape(v):
    #     return "".join(WINDOWS_ESCAPE_DCT.get(c, c) for c in v)
    cmd_escape = strings.quote

    def set_prompt():
        return "prompt "+PROMPT+"$g"

    def cmd():
        return "%windir%\system32\cmd.exe"

    def to_text(value):
        return value.decode("latin1")

else:
    cmd_escape = strings.quote

    def set_prompt():
        return "set prompt="+cmd_escape(PROMPT+">")

    def cmd():
        return "bash"

    def to_text(value):
        return value.decode("latin1")

class Command(object):
    """
    FASTER Process CLASS - OPENS A COMMAND_LINE APP (CMD on windows) AND KEEPS IT OPEN FOR MULTIPLE COMMANDS
    EACH WORKING DIRECTORY WILL HAVE ITS OWN PROCESS, MULTIPLE PROCESSES WILL OPEN FOR THE SAME DIR IF MULTIPLE
    THREADS ARE REQUESTING Commands
    """

    available_locker = Lock("cmd lock")
    available_process = {}

    def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1):
        shell = True
        self.name=name
        self.key = (cwd, wrap(env), debug, shell)
        self.stdout = Queue("stdout for "+name)
        self.stderr = Queue("stderr for "+name)

        with Command.available_locker:
            avail = Command.available_process.setdefault(self.key, [])
            if not avail:
                self.process = Process("command shell", [cmd()], cwd, env, debug, shell, bufsize)
                self.process.stdin.add(set_prompt())
                self.process.stdin.add("echo %errorlevel%")
                _wait_for_start(self.process.stdout, Null)
            else:
                self.process = avail.pop()

        self.process.stdin.add(" ".join(cmd_escape(p) for p in params))
        self.process.stdin.add("echo %errorlevel%")
        self.stdout_thread = Thread.run("", self._stream_relay, self.process.stdout, self.stdout)
        self.stderr_thread = Thread.run("", self._stream_relay, self.process.stderr, self.stderr)
        self.returncode = None

    def join(self, raise_on_error=False, till=None):
        try:
            try:
                # WAIT FOR COMMAND LINE RESPONSE ON stdout
                self.stdout_thread.join()
            except Exception as e:
                Log.error("unexpected problem processing stdout", cause=e)

            try:
                self.stderr_thread.please_stop.go()
                self.stderr_thread.join()
            except Exception as e:
                Log.error("unexpected problem processing stderr", cause=e)

            if raise_on_error and self.returncode != 0:
                Log.error(
                    "{{process}} FAIL: returncode={{code}}\n{{stderr}}",
                    process=self.name,
                    code=self.returncode,
                    stderr=list(self.stderr)
                )
            return self
        finally:
            with Command.available_locker:
                Command.available_process[self.key].append(self.process)

    def _stream_relay(self, source, destination, please_stop=None):
        """
        :param source:
        :param destination:
        :param error: Throw error if line shows up
        :param please_stop:
        :return:
        """
        prompt_count = 0
        prompt = PROMPT + ">"
        line_count = 0

        while not please_stop:
            value = source.pop(till=please_stop)
            if value is None:
                destination.add(THREAD_STOP)
                return
            elif value is THREAD_STOP:
                destination.add(THREAD_STOP)
                return
            elif line_count==0 and "is not recognized as an internal or external command" in value:
                Log.error("Problem with command: {{desc}}", desc=value)
            elif value.startswith(prompt):
                if prompt_count:
                    # GET THE ERROR LEVEL
                    self.returncode = int(source.pop(till=please_stop))
                    destination.add(THREAD_STOP)
                    return
                else:
                    prompt_count += 1
            else:
                line_count += 1
                destination.add(value)

def _wait_for_start(source, destination):
    prompt = PROMPT + ">"

    while True:
        value = source.pop()
        if value.startswith(prompt):
            # GET THE ERROR LEVEL
            returncode = int(source.pop())
            destination.add(THREAD_STOP)
            return
        destination.add(value)
<|endoftext|>


<commit_before>import json
import boto3
from moto.core.exceptions import InvalidNextTokenException
from moto.core.models import ConfigQueryModel
from moto.iam import iam_backends

class RoleConfigQuery(ConfigQueryModel):
    def list_config_service_resources(
        self,
        resource_ids,
        resource_name,
        limit,
        next_token,
        backend_region=None,
        resource_region=None,
        aggregator=None,
    ):
        # IAM roles are "global" and aren't assigned into any availability zone
        # The resource ID is a AWS-assigned random string like "AROA0BSVNSZKXVHS00SBJ"
        # The resource name is a user-assigned string like "MyDevelopmentAdminRole"
        # Stored in moto backend with the AWS-assigned random string like "AROA0BSVNSZKXVHS00SBJ"

        # Grab roles from backend; need the full values since names and id's are different
        role_list = list(self.backends["global"].roles.values())

        if not role_list:
            return [], None

        # Filter by resource name or ids
        if resource_name or resource_ids:
            filtered_roles = []
            # resource_name takes precedence over resource_ids
            if resource_name:
                for role in role_list:
                    if role.name == resource_name:
                        filtered_roles = [role]
                        break
                # but if both are passed, it must be a subset
                if filtered_roles and resource_ids:
                    if filtered_roles[0].id not in resource_ids:
                        return [], None
            else:
                for role in role_list:
                    if role.id in resource_ids:
                        filtered_roles.append(role)

            # Filtered roles are now the subject for the listing
            role_list = filtered_roles

        if aggregator:
            # IAM is a little special; Roles are created in us-east-1 (which AWS calls the "global" region)
            # However, the resource will return in the aggregator (in duplicate) for each region in the aggregator
            # Therefore, we'll need to find out the regions where the aggregators are running, and then duplicate the resource there

            # In practice, it looks like AWS will only duplicate these resources if you've "used" any roles in the region, but since
            # we can't really tell if this has happened in moto, we'll just bind this to the regions in your aggregator
            aggregated_regions = []
            aggregator_sources = aggregator.get(
                "account_aggregation_sources"
            ) or aggregator.get("organization_aggregation_source")
            for source in aggregator_sources:
                source_dict = source.__dict__
                if source_dict["all_aws_regions"]:
                    aggregated_regions = boto3.Session().get_available_regions("config")
                    break
                for region in source_dict["aws_regions"]:
                    aggregated_regions.append(region)

            duplicate_role_list = []
            for region in list(set(aggregated_regions)):
                for role in role_list:
                    duplicate_role_list.append(
                        {
                            "_id": "{}{}".format(
                                role.id, region
                            ),  # this is only for sorting, isn't returned outside of this functin
                            "type": "AWS::IAM::Role",
                            "id": role.id,
                            "name": role.name,
                            "region": region,
                        }
                    )

            # Pagination logic, sort by role id
            sorted_roles = sorted(duplicate_role_list, key=lambda role: role["_id"])
        else:
            # Non-aggregated queries are in the else block, and we can treat these like a normal config resource
            # Pagination logic, sort by role id
            sorted_roles = sorted(role_list, key=lambda role: role.id)

        new_token = None

        # Get the start:
        if not next_token:
            start = 0
        else:
            try:
                # Find the index of the next
                start = next(
                    index
                    for (index, r) in enumerate(sorted_roles)
                    if next_token == (r["_id"] if aggregator else r.id)
                )
            except StopIteration:
                raise InvalidNextTokenException()

        # Get the list of items to collect:
        role_list = sorted_roles[start : (start + limit)]

        if len(sorted_roles) > (start + limit):
            record = sorted_roles[start + limit]
            new_token = record["_id"] if aggregator else record.id

        return (
            [
                {
                    "type": "AWS::IAM::Role",
                    "id": role["id"] if aggregator else role.id,
                    "name": role["name"] if aggregator else role.name,
                    "region": role["region"] if aggregator else "global",
                }
                for role in role_list
            ],
            new_token,
        )

    def get_config_resource(
        self, resource_id, resource_name=None, backend_region=None, resource_region=None
    ):

        role = self.backends["global"].roles.get(resource_id, {})

        if not role:
            return

        if resource_name and role.name != resource_name:
            return

        # Format the role to the AWS Config format:
        config_data = role.to_config_dict()

        # The 'configuration' field is also a JSON string:
        config_data["configuration"] = json.dumps(config_data["configuration"])

        # Supplementary config need all values converted to JSON strings if they are not strings already:
        for field, value in config_data["supplementaryConfiguration"].items():
            if not isinstance(value, str):
                config_data["supplementaryConfiguration"][field] = json.dumps(value)

        return config_data

class PolicyConfigQuery(ConfigQueryModel):
    def list_config_service_resources(
        self,
        resource_ids,
        resource_name,
        limit,
        next_token,
        backend_region=None,
        resource_region=None,
        aggregator=None,
    ):
        # IAM policies are "global" and aren't assigned into any availability zone
        # The resource ID is a AWS-assigned random string like "ANPA0BSVNSZK00SJSPVUJ"
        # The resource name is a user-assigned string like "my-development-policy"
        # Stored in moto backend with the arn like "arn:aws:iam::123456789012:policy/my-development-policy"

        policy_list = list(self.backends["global"].managed_policies.values())

        # We don't want to include AWS Managed Policies. This technically needs to
        # respect the configuration recorder's 'includeGlobalResourceTypes' setting,
        # but it's default set be default, and moto's config doesn't yet support
        # custom configuration recorders, we'll just behave as default.
        policy_list = list(
            filter(
                lambda policy: not policy.arn.startswith("arn:aws:iam::aws"),
                policy_list,
            )
        )

        if not policy_list:
            return [], None

        # Filter by resource name or ids
        if resource_name or resource_ids:
            filtered_policies = []
            # resource_name takes precedence over resource_ids
            if resource_name:
                for policy in policy_list:
                    if policy.name == resource_name:
                        filtered_policies = [policy]
                        break
                # but if both are passed, it must be a subset
                if filtered_policies and resource_ids:
                    if filtered_policies[0].id not in resource_ids:
                        return [], None

            else:
                for policy in policy_list:
                    if policy.id in resource_ids:
                        filtered_policies.append(policy)

            # Filtered roles are now the subject for the listing
            policy_list = filtered_policies

        if aggregator:
            # IAM is a little special; Policies are created in us-east-1 (which AWS calls the "global" region)
            # However, the resource will return in the aggregator (in duplicate) for each region in the aggregator
            # Therefore, we'll need to find out the regions where the aggregators are running, and then duplicate the resource there

            # In practice, it looks like AWS will only duplicate these resources if you've "used" any policies in the region, but since
            # we can't really tell if this has happened in moto, we'll just bind this to the regions in your aggregator
            aggregated_regions = []
            aggregator_sources = aggregator.get(
                "account_aggregation_sources"
            ) or aggregator.get("organization_aggregation_source")
            for source in aggregator_sources:
                source_dict = source.__dict__
                if source_dict["all_aws_regions"]:
                    aggregated_regions = boto3.Session().get_available_regions("config")
                    break
                for region in source_dict["aws_regions"]:
                    aggregated_regions.append(region)

            duplicate_policy_list = []
            for region in list(set(aggregated_regions)):
                for policy in policy_list:
                    duplicate_policy_list.append(
                        {
                            "_id": "{}{}".format(
                                policy.id, region
                            ),  # this is only for sorting, isn't returned outside of this functin
                            "type": "AWS::IAM::Policy",
                            "id": policy.id,
                            "name": policy.name,
                            "region": region,
                        }
                    )

            # Pagination logic, sort by role id
            sorted_policies = sorted(
                duplicate_policy_list, key=lambda policy: policy["_id"]
            )

        else:
            # Non-aggregated queries are in the else block, and we can treat these like a normal config resource
            # Pagination logic, sort by role id
            sorted_policies = sorted(policy_list, key=lambda role: role.id)

        new_token = None

        # Get the start:
        if not next_token:
            start = 0
        else:
            try:
                # Find the index of the next
                start = next(
                    index
                    for (index, p) in enumerate(sorted_policies)
                    if next_token == (p["_id"] if aggregator else p.id)
                )
            except StopIteration:
                raise InvalidNextTokenException()

        # Get the list of items to collect:
        policy_list = sorted_policies[start : (start + limit)]

        if len(sorted_policies) > (start + limit):
            record = sorted_policies[start + limit]
            new_token = record["_id"] if aggregator else record.id

        return (
            [
                {
                    "type": "AWS::IAM::Policy",
                    "id": policy["id"] if aggregator else policy.id,
                    "name": policy["name"] if aggregator else policy.name,
                    "region": policy["region"] if aggregator else "global",
                }
                for policy in policy_list
            ],
            new_token,
        )

    def get_config_resource(
        self, resource_id, resource_name=None, backend_region=None, resource_region=None
    ):
        # policies are listed in the backend as arns, but we have to accept the PolicyID as the resource_id
        # we'll make a really crude search for it
        policy = None
        for arn in self.backends["global"].managed_policies.keys():
            policy_candidate = self.backends["global"].managed_policies[arn]
            if policy_candidate.id == resource_id:
                policy = policy_candidate
                break

        if not policy:
            return

        if resource_name and policy.name != resource_name:
            return

        # Format the policy to the AWS Config format:
        config_data = policy.to_config_dict()

        # The 'configuration' field is also a JSON string:
        config_data["configuration"] = json.dumps(config_data["configuration"])

        # Supplementary config need all values converted to JSON strings if they are not strings already:
        for field, value in config_data["supplementaryConfiguration"].items():
            if not isinstance(value, str):
                config_data["supplementaryConfiguration"][field] = json.dumps(value)

        return config_data

role_config_query = RoleConfigQuery(iam_backends)
policy_config_query = PolicyConfigQuery(iam_backends)
<commit_msg>use get instead of direct dict access<commit_after>import json
import boto3
from moto.core.exceptions import InvalidNextTokenException
from moto.core.models import ConfigQueryModel
from moto.iam import iam_backends

class RoleConfigQuery(ConfigQueryModel):
    def list_config_service_resources(
        self,
        resource_ids,
        resource_name,
        limit,
        next_token,
        backend_region=None,
        resource_region=None,
        aggregator=None,
    ):
        # IAM roles are "global" and aren't assigned into any availability zone
        # The resource ID is a AWS-assigned random string like "AROA0BSVNSZKXVHS00SBJ"
        # The resource name is a user-assigned string like "MyDevelopmentAdminRole"
        # Stored in moto backend with the AWS-assigned random string like "AROA0BSVNSZKXVHS00SBJ"

        # Grab roles from backend; need the full values since names and id's are different
        role_list = list(self.backends["global"].roles.values())

        if not role_list:
            return [], None

        # Filter by resource name or ids
        if resource_name or resource_ids:
            filtered_roles = []
            # resource_name takes precedence over resource_ids
            if resource_name:
                for role in role_list:
                    if role.name == resource_name:
                        filtered_roles = [role]
                        break
                # but if both are passed, it must be a subset
                if filtered_roles and resource_ids:
                    if filtered_roles[0].id not in resource_ids:
                        return [], None
            else:
                for role in role_list:
                    if role.id in resource_ids:
                        filtered_roles.append(role)

            # Filtered roles are now the subject for the listing
            role_list = filtered_roles

        if aggregator:
            # IAM is a little special; Roles are created in us-east-1 (which AWS calls the "global" region)
            # However, the resource will return in the aggregator (in duplicate) for each region in the aggregator
            # Therefore, we'll need to find out the regions where the aggregators are running, and then duplicate the resource there

            # In practice, it looks like AWS will only duplicate these resources if you've "used" any roles in the region, but since
            # we can't really tell if this has happened in moto, we'll just bind this to the regions in your aggregator
            aggregated_regions = []
            aggregator_sources = aggregator.get(
                "account_aggregation_sources"
            ) or aggregator.get("organization_aggregation_source")
            for source in aggregator_sources:
                source_dict = source.__dict__
                if source_dict.get("all_aws_regions", False):
                    aggregated_regions = boto3.Session().get_available_regions("config")
                    break
                for region in source_dict.get("aws_regions", []):
                    aggregated_regions.append(region)

            duplicate_role_list = []
            for region in list(set(aggregated_regions)):
                for role in role_list:
                    duplicate_role_list.append(
                        {
                            "_id": "{}{}".format(
                                role.id, region
                            ),  # this is only for sorting, isn't returned outside of this functin
                            "type": "AWS::IAM::Role",
                            "id": role.id,
                            "name": role.name,
                            "region": region,
                        }
                    )

            # Pagination logic, sort by role id
            sorted_roles = sorted(duplicate_role_list, key=lambda role: role["_id"])
        else:
            # Non-aggregated queries are in the else block, and we can treat these like a normal config resource
            # Pagination logic, sort by role id
            sorted_roles = sorted(role_list, key=lambda role: role.id)

        new_token = None

        # Get the start:
        if not next_token:
            start = 0
        else:
            try:
                # Find the index of the next
                start = next(
                    index
                    for (index, r) in enumerate(sorted_roles)
                    if next_token == (r["_id"] if aggregator else r.id)
                )
            except StopIteration:
                raise InvalidNextTokenException()

        # Get the list of items to collect:
        role_list = sorted_roles[start : (start + limit)]

        if len(sorted_roles) > (start + limit):
            record = sorted_roles[start + limit]
            new_token = record["_id"] if aggregator else record.id

        return (
            [
                {
                    "type": "AWS::IAM::Role",
                    "id": role["id"] if aggregator else role.id,
                    "name": role["name"] if aggregator else role.name,
                    "region": role["region"] if aggregator else "global",
                }
                for role in role_list
            ],
            new_token,
        )

    def get_config_resource(
        self, resource_id, resource_name=None, backend_region=None, resource_region=None
    ):

        role = self.backends["global"].roles.get(resource_id, {})

        if not role:
            return

        if resource_name and role.name != resource_name:
            return

        # Format the role to the AWS Config format:
        config_data = role.to_config_dict()

        # The 'configuration' field is also a JSON string:
        config_data["configuration"] = json.dumps(config_data["configuration"])

        # Supplementary config need all values converted to JSON strings if they are not strings already:
        for field, value in config_data["supplementaryConfiguration"].items():
            if not isinstance(value, str):
                config_data["supplementaryConfiguration"][field] = json.dumps(value)

        return config_data

class PolicyConfigQuery(ConfigQueryModel):
    def list_config_service_resources(
        self,
        resource_ids,
        resource_name,
        limit,
        next_token,
        backend_region=None,
        resource_region=None,
        aggregator=None,
    ):
        # IAM policies are "global" and aren't assigned into any availability zone
        # The resource ID is a AWS-assigned random string like "ANPA0BSVNSZK00SJSPVUJ"
        # The resource name is a user-assigned string like "my-development-policy"
        # Stored in moto backend with the arn like "arn:aws:iam::123456789012:policy/my-development-policy"

        policy_list = list(self.backends["global"].managed_policies.values())

        # We don't want to include AWS Managed Policies. This technically needs to
        # respect the configuration recorder's 'includeGlobalResourceTypes' setting,
        # but it's default set be default, and moto's config doesn't yet support
        # custom configuration recorders, we'll just behave as default.
        policy_list = list(
            filter(
                lambda policy: not policy.arn.startswith("arn:aws:iam::aws"),
                policy_list,
            )
        )

        if not policy_list:
            return [], None

        # Filter by resource name or ids
        if resource_name or resource_ids:
            filtered_policies = []
            # resource_name takes precedence over resource_ids
            if resource_name:
                for policy in policy_list:
                    if policy.name == resource_name:
                        filtered_policies = [policy]
                        break
                # but if both are passed, it must be a subset
                if filtered_policies and resource_ids:
                    if filtered_policies[0].id not in resource_ids:
                        return [], None

            else:
                for policy in policy_list:
                    if policy.id in resource_ids:
                        filtered_policies.append(policy)

            # Filtered roles are now the subject for the listing
            policy_list = filtered_policies

        if aggregator:
            # IAM is a little special; Policies are created in us-east-1 (which AWS calls the "global" region)
            # However, the resource will return in the aggregator (in duplicate) for each region in the aggregator
            # Therefore, we'll need to find out the regions where the aggregators are running, and then duplicate the resource there

            # In practice, it looks like AWS will only duplicate these resources if you've "used" any policies in the region, but since
            # we can't really tell if this has happened in moto, we'll just bind this to the regions in your aggregator
            aggregated_regions = []
            aggregator_sources = aggregator.get(
                "account_aggregation_sources"
            ) or aggregator.get("organization_aggregation_source")
            for source in aggregator_sources:
                source_dict = source.__dict__
                if source_dict["all_aws_regions"]:
                    aggregated_regions = boto3.Session().get_available_regions("config")
                    break
                for region in source_dict["aws_regions"]:
                    aggregated_regions.append(region)

            duplicate_policy_list = []
            for region in list(set(aggregated_regions)):
                for policy in policy_list:
                    duplicate_policy_list.append(
                        {
                            "_id": "{}{}".format(
                                policy.id, region
                            ),  # this is only for sorting, isn't returned outside of this functin
                            "type": "AWS::IAM::Policy",
                            "id": policy.id,
                            "name": policy.name,
                            "region": region,
                        }
                    )

            # Pagination logic, sort by role id
            sorted_policies = sorted(
                duplicate_policy_list, key=lambda policy: policy["_id"]
            )

        else:
            # Non-aggregated queries are in the else block, and we can treat these like a normal config resource
            # Pagination logic, sort by role id
            sorted_policies = sorted(policy_list, key=lambda role: role.id)

        new_token = None

        # Get the start:
        if not next_token:
            start = 0
        else:
            try:
                # Find the index of the next
                start = next(
                    index
                    for (index, p) in enumerate(sorted_policies)
                    if next_token == (p["_id"] if aggregator else p.id)
                )
            except StopIteration:
                raise InvalidNextTokenException()

        # Get the list of items to collect:
        policy_list = sorted_policies[start : (start + limit)]

        if len(sorted_policies) > (start + limit):
            record = sorted_policies[start + limit]
            new_token = record["_id"] if aggregator else record.id

        return (
            [
                {
                    "type": "AWS::IAM::Policy",
                    "id": policy["id"] if aggregator else policy.id,
                    "name": policy["name"] if aggregator else policy.name,
                    "region": policy["region"] if aggregator else "global",
                }
                for policy in policy_list
            ],
            new_token,
        )

    def get_config_resource(
        self, resource_id, resource_name=None, backend_region=None, resource_region=None
    ):
        # policies are listed in the backend as arns, but we have to accept the PolicyID as the resource_id
        # we'll make a really crude search for it
        policy = None
        for arn in self.backends["global"].managed_policies.keys():
            policy_candidate = self.backends["global"].managed_policies[arn]
            if policy_candidate.id == resource_id:
                policy = policy_candidate
                break

        if not policy:
            return

        if resource_name and policy.name != resource_name:
            return

        # Format the policy to the AWS Config format:
        config_data = policy.to_config_dict()

        # The 'configuration' field is also a JSON string:
        config_data["configuration"] = json.dumps(config_data["configuration"])

        # Supplementary config need all values converted to JSON strings if they are not strings already:
        for field, value in config_data["supplementaryConfiguration"].items():
            if not isinstance(value, str):
                config_data["supplementaryConfiguration"][field] = json.dumps(value)

        return config_data

role_config_query = RoleConfigQuery(iam_backends)
policy_config_query = PolicyConfigQuery(iam_backends)
<|endoftext|>


<commit_before># Copyright 2012 Christoph Reiter
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.

from pgi.repository import GObject as GObjectModule

GObject = GObjectModule.Object
__all__ = ["GObject"]

from pgi.gtype import PGType as GType
__all__.append("GType")

TYPE_INVALID = GObjectModule.type_from_name('invalid')
TYPE_NONE = GObjectModule.type_from_name('void')
TYPE_INTERFACE = GObjectModule.type_from_name('GInterface')
TYPE_CHAR = GObjectModule.type_from_name('gchar')
TYPE_UCHAR = GObjectModule.type_from_name('guchar')
TYPE_BOOLEAN = GObjectModule.type_from_name('gboolean')
TYPE_INT = GObjectModule.type_from_name('gint')
TYPE_UINT = GObjectModule.type_from_name('guint')
TYPE_LONG = GObjectModule.type_from_name('glong')
TYPE_ULONG = GObjectModule.type_from_name('gulong')
TYPE_INT64 = GObjectModule.type_from_name('gint64')
TYPE_UINT64 = GObjectModule.type_from_name('guint64')
TYPE_ENUM = GObjectModule.type_from_name('GEnum')
TYPE_FLAGS = GObjectModule.type_from_name('GFlags')
TYPE_FLOAT = GObjectModule.type_from_name('gfloat')
TYPE_DOUBLE = GObjectModule.type_from_name('gdouble')
TYPE_STRING = GObjectModule.type_from_name('gchararray')
TYPE_POINTER = GObjectModule.type_from_name('gpointer')
TYPE_BOXED = GObjectModule.type_from_name('GBoxed')
TYPE_PARAM = GObjectModule.type_from_name('GParam')
TYPE_OBJECT = GObjectModule.type_from_name('GObject')
TYPE_PYOBJECT = GObjectModule.type_from_name('PyObject')
TYPE_GTYPE = GObjectModule.type_from_name('GType')
TYPE_STRV = GObjectModule.type_from_name('GStrv')
TYPE_VARIANT = GObjectModule.type_from_name('GVariant')
TYPE_GSTRING = GObjectModule.type_from_name('GString')
TYPE_UNICHAR = TYPE_UINT
__all__ += ['TYPE_INVALID', 'TYPE_NONE', 'TYPE_INTERFACE', 'TYPE_CHAR',
            'TYPE_UCHAR', 'TYPE_BOOLEAN', 'TYPE_INT', 'TYPE_UINT', 'TYPE_LONG',
            'TYPE_ULONG', 'TYPE_INT64', 'TYPE_UINT64', 'TYPE_ENUM', 'TYPE_FLAGS',
            'TYPE_FLOAT', 'TYPE_DOUBLE', 'TYPE_STRING', 'TYPE_POINTER',
            'TYPE_BOXED', 'TYPE_PARAM', 'TYPE_OBJECT', 'TYPE_PYOBJECT',
            'TYPE_GTYPE', 'TYPE_STRV', 'TYPE_VARIANT', 'TYPE_GSTRING', 'TYPE_UNICHAR']
<commit_msg>Add missing copyright from pygobject<commit_after># -*- Mode: Python; py-indent-offset: 4 -*-
# vim: tabstop=4 shiftwidth=4 expandtab
#
# Copyright (C) 2013 Christoph Reiter
# Copyright (C) 2012 Canonical Ltd.
# Author: Martin Pitt <martin.pitt@ubuntu.com>
# Copyright (C) 2012 Simon Feltman <sfeltman@src.gnome.org>
# Copyright (C) 2012 Bastian Winkler <buz@netbuz.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
# USA

from pgi.repository import GObject as GObjectModule

GObject = GObjectModule.Object
__all__ = ["GObject"]

from pgi.gtype import PGType as GType
__all__.append("GType")

TYPE_INVALID = GObjectModule.type_from_name('invalid')
TYPE_NONE = GObjectModule.type_from_name('void')
TYPE_INTERFACE = GObjectModule.type_from_name('GInterface')
TYPE_CHAR = GObjectModule.type_from_name('gchar')
TYPE_UCHAR = GObjectModule.type_from_name('guchar')
TYPE_BOOLEAN = GObjectModule.type_from_name('gboolean')
TYPE_INT = GObjectModule.type_from_name('gint')
TYPE_UINT = GObjectModule.type_from_name('guint')
TYPE_LONG = GObjectModule.type_from_name('glong')
TYPE_ULONG = GObjectModule.type_from_name('gulong')
TYPE_INT64 = GObjectModule.type_from_name('gint64')
TYPE_UINT64 = GObjectModule.type_from_name('guint64')
TYPE_ENUM = GObjectModule.type_from_name('GEnum')
TYPE_FLAGS = GObjectModule.type_from_name('GFlags')
TYPE_FLOAT = GObjectModule.type_from_name('gfloat')
TYPE_DOUBLE = GObjectModule.type_from_name('gdouble')
TYPE_STRING = GObjectModule.type_from_name('gchararray')
TYPE_POINTER = GObjectModule.type_from_name('gpointer')
TYPE_BOXED = GObjectModule.type_from_name('GBoxed')
TYPE_PARAM = GObjectModule.type_from_name('GParam')
TYPE_OBJECT = GObjectModule.type_from_name('GObject')
TYPE_PYOBJECT = GObjectModule.type_from_name('PyObject')
TYPE_GTYPE = GObjectModule.type_from_name('GType')
TYPE_STRV = GObjectModule.type_from_name('GStrv')
TYPE_VARIANT = GObjectModule.type_from_name('GVariant')
TYPE_GSTRING = GObjectModule.type_from_name('GString')
TYPE_UNICHAR = TYPE_UINT
__all__ += ['TYPE_INVALID', 'TYPE_NONE', 'TYPE_INTERFACE', 'TYPE_CHAR',
            'TYPE_UCHAR', 'TYPE_BOOLEAN', 'TYPE_INT', 'TYPE_UINT', 'TYPE_LONG',
            'TYPE_ULONG', 'TYPE_INT64', 'TYPE_UINT64', 'TYPE_ENUM', 'TYPE_FLAGS',
            'TYPE_FLOAT', 'TYPE_DOUBLE', 'TYPE_STRING', 'TYPE_POINTER',
            'TYPE_BOXED', 'TYPE_PARAM', 'TYPE_OBJECT', 'TYPE_PYOBJECT',
            'TYPE_GTYPE', 'TYPE_STRV', 'TYPE_VARIANT', 'TYPE_GSTRING', 'TYPE_UNICHAR']
<|endoftext|>