<commit_before># -*- coding: utf-8 -*-

"""WSGI middleware to log all requests to Dynamodb on AWS.
Username and API Key are infered from following environment variables:
- AWS_ACCESS_KEY_ID
- AWS_SECRET_ACCESS_KEY
The user needs a policy on AWS that allows it to PUT to DynamoDB.
You can ignore body of the requests by controlling the regexes set by env var:
- REQUEST_LOGGER_MIMETYPE_IGNORE_REGEX
- REQUEST_LOGGER_PATH_IGNORE_REGEX
"""

import logging
logger = logging.getLogger(__name__)

import threading
import os

from boto.dynamodb2.fields import HashKey
from boto.dynamodb2.table import Table
from boto import dynamodb2

import re

import datetime

DEFAULTS = {
    'request_logger.max_body_size_in_bytes': os.environ.get('REQUEST_LOGGER_MAX_BODY_SIZE_IN_BYTES', 2000),
    'request_logger.path_ignore_regex': os.environ.get('REQUEST_LOGGER_PATH_IGNORE_REGEX', '.*/auth/.*'),
    'request_logger.mimetype_ignore_regex': os.environ.get('REQUEST_LOGGER_MIMETYPE_IGNORE_REGEX', '.*multipart.*'),
    'request_logger.request_id_header_name': os.environ.get('REQUEST_LOGGER_REQUEST_ID_HEADER_NAME', 'X_REQUEST_ID'),
}

# 20 KB.
MAX_BODY_SIZE_IN_BYTES = 20000

IGNORE_PATH_VALIDATOR = re.compile('{0}'.format(DEFAULTS['request_logger.path_ignore_regex']))
IGNORE_MIMETYPE_VALIDATOR = re.compile('{0}'.format(DEFAULTS['request_logger.mimetype_ignore_regex']))

WRITE_METHODS = ('POST', 'PUT', 'DELETE', 'PATCH')
REQUEST_ID_HEADER_NAME = DEFAULTS['request_logger.request_id_header_name']

def client_factory():
    """Return an AmazonDB client that provides a
      ``put_item(data=data)`` method.
    """

    return Table(
        'request_storer',
        schema=[HashKey('request_id')],
        connection=dynamodb2.connect_to_region('eu-west-1')
    )

class RequestLoggerTweenFactory(object):
    """Simple pyramid tween to log all of our requests by Heroku request id."""

    def __init__(self, handler, registry, client=None):
        self.handler = handler
        self.registry = registry
        self.settings = registry.settings
        self.client = client
        if not self.client:
            # If we are testing and haven't supplied a client, mock it out.
            if self.settings.get('mode') == 'testing':
                from mock import Mock
                self.client = Mock()
            else:
                self.client = client_factory()

    def __call__(self, request):
        """Request logger pyramid tween, logs requests that have errored and
        are of HTTP WRITE_METHODS to DynamoDB."""

        # Unpack the request.
        path = request.path
        headers = request.headers

        # Figure out if we'll want to log an exception or not.
        request_id = headers.get(REQUEST_ID_HEADER_NAME, None)
        should_log_exc = request_id and request.method.upper() in WRITE_METHODS

        # Let the app actually handle the request.
        response = self.handler(request)

        # Then if the request was interesting and resulted in
        # an error response, then spawn a green thread to log
        # the request data in the background.
        if should_log_exc and response.status_int > 399:
            body = self.get_body(request)
            self.log_request(request_id, path, headers, body)

        return response

    def get_body(self, request):
        """Read the request body upto a maximum length."""

        if IGNORE_MIMETYPE_VALIDATOR.match(request.content_type):
            return 'N/a - multipart request.'

        if IGNORE_PATH_VALIDATOR.match(request.path):
            return 'N/a - may contain password'

        # Get the body file and wind it back to the beginning.
        sock = request.body_file_seekable
        start_pos = sock.tell()
        sock.seek(0)

        # Read upto a maximum length.
        body = sock.read(MAX_BODY_SIZE_IN_BYTES)

        # And just for sanity's sake, put it back where we
        # found it.
        sock.seek(start_pos)
        return body

    def put_to_dynamodb(self, data):
        """Make a PUT request to dynamodb2 and insert the data"""

        try:
            self.client.put_item(data=data)
        except UnicodeDecodeError:
            data['body'] = 'Unicode error when parsing'
            self.client.put_item(data=data)

    def log_request(self, key, path, headers, body):
        """Log the path, headers and body of the HTTP request on a
        key value store db, key is heroku request id.
        """

        if not body:
            body = {}

        # Build a dict with key, headers and body.
        now = datetime.datetime.now().isoformat()
        data = {'request_id': key, 'body': body, 'path': path, 'created': now}
        for k, v in headers.items():
            data[k] = v

        # Fire and forget.
        t = threading.Thread(target=self.put_to_dynamodb, args=(data,))
        t.start()
<commit_msg>request logger: trying to log 5xx responses from pyramid.<commit_after># -*- coding: utf-8 -*-

"""WSGI middleware to log all requests to Dynamodb on AWS.
Username and API Key are infered from following environment variables:
- AWS_ACCESS_KEY_ID
- AWS_SECRET_ACCESS_KEY
The user needs a policy on AWS that allows it to PUT to DynamoDB.
You can ignore body of the requests by controlling the regexes set by env var:
- REQUEST_LOGGER_MIMETYPE_IGNORE_REGEX
- REQUEST_LOGGER_PATH_IGNORE_REGEX
"""

import logging
logger = logging.getLogger(__name__)

import threading
import os

from boto.dynamodb2.fields import HashKey
from boto.dynamodb2.table import Table
from boto import dynamodb2

import re

import datetime

DEFAULTS = {
    'request_logger.max_body_size_in_bytes': os.environ.get('REQUEST_LOGGER_MAX_BODY_SIZE_IN_BYTES', 2000),
    'request_logger.path_ignore_regex': os.environ.get('REQUEST_LOGGER_PATH_IGNORE_REGEX', '.*/auth/.*'),
    'request_logger.mimetype_ignore_regex': os.environ.get('REQUEST_LOGGER_MIMETYPE_IGNORE_REGEX', '.*multipart.*'),
    'request_logger.request_id_header_name': os.environ.get('REQUEST_LOGGER_REQUEST_ID_HEADER_NAME', 'X_REQUEST_ID'),
}

# 20 KB.
MAX_BODY_SIZE_IN_BYTES = 20000

IGNORE_PATH_VALIDATOR = re.compile('{0}'.format(DEFAULTS['request_logger.path_ignore_regex']))
IGNORE_MIMETYPE_VALIDATOR = re.compile('{0}'.format(DEFAULTS['request_logger.mimetype_ignore_regex']))

WRITE_METHODS = ('POST', 'PUT', 'DELETE', 'PATCH')
REQUEST_ID_HEADER_NAME = DEFAULTS['request_logger.request_id_header_name']

def client_factory():
    """Return an AmazonDB client that provides a
      ``put_item(data=data)`` method.
    """

    return Table(
        'request_storer',
        schema=[HashKey('request_id')],
        connection=dynamodb2.connect_to_region('eu-west-1')
    )

class RequestLoggerTweenFactory(object):
    """Simple pyramid tween to log all of our requests by Heroku request id."""

    def __init__(self, handler, registry, client=None):
        self.handler = handler
        self.registry = registry
        self.settings = registry.settings
        self.client = client
        if not self.client:
            # If we are testing and haven't supplied a client, mock it out.
            if self.settings.get('mode') == 'testing':
                from mock import Mock
                self.client = Mock()
            else:
                self.client = client_factory()

    def __call__(self, request):
        """Request logger pyramid tween, logs requests that have errored and
        are of HTTP WRITE_METHODS to DynamoDB."""

        # Unpack the request.
        path = request.path
        headers = request.headers

        # Figure out if we'll want to log an exception or not.
        request_id = headers.get(REQUEST_ID_HEADER_NAME, None)
        should_log_exc = request_id and request.method.upper() in WRITE_METHODS

        # Let the app actually handle the request.
        try:
            response = self.handler(request)
        except Exception:
            # Extract the information and re-raise.
            body = self.get_body(request)
            self.log_request(request_id, path, headers, body)
            raise

        # Then if the request was interesting and resulted in
        # an error response, then spawn a green thread to log
        # the request data in the background.
        if should_log_exc and response.status_int > 399:
            body = self.get_body(request)
            self.log_request(request_id, path, headers, body)

        return response

    def get_body(self, request):
        """Read the request body upto a maximum length."""

        if IGNORE_MIMETYPE_VALIDATOR.match(request.content_type):
            return 'N/a - multipart request.'

        if IGNORE_PATH_VALIDATOR.match(request.path):
            return 'N/a - may contain password'

        # Get the body file and wind it back to the beginning.
        sock = request.body_file_seekable
        start_pos = sock.tell()
        sock.seek(0)

        # Read upto a maximum length.
        body = sock.read(MAX_BODY_SIZE_IN_BYTES)

        # And just for sanity's sake, put it back where we
        # found it.
        sock.seek(start_pos)
        return body

    def put_to_dynamodb(self, data):
        """Make a PUT request to dynamodb2 and insert the data"""

        try:
            self.client.put_item(data=data)
        except UnicodeDecodeError:
            data['body'] = 'Unicode error when parsing'
            self.client.put_item(data=data)

    def log_request(self, key, path, headers, body):
        """Log the path, headers and body of the HTTP request on a
        key value store db, key is heroku request id.
        """

        if not body:
            body = {}

        # Build a dict with key, headers and body.
        now = datetime.datetime.now().isoformat()
        data = {'request_id': key, 'body': body, 'path': path, 'created': now}
        for k, v in headers.items():
            data[k] = v

        # Fire and forget.
        t = threading.Thread(target=self.put_to_dynamodb, args=(data,))
        t.start()
<|endoftext|>


<commit_before>"""@author: Bryan Silverthorn <bcs@cargo-cult.org>"""

import re
import cargo
import borg

logger = cargo.get_logger(__name__)

def basic_command(relative):
    """Prepare a basic competition solver command."""

    return ["{{root}}/{0}".format(relative), "{task}", "{seed}"]

#commands = {
    ## complete
    #"precosat-570": ["{root}/precosat-570-239dbbe-100801/precosat", "--seed={seed}", "{task}"],
    #"lingeling-276": ["{root}/lingeling-276-6264d55-100731/lingeling", "--seed={seed}", "{task}"],
    #"cryptominisat-2.9.0": ["{root}/cryptominisat-2.9.0/cryptominisat-2.9.0Linux64", "--randomize={seed}", "{task}"],
    #"glucosER": ["{root}/glucosER/glucoser_static", "{task}"],
    #"glucose": ["{root}/glucose/glucose_static", "{task}"],
    #"SApperloT": ["{root}/SApperloT/SApperloT-base", "-seed={seed}", "{task}"],
    #"march_hi": basic_command("march_hi/march_hi"),
    #"kcnfs-2006": ["{root}/kcnfs-2006/kcnfs-2006", "{task}"],
    ## incomplete
    #"TNM": basic_command("TNM/TNM"),
    #"gnovelty+2": basic_command("gnovelty+2/gnovelty+2"),
    #"hybridGM3": basic_command("hybridGM3/hybridGM3"),
    #"adaptg2wsat2009++": basic_command("adaptg2wsat2009++/adaptg2wsat2009++"),
    #"iPAWS": basic_command("iPAWS/iPAWS"),
    #"FH": basic_command("FH/FH"),
    #"NCVWr": basic_command("NCVWr/NCVWr"),
    #}

def parse_sat_output(stdout):
    """Parse a solver's standard competition-format output."""

    match = re.search(r"^s +(.+)$", stdout, re.M)

    if match:
        (answer_type,) = map(str.upper, match.groups())

        if answer_type == "SATISFIABLE":
            answer = []

            for line in re.findall(r"^v ([ \-0-9]*)$", stdout, re.M):
                answer.extend(map(int, line.split()))

            if answer[-1] == 0:
                return answer[:-1]
        elif answer_type == "UNSATISFIABLE":
            return False

    return None

class SAT_SolverFactory(object):
    """Construct a basic competition solver callable."""

    def __init__(self, root, command):
        """Initialize."""

        self._root = root
        self._command = command

    def __call__(self, task, stm_queue = None, solver_id = None):
        return \
            borg.solver_io.RunningSolver(
                parse_sat_output,
                self._command,
                self._root(),
                task.path,
                stm_queue = stm_queue,
                solver_id = solver_id,
                )

<commit_msg>Breaking out solver lists.<commit_after>"""@author: Bryan Silverthorn <bcs@cargo-cult.org>"""

import re
import cargo
import borg

logger = cargo.get_logger(__name__)

def parse_sat_output(stdout):
    """Parse a solver's standard competition-format output."""

    match = re.search(r"^s +(.+)$", stdout, re.M)

    if match:
        (answer_type,) = map(str.upper, match.groups())

        if answer_type == "SATISFIABLE":
            answer = []

            for line in re.findall(r"^v ([ \-0-9]*)$", stdout, re.M):
                answer.extend(map(int, line.split()))

            if answer[-1] == 0:
                return answer[:-1]
        elif answer_type == "UNSATISFIABLE":
            return False

    return None

class SAT_SolverFactory(object):
    """Construct a basic competition solver callable."""

    def __init__(self, root, command):
        """Initialize."""

        self._root = root
        self._command = command

    def __call__(self, task, stm_queue = None, solver_id = None):
        return \
            borg.solver_io.RunningSolver(
                parse_sat_output,
                self._command,
                self._root(),
                task.path,
                stm_queue = stm_queue,
                solver_id = solver_id,
                )

<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

NAMESPACES = {
    'http://www.w3.org/1999/02/22-rdf-syntax-ns#': 'rdf',
    'http://www.w3.org/2000/01/rdf-schema#': 'rdfs',
    'http://www.w3.org/2001/XMLSchema#': 'xsd',
    'http://www.w3.org/2002/07/owl#': 'owl',
    'http://www.w3.org/2004/02/skos/core#': 'skos',
    'http://www.w3.org/ns/prov#': 'prov',
    'http://purl.org/dc/elements/1.1/': 'dc',
    'http://purl.org/dc/terms/': 'dct',
    'http://schema.org/description': 'schema',
    # lexical namespaces
    'http://lemon-model.net/lemon#': 'lemon',
    'http://www.lexinfo.net/ontology/2.0/lexinfo#': 'lexinfo',
    'http://id.loc.gov/vocabulary/iso639-1/': 'lang',
    # wikidata, dbpedia, geo
    'http://dbpedia.org/ontology/': 'dbo',
    'http://www.opengis.net/ont/geosparql#': 'geo',
    'http://www.wikidata.org/prop/direct/': 'wdt',
    'http://www.wikidata.org/entity/': 'wd',
    'http://sws.geonames.org': 'gn',
    # weblyzard namespaces
    'http://weblyzard.com/skb/lexicon/': 'skblex',
    'http://weblyzard.com/skb/property/': 'skbprop',
    'http://weblyzard.com/skb/entity/': 'skbentity',
    'http://weblyzard.com/skb/entity/agent/': 'agent',
    # weblyzard keywords per language
    'http://weblyzard.com/skb/keyword/en#': 'skbkwen',
    'http://weblyzard.com/skb/keyword/de#': 'skbkwde',
    'http://weblyzard.com/skb/keyword/fr#': 'skbkwfr',
    'http://weblyzard.com/skb/keyword/es#': 'skbkwes',
    'http://weblyzard.com/skb/keyword/it#': 'skbkwit',
    'http://weblyzard.com/skb/keyword/pt#': 'skbkwpt',
    'http://weblyzard.com/skb/keyword/zh#': 'skbkwzh',
    'http://weblyzard.com/skb/keyword/ru#': 'skbkwru',
    'http://weblyzard.com/skb/keyword/ar#': 'skbkwar',
    'http://weblyzard.com/skb/keyword/nl#': 'skbkwnl',
}

PREFIXES = '\n'.join(['PREFIX {value}: <{key}>'.format(value=value,
                                                       key=key) \
                      for key, value in NAMESPACES.items()])

def prefix_uri(uri, namespaces=None):
    '''
    Replaces a sub-path from the uri with the most specific prefix as defined
    in the namespaces.

    :param uri: The URI to modify.
    :type uri: str
    :param namespaces: A mapping of namespace to prefix (without ':').
    :type namespaces: `dict`
    :returns: The modified URI if applicable
    :rtype: str
    '''
    if namespaces is None:
        namespaces = NAMESPACES
    # replace most specific/longest prefix, hence sorted
    for namespace in sorted(list(namespaces.keys()), key=len, reverse=True):
        if namespace in uri:
            replaced = uri.replace(namespace, '{}:'.format(namespaces[namespace]))
            if '/' in replaced or '#' in replaced:
                # slashes or hashes in prefixed URIs not allowed
                continue
            else:
                return replaced
    return uri

def replace_prefix(uri, namespaces=None):
    '''
    Replaces a prefix with the full namespace path.

    :param uri: The URI to modify.
    :type uri: str
    :param namespaces: A mapping of namespace to prefix (without ':').
    :type namespaces: `dict`
    :returns: The modified URI if applicable
    :rtype: str
    '''
    if namespaces is None:
        namespaces = NAMESPACES
    for namespace, prefix in namespaces.items():
        if prefix in uri:
            return uri.replace('{}:'.format(prefix), namespace)
    return uri
<commit_msg>chg: first checking if uri starts with http, else return immediately<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

NAMESPACES = {
    'http://www.w3.org/1999/02/22-rdf-syntax-ns#': 'rdf',
    'http://www.w3.org/2000/01/rdf-schema#': 'rdfs',
    'http://www.w3.org/2001/XMLSchema#': 'xsd',
    'http://www.w3.org/2002/07/owl#': 'owl',
    'http://www.w3.org/2004/02/skos/core#': 'skos',
    'http://www.w3.org/ns/prov#': 'prov',
    'http://purl.org/dc/elements/1.1/': 'dc',
    'http://purl.org/dc/terms/': 'dct',
    'http://schema.org/description': 'schema',
    # lexical namespaces
    'http://lemon-model.net/lemon#': 'lemon',
    'http://www.lexinfo.net/ontology/2.0/lexinfo#': 'lexinfo',
    'http://id.loc.gov/vocabulary/iso639-1/': 'lang',
    # wikidata, dbpedia, geo
    'http://dbpedia.org/ontology/': 'dbo',
    'http://www.opengis.net/ont/geosparql#': 'geo',
    'http://www.wikidata.org/prop/direct/': 'wdt',
    'http://www.wikidata.org/entity/': 'wd',
    'http://sws.geonames.org': 'gn',
    # weblyzard namespaces
    'http://weblyzard.com/skb/lexicon/': 'skblex',
    'http://weblyzard.com/skb/property/': 'skbprop',
    'http://weblyzard.com/skb/entity/': 'skbentity',
    'http://weblyzard.com/skb/entity/agent/': 'agent',
    # weblyzard keywords per language
    'http://weblyzard.com/skb/keyword/en#': 'skbkwen',
    'http://weblyzard.com/skb/keyword/de#': 'skbkwde',
    'http://weblyzard.com/skb/keyword/fr#': 'skbkwfr',
    'http://weblyzard.com/skb/keyword/es#': 'skbkwes',
    'http://weblyzard.com/skb/keyword/it#': 'skbkwit',
    'http://weblyzard.com/skb/keyword/pt#': 'skbkwpt',
    'http://weblyzard.com/skb/keyword/zh#': 'skbkwzh',
    'http://weblyzard.com/skb/keyword/ru#': 'skbkwru',
    'http://weblyzard.com/skb/keyword/ar#': 'skbkwar',
    'http://weblyzard.com/skb/keyword/nl#': 'skbkwnl',
}

PREFIXES = '\n'.join(['PREFIX {value}: <{key}>'.format(value=value,
                                                       key=key) \
                      for key, value in NAMESPACES.items()])

def prefix_uri(uri, namespaces=None):
    '''
    Replaces a sub-path from the uri with the most specific prefix as defined
    in the namespaces.

    :param uri: The URI to modify.
    :type uri: str
    :param namespaces: A mapping of namespace to prefix (without ':').
    :type namespaces: `dict`
    :returns: The modified URI if applicable
    :rtype: str
    '''
    if namespaces is None:
        namespaces = NAMESPACES
    if not uri.startswith('http'):
        return uri
    # replace most specific/longest prefix, hence sorted
    for namespace in sorted(list(namespaces.keys()), key=len, reverse=True):
        if namespace in uri:
            replaced = uri.replace(namespace, '{}:'.format(namespaces[namespace]))
            if '/' in replaced or '#' in replaced:
                # slashes or hashes in prefixed URIs not allowed
                continue
            else:
                return replaced
    return uri

def replace_prefix(uri, namespaces=None):
    '''
    Replaces a prefix with the full namespace path.

    :param uri: The URI to modify.
    :type uri: str
    :param namespaces: A mapping of namespace to prefix (without ':').
    :type namespaces: `dict`
    :returns: The modified URI if applicable
    :rtype: str
    '''
    if namespaces is None:
        namespaces = NAMESPACES
    for namespace, prefix in namespaces.items():
        if prefix in uri:
            return uri.replace('{}:'.format(prefix), namespace)
    return uri
<|endoftext|>


<commit_before>"""Define resources model classes.

Defines the basic attributes & interface of any resource type class,
responsible for the resource static & dynamic information.
"""
# pylint: disable=too-many-instance-attributes,no-self-use,broad-except
import os
from bdb import BdbQuit

from ipdbugger import debug
from attrdict import AttrDict

from rotest.common import core_log
from rotest.common.utils import get_work_dir
from rotest.management.common.utils import HOST_PORT_SEPARATOR

class ConvertToKwargsMeta(type):
    """Metaclass that validates no positional args are passed to constructor.

    This enables avoiding requesting resources is coherent, i.e.:
        Assuming ResClass gets 'x' in the __init__, then when requesting:
            res1 = ResClass(5)
        The x=5 is not passed to kwargs, thus is not propagated when
        creating the actual resource when the test starts.
    """
    def __call__(cls, *args, **kwargs):
        if len(args) > 0:
            raise RuntimeError("BaseResource constructors must not get "
                               "positional arguments")

        resource = type.__call__(cls, *args, **kwargs)
        resource.kwargs = kwargs
        for field_name, field_value in kwargs.iteritems():
            setattr(resource, field_name, field_value)

        if isinstance(resource.data, AttrDict):
            resource.data.update(kwargs)

class BaseResource(object):
    """Represent the common interface of all the resources.

    To implement a resource, you may override:
    initialize, connect, finalize, validate, create_sub_resources, store_state.
    Also, assign a data container class by setting the
    attribute 'DATA_CLASS', which should point to a subclass of
    :class:`rotest.management.models.resource_data.ResourceData`.
    Resource without a data class (also called 'Services') will be handled
    locally, without involving the server.

    Attributes:
        DATA_CLASS (class): class of the resource's global data container.
        logger (logger): resource's logger instance.
        data (ResourceData): assigned data instance.
        config (AttrDict): run configuration.
        work_dir (str): working directory for this resource.
        save_state (bool): flag to indicate if the resource is a duplication.
        force_initialize (bool): a flag to determine if the resource will be
            initialized even if the validation succeeds.
    """
    __metaclass__ = ConvertToKwargsMeta

    DATA_CLASS = None

    _SHELL_CLIENT = None
    _SHELL_REQUEST_NAME = 'shell_resource'

    def __init__(self, data=None, **kwargs):
        # We use core_log as default logger in case
        # that resource is used outside case.
        self.kwargs = None
        self.logger = core_log

        if data is not None:
            self.data = data
            for field_name, field_value in self.data.get_fields().iteritems():
                setattr(self, field_name, field_value)

        else:
            self.data = AttrDict()
            self.name = "%s-%d" % (self.__class__.__name__, id(self))

        self.config = None
        self.parent = None
        self.work_dir = None
        self.save_state = None
        self.force_initialize = None

        self._sub_resources = None

    def create_sub_resources(self):
        """Create and return the sub resources if needed.

        Override and assign sub-resources to fields in the current resource,
        using the 'data' object.

        Returns:
            iterable. sub-resources created.
        """
        return ()

    def is_available(self, user_name=""):
        """Return whether resource is available for the given user.

        Args:
            user_name (str): user name to be checked. Empty string means
                available to all.

        Returns:
            bool. determine whether resource is available for the given user.

        Note:
            If this method is called from code then leaf is equal to 'self'.
            If it is being called by BaseResources table in DB then leaf's
            'is_available' method will be called.
        """
        leaf = self.leaf  # 'leaf' is a property.
        if leaf == self:
            user_name = user_name.split(HOST_PORT_SEPARATOR)[0]
            return self.reserved in [user_name, ""] and self.owner == ""

        return leaf.is_available(user_name)

    def __repr__(self):
        return "%s(%s)" % (self.__class__.__name__, self.data)

    def get_sub_resources(self):
        """Return an iterable to the resource's sub-resources."""
        return (sub_resource for sub_resource in self._sub_resources)

    def set_sub_resources(self):
        """Create and set the sub resources if needed."""
        self._sub_resources = tuple(self.create_sub_resources())
        for resource in self.get_sub_resources():
            resource.parent = self
            resource.set_sub_resources()

    def _safe_execute(self, callbacks, *args, **kwargs):
        """Executes all the callbacks, even if one or more fails.

        Args:
            callbacks (list): callbacks to be called one after the other (even
                if one of them fails).
            args (list): args passed for each callback when invoked.
            kwargs (dict): keyword-args passed for each callback when invoked.

        Raises:
            RuntimeError: when one, or more, of the callbacks fail.
        """
        error_messages = []
        for callback in callbacks:
            try:
                self.logger.debug("Starting %s", callback)
                callback(*args, **kwargs)
                self.logger.debug("%s ended successfully", callback)

            except Exception as ex:
                self.logger.exception("%s failed", callback)
                error_messages.append("%s: %s" % (callback, ex))

        if len(error_messages) != 0:
            raise RuntimeError("Some of the callbacks have failed. "
                               "Reasons: %s" % "\n".join(error_messages))

    def set_work_dir(self, resource_name, containing_work_dir):
        """Set the work directory under the given case's work directory.

        Args:
            resource_name (str): name of resource.
            containing_work_dir (str): root work directory.
        """
        self.work_dir = get_work_dir(containing_work_dir, resource_name)

        for resource in self.get_sub_resources():
            resource.set_work_dir(resource.name, self.work_dir)

    def store_state(self, state_dir_path):
        """Hook method for backing up the resource state.

        Args:
            state_dir_path (str): path of state directory to be saved.
        """
        self._safe_execute([resource.store_state_dir for resource
                            in self.get_sub_resources()], state_dir_path)

    def store_state_dir(self, dir_name):
        """Store the resource state under a sub-directory of the work_dir.

        Create a directory under the resource work directory and calls
        store_state on that directory.

        Args:
            dir_name (str): sub-directory name.
        """
        store_dir = os.path.join(self.work_dir, dir_name)
        self.logger.debug("Creating dir %r", store_dir)

        # In case a state dir already exists, create a new one.
        state_dir_index = 1
        while os.path.exists(store_dir):
            state_dir_index += 1
            store_dir = os.path.join(self.work_dir,
                                     dir_name + str(state_dir_index))

        os.makedirs(store_dir)

        self.logger.debug("Storing resource %r state", self.name)
        self.store_state(store_dir)
        self.logger.debug("Resource %r state stored", self.name)

    def initialize(self):
        """Hook method for setting up the resource before using it.

        Will be called by the resource client once the connection to the
        resource was successful. Override to specify the resource's
        initialization procedure (remember to call 'super' at the beginning).
        """
        self.logger.debug("Initializing resource %r", self.name)

    def connect(self):
        """Setup a connection session to the resource.

        Will be called by the resource client once the
        resource is locked successfully.
        """
        self.logger.debug("Connecting resource %r", self.name)

        for resource in self.get_sub_resources():
            resource.connect()

    def finalize(self):
        """Hook method for cleaning up the resource after using it.

        Will be called by the resource client before the resource is released.
        Override to specify the resource's finalization procedure
        (remember to call 'super' at the end).
        """
        # Create a list of resources finalize calls in a specific order.
        self.logger.debug("Finalizing resource %r", self.name)
        finalization_methods = [resource.finalize for resource
                                in self.get_sub_resources()]
        self._safe_execute(finalization_methods)

    def validate(self):
        """Validate whether the resource is ready for work or not.

        If this method failed, the resource client will call the 'initialize'
        method to setup the resource.

        Returns:
            bool. True if validation succeeded, False otherwise.
        """
        return False

    def enable_debug(self):
        """Wrap the resource methods with debugger."""
        debug(self.initialize, ignore_exceptions=[KeyboardInterrupt, BdbQuit])
        debug(self.finalize, ignore_exceptions=[KeyboardInterrupt, BdbQuit])
        debug(self.validate, ignore_exceptions=[KeyboardInterrupt, BdbQuit])
        debug(self.store_state, ignore_exceptions=[KeyboardInterrupt, BdbQuit])
        for resource in self.get_sub_resources():
            resource.enable_debug()

    @classmethod
    def lock(cls, config=None, skip_init=False, **kwargs):
        """Lock an instance of this resource class.

        Args:
            config (str): path to the json config file.
            skip_init (bool): whether to skip initialization or not.
            kwargs (dict): additional query parameters for the request,
                e.g. name or group.

        Returns:
            BaseResource. locked and initialized resource, ready for work.
        """
        # These runtime imports are done to avoid cyclic imports.
        from rotest.core.runner import _parse_config_file
        from rotest.management.client.manager import (ClientResourceManager,
                                                      ResourceRequest)

        if BaseResource._SHELL_CLIENT is None:
            BaseResource._SHELL_CLIENT = ClientResourceManager()

        resource_request = ResourceRequest(BaseResource._SHELL_REQUEST_NAME,
                                           cls,
                                           **kwargs)

        config_dict = None
        if config is not None:
            config_dict = _parse_config_file(config)

        result = BaseResource._SHELL_CLIENT.request_resources(
                                                        [resource_request],
                                                        config=config_dict,
                                                        skip_init=skip_init,
                                                        use_previous=False)

        return result[BaseResource._SHELL_REQUEST_NAME]

    def release(self):
        """Release the resource, assuming it was locked with a shell client."""
        if BaseResource._SHELL_CLIENT is not None:
            BaseResource._SHELL_CLIENT.release_resources(
                {BaseResource._SHELL_REQUEST_NAME: self},
                force_release=True)
<commit_msg>fixed by tox<commit_after>"""Define resources model classes.

Defines the basic attributes & interface of any resource type class,
responsible for the resource static & dynamic information.
"""
# pylint: disable=too-many-instance-attributes,no-self-use,broad-except
import os
from bdb import BdbQuit

from ipdbugger import debug
from attrdict import AttrDict

from rotest.common import core_log
from rotest.common.utils import get_work_dir
from rotest.management.common.utils import HOST_PORT_SEPARATOR

class ConvertToKwargsMeta(type):
    """Metaclass that validates no positional args are passed to constructor.

    This enables avoiding requesting resources is coherent, i.e.:
        Assuming ResClass gets 'x' in the __init__, then when requesting:
            res1 = ResClass(5)
        The x=5 is not passed to kwargs, thus is not propagated when
        creating the actual resource when the test starts.
    """
    def __call__(cls, *args, **kwargs):
        if len(args) > 0:
            raise RuntimeError("BaseResource constructors must not get "
                               "positional arguments")

        resource = type.__call__(cls, *args, **kwargs)
        resource.kwargs = kwargs
        for field_name, field_value in kwargs.iteritems():
            setattr(resource, field_name, field_value)

        if isinstance(resource.data, AttrDict):
            resource.data.update(kwargs)

class BaseResource(object):
    """Represent the common interface of all the resources.

    To implement a resource, you may override:
    initialize, connect, finalize, validate, create_sub_resources, store_state.
    Also, assign a data container class by setting the
    attribute 'DATA_CLASS', which should point to a subclass of
    :class:`rotest.management.models.resource_data.ResourceData`.
    Resource without a data class (also called 'Services') will be handled
    locally, without involving the server.

    Attributes:
        DATA_CLASS (class): class of the resource's global data container.
        logger (logger): resource's logger instance.
        data (ResourceData): assigned data instance.
        config (AttrDict): run configuration.
        work_dir (str): working directory for this resource.
        save_state (bool): flag to indicate if the resource is a duplication.
        force_initialize (bool): a flag to determine if the resource will be
            initialized even if the validation succeeds.
    """
    __metaclass__ = ConvertToKwargsMeta

    DATA_CLASS = None

    _SHELL_CLIENT = None
    _SHELL_REQUEST_NAME = 'shell_resource'

    def __init__(self, data=None, **kwargs):
        # We use core_log as default logger in case
        # that resource is used outside case.
        self.kwargs = kwargs
        self.logger = core_log

        if data is not None:
            self.data = data
            for field_name, field_value in self.data.get_fields().iteritems():
                setattr(self, field_name, field_value)

        else:
            self.data = AttrDict()
            self.name = "%s-%d" % (self.__class__.__name__, id(self))

        self.config = None
        self.parent = None
        self.work_dir = None
        self.save_state = None
        self.force_initialize = None

        self._sub_resources = None

    def create_sub_resources(self):
        """Create and return the sub resources if needed.

        Override and assign sub-resources to fields in the current resource,
        using the 'data' object.

        Returns:
            iterable. sub-resources created.
        """
        return ()

    def is_available(self, user_name=""):
        """Return whether resource is available for the given user.

        Args:
            user_name (str): user name to be checked. Empty string means
                available to all.

        Returns:
            bool. determine whether resource is available for the given user.

        Note:
            If this method is called from code then leaf is equal to 'self'.
            If it is being called by BaseResources table in DB then leaf's
            'is_available' method will be called.
        """
        leaf = self.leaf  # 'leaf' is a property.
        if leaf == self:
            user_name = user_name.split(HOST_PORT_SEPARATOR)[0]
            return self.reserved in [user_name, ""] and self.owner == ""

        return leaf.is_available(user_name)

    def __repr__(self):
        return "%s(%s)" % (self.__class__.__name__, self.data)

    def get_sub_resources(self):
        """Return an iterable to the resource's sub-resources."""
        return (sub_resource for sub_resource in self._sub_resources)

    def set_sub_resources(self):
        """Create and set the sub resources if needed."""
        self._sub_resources = tuple(self.create_sub_resources())
        for resource in self.get_sub_resources():
            resource.parent = self
            resource.set_sub_resources()

    def _safe_execute(self, callbacks, *args, **kwargs):
        """Executes all the callbacks, even if one or more fails.

        Args:
            callbacks (list): callbacks to be called one after the other (even
                if one of them fails).
            args (list): args passed for each callback when invoked.
            kwargs (dict): keyword-args passed for each callback when invoked.

        Raises:
            RuntimeError: when one, or more, of the callbacks fail.
        """
        error_messages = []
        for callback in callbacks:
            try:
                self.logger.debug("Starting %s", callback)
                callback(*args, **kwargs)
                self.logger.debug("%s ended successfully", callback)

            except Exception as ex:
                self.logger.exception("%s failed", callback)
                error_messages.append("%s: %s" % (callback, ex))

        if len(error_messages) != 0:
            raise RuntimeError("Some of the callbacks have failed. "
                               "Reasons: %s" % "\n".join(error_messages))

    def set_work_dir(self, resource_name, containing_work_dir):
        """Set the work directory under the given case's work directory.

        Args:
            resource_name (str): name of resource.
            containing_work_dir (str): root work directory.
        """
        self.work_dir = get_work_dir(containing_work_dir, resource_name)

        for resource in self.get_sub_resources():
            resource.set_work_dir(resource.name, self.work_dir)

    def store_state(self, state_dir_path):
        """Hook method for backing up the resource state.

        Args:
            state_dir_path (str): path of state directory to be saved.
        """
        self._safe_execute([resource.store_state_dir for resource
                            in self.get_sub_resources()], state_dir_path)

    def store_state_dir(self, dir_name):
        """Store the resource state under a sub-directory of the work_dir.

        Create a directory under the resource work directory and calls
        store_state on that directory.

        Args:
            dir_name (str): sub-directory name.
        """
        store_dir = os.path.join(self.work_dir, dir_name)
        self.logger.debug("Creating dir %r", store_dir)

        # In case a state dir already exists, create a new one.
        state_dir_index = 1
        while os.path.exists(store_dir):
            state_dir_index += 1
            store_dir = os.path.join(self.work_dir,
                                     dir_name + str(state_dir_index))

        os.makedirs(store_dir)

        self.logger.debug("Storing resource %r state", self.name)
        self.store_state(store_dir)
        self.logger.debug("Resource %r state stored", self.name)

    def initialize(self):
        """Hook method for setting up the resource before using it.

        Will be called by the resource client once the connection to the
        resource was successful. Override to specify the resource's
        initialization procedure (remember to call 'super' at the beginning).
        """
        self.logger.debug("Initializing resource %r", self.name)

    def connect(self):
        """Setup a connection session to the resource.

        Will be called by the resource client once the
        resource is locked successfully.
        """
        self.logger.debug("Connecting resource %r", self.name)

        for resource in self.get_sub_resources():
            resource.connect()

    def finalize(self):
        """Hook method for cleaning up the resource after using it.

        Will be called by the resource client before the resource is released.
        Override to specify the resource's finalization procedure
        (remember to call 'super' at the end).
        """
        # Create a list of resources finalize calls in a specific order.
        self.logger.debug("Finalizing resource %r", self.name)
        finalization_methods = [resource.finalize for resource
                                in self.get_sub_resources()]
        self._safe_execute(finalization_methods)

    def validate(self):
        """Validate whether the resource is ready for work or not.

        If this method failed, the resource client will call the 'initialize'
        method to setup the resource.

        Returns:
            bool. True if validation succeeded, False otherwise.
        """
        return False

    def enable_debug(self):
        """Wrap the resource methods with debugger."""
        debug(self.initialize, ignore_exceptions=[KeyboardInterrupt, BdbQuit])
        debug(self.finalize, ignore_exceptions=[KeyboardInterrupt, BdbQuit])
        debug(self.validate, ignore_exceptions=[KeyboardInterrupt, BdbQuit])
        debug(self.store_state, ignore_exceptions=[KeyboardInterrupt, BdbQuit])
        for resource in self.get_sub_resources():
            resource.enable_debug()

    @classmethod
    def lock(cls, config=None, skip_init=False, **kwargs):
        """Lock an instance of this resource class.

        Args:
            config (str): path to the json config file.
            skip_init (bool): whether to skip initialization or not.
            kwargs (dict): additional query parameters for the request,
                e.g. name or group.

        Returns:
            BaseResource. locked and initialized resource, ready for work.
        """
        # These runtime imports are done to avoid cyclic imports.
        from rotest.core.runner import _parse_config_file
        from rotest.management.client.manager import (ClientResourceManager,
                                                      ResourceRequest)

        if BaseResource._SHELL_CLIENT is None:
            BaseResource._SHELL_CLIENT = ClientResourceManager()

        resource_request = ResourceRequest(BaseResource._SHELL_REQUEST_NAME,
                                           cls,
                                           **kwargs)

        config_dict = None
        if config is not None:
            config_dict = _parse_config_file(config)

        result = BaseResource._SHELL_CLIENT.request_resources(
                                                        [resource_request],
                                                        config=config_dict,
                                                        skip_init=skip_init,
                                                        use_previous=False)

        return result[BaseResource._SHELL_REQUEST_NAME]

    def release(self):
        """Release the resource, assuming it was locked with a shell client."""
        if BaseResource._SHELL_CLIENT is not None:
            BaseResource._SHELL_CLIENT.release_resources(
                {BaseResource._SHELL_REQUEST_NAME: self},
                force_release=True)
<|endoftext|>


<commit_before># UNLESS EXEPTIONALLY REQUIRED TRY TO AVOID IMPORTING ANY MODULES HERE
# ESPECIALLY ANYTHING IN SAS, SASMODELS NAMESPACE
from PyQt5.QtWidgets import QMainWindow
from PyQt5.QtWidgets import QMdiArea
from PyQt5.QtWidgets import QSplashScreen
from PyQt5.QtWidgets import QApplication
from PyQt5.QtGui import QPixmap
from PyQt5.QtCore import Qt
import os
import sys
# Local UI
from sas.qtgui.UI import main_resources_rc
from .UI.MainWindowUI import Ui_SasView

class MainSasViewWindow(QMainWindow, Ui_SasView):
    # Main window of the application
    def __init__(self, screen_resolution, parent=None):
        super(MainSasViewWindow, self).__init__(parent)
        self.setupUi(self)

        # define workspace for dialogs.
        self.workspace = QMdiArea(self)
        self.screen_width =  screen_resolution.width()
        self.screen_height = screen_resolution.height()
        self.setCentralWidget(self.workspace)

        # Temporary solution for problem with menubar on Mac
        if sys.platform == "darwin":  # Mac
            self.menubar.setNativeMenuBar(False)

        # Create the gui manager
        from .GuiManager import GuiManager
        try:
            self.guiManager = GuiManager(self)
        except Exception as ex:
            import logging
            logging.error("Application failed with: "+str(ex))

    def closeEvent(self, event):
        if self.guiManager.quitApplication():
            event.accept()
        else:
            event.ignore()

def SplashScreen():
    """
    Displays splash screen as soon as humanely possible.
    The screen will disappear as soon as the event loop starts.
    """
    pixmap_path = "images/SVwelcome_mini.png"
    if os.path.splitext(sys.argv[0])[1].lower() == ".py":
        pixmap_path = "src/sas/qtgui/images/SVwelcome_mini.png"
    pixmap = QPixmap(pixmap_path)
    splashScreen = QSplashScreen(pixmap)
    return splashScreen

def run_sasview():
    app = QApplication([])

    #Initialize logger
    from sas.logger_config import SetupLogger
    SetupLogger(__name__).config_development()

    # initialize OpenCL setting
    from sas import get_custom_config
    SAS_OPENCL = get_custom_config().SAS_OPENCL
    if SAS_OPENCL and "SAS_OPENCL" not in os.environ:
        os.environ["SAS_OPENCL"] = SAS_OPENCL

    # Make the event loop interruptable quickly
    import signal
    signal.signal(signal.SIGINT, signal.SIG_DFL)

    # Main must have reference to the splash screen, so making it explicit
    splash = SplashScreen()
    splash.show()
    app.setAttribute(Qt.AA_EnableHighDpiScaling)
    # Main application style.
    #app.setStyle('Fusion')

    # fix for pyinstaller packages app to avoid ReactorAlreadyInstalledError
    import sys
    if 'twisted.internet.reactor' in sys.modules:
        del sys.modules['twisted.internet.reactor']

    # DO NOT move the following import to the top!
    # (unless you know what you're doing)
    import qt5reactor
    # Using the Qt5 reactor wrapper from https://github.com/ghtdak/qtreactor
    qt5reactor.install()

    # DO NOT move the following import to the top!
    from twisted.internet import reactor

    screen_resolution = app.desktop().screenGeometry()

    # Show the main SV window
    mainwindow = MainSasViewWindow(screen_resolution)
    mainwindow.showMaximized()

    # no more splash screen
    splash.finish(mainwindow)

    # Time for the welcome window
    mainwindow.guiManager.showWelcomeMessage()

    # No need to .exec_ - the reactor takes care of it.
    reactor.run()

if __name__ == "__main__":
    run_sasview()
<commit_msg>Set dll path to ~/.sasview/compiled_models on startup. Fixes #1560.<commit_after># UNLESS EXEPTIONALLY REQUIRED TRY TO AVOID IMPORTING ANY MODULES HERE
# ESPECIALLY ANYTHING IN SAS, SASMODELS NAMESPACE
import os
import sys

from PyQt5.QtWidgets import QMainWindow
from PyQt5.QtWidgets import QMdiArea
from PyQt5.QtWidgets import QSplashScreen
from PyQt5.QtWidgets import QApplication
from PyQt5.QtGui import QPixmap
from PyQt5.QtCore import Qt

# Local UI
from sas.qtgui.UI import main_resources_rc
from .UI.MainWindowUI import Ui_SasView

class MainSasViewWindow(QMainWindow, Ui_SasView):
    # Main window of the application
    def __init__(self, screen_resolution, parent=None):
        super(MainSasViewWindow, self).__init__(parent)
        self.setupUi(self)

        # define workspace for dialogs.
        self.workspace = QMdiArea(self)
        self.screen_width =  screen_resolution.width()
        self.screen_height = screen_resolution.height()
        self.setCentralWidget(self.workspace)

        # Temporary solution for problem with menubar on Mac
        if sys.platform == "darwin":  # Mac
            self.menubar.setNativeMenuBar(False)

        # Create the gui manager
        from .GuiManager import GuiManager
        try:
            self.guiManager = GuiManager(self)
        except Exception as ex:
            import logging
            logging.error("Application failed with: "+str(ex))

    def closeEvent(self, event):
        if self.guiManager.quitApplication():
            event.accept()
        else:
            event.ignore()

def SplashScreen():
    """
    Displays splash screen as soon as humanely possible.
    The screen will disappear as soon as the event loop starts.
    """
    pixmap_path = "images/SVwelcome_mini.png"
    if os.path.splitext(sys.argv[0])[1].lower() == ".py":
        pixmap_path = "src/sas/qtgui/images/SVwelcome_mini.png"
    pixmap = QPixmap(pixmap_path)
    splashScreen = QSplashScreen(pixmap)
    return splashScreen

def run_sasview():
    app = QApplication([])

    #Initialize logger
    from sas.logger_config import SetupLogger
    SetupLogger(__name__).config_development()

    # initialize sasmodels settings
    from sas import get_custom_config, get_user_dir
    if "SAS_DLL_PATH" not in os.environ:
        os.environ["SAS_DLL_PATH"] = os.path.join(
            get_user_dir(), "compiled_models")
    SAS_OPENCL = get_custom_config().SAS_OPENCL
    if SAS_OPENCL and "SAS_OPENCL" not in os.environ:
        os.environ["SAS_OPENCL"] = SAS_OPENCL

    # Make the event loop interruptable quickly
    import signal
    signal.signal(signal.SIGINT, signal.SIG_DFL)

    # Main must have reference to the splash screen, so making it explicit
    splash = SplashScreen()
    splash.show()
    app.setAttribute(Qt.AA_EnableHighDpiScaling)
    # Main application style.
    #app.setStyle('Fusion')

    # fix for pyinstaller packages app to avoid ReactorAlreadyInstalledError
    if 'twisted.internet.reactor' in sys.modules:
        del sys.modules['twisted.internet.reactor']

    # DO NOT move the following import to the top!
    # (unless you know what you're doing)
    import qt5reactor
    # Using the Qt5 reactor wrapper from https://github.com/ghtdak/qtreactor
    qt5reactor.install()

    # DO NOT move the following import to the top!
    from twisted.internet import reactor

    screen_resolution = app.desktop().screenGeometry()

    # Show the main SV window
    mainwindow = MainSasViewWindow(screen_resolution)
    mainwindow.showMaximized()

    # no more splash screen
    splash.finish(mainwindow)

    # Time for the welcome window
    mainwindow.guiManager.showWelcomeMessage()

    # No need to .exec_ - the reactor takes care of it.
    reactor.run()

if __name__ == "__main__":
    run_sasview()
<|endoftext|>


<commit_before>"""
Created on 6 Oct 2017

@author: Bruno Beloff (bruno.beloff@southcoastscience.com)

https://github.com/aws/aws-iot-device-sdk-python
https://s3.amazonaws.com/aws-iot-device-sdk-python-docs/sphinx/html/index.html

https://github.com/aws/aws-iot-device-sdk-python/issues/57

https://stackoverflow.com/questions/20083858/how-to-extract-value-from-bound-method-in-python

https://github.com/aws/aws-iot-device-sdk-python-V2
"""

import logging

from AWSIoTPythonSDK.exception.AWSIoTExceptions import connectError, connectTimeoutException
from AWSIoTPythonSDK.exception.AWSIoTExceptions import disconnectError, disconnectTimeoutException

import AWSIoTPythonSDK.MQTTLib as MQTTLib

from scs_core.data.json import JSONify
from scs_core.data.str import Str

# --------------------------------------------------------------------------------------------------------------------

class MQTTClient(object):
    """
    classdocs
    """
    __KEEP_ALIVE_INTERVAL =         600                     # recommended: 30 default: 600 (sec)

    __PORT =                        8883

    __QUEUE_SIZE =                  -1                      # recommended: infinite
    __QUEUE_DROP_BEHAVIOUR =        MQTTLib.DROP_OLDEST     # not required for infinite queue
    __QUEUE_DRAINING_FREQUENCY =    2                       # recommended: 2 (Hz)

    __RECONN_BASE =                 1                       # recommended: 1 (sec)
    __RECONN_MAX =                  32                      # recommended: 32 or 128 (sec), was 10
    __RECONN_STABLE =               20                      # recommended: 20 (sec), was 10

    __DISCONNECT_TIMEOUT =          10                      # recommended: 10 (sec), was 40
    __OPERATION_TIMEOUT =           5                       # recommended: 5 (sec), was 20

    __PUB_QOS =                     1
    __SUB_QOS =                     1

    # ----------------------------------------------------------------------------------------------------------------

    @staticmethod
    def __assert_logger(level):
        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')

        stream_handler = logging.StreamHandler()
        stream_handler.setFormatter(formatter)

        logger = logging.getLogger("AWSIoTPythonSDK.core")
        logger.setLevel(level)
        logger.addHandler(stream_handler)

    # ----------------------------------------------------------------------------------------------------------------

    def __init__(self, *subscribers):
        """
        Constructor
        """
        self.__client = None
        self.__subscribers = subscribers

    # ----------------------------------------------------------------------------------------------------------------

    def connect(self, auth, debug):
        # logging...
        if debug:
            self.__assert_logger(logging.DEBUG)

        # client...
        self.__client = MQTTLib.AWSIoTMQTTClient(auth.client_id)

        # configuration...
        self.__client.configureEndpoint(auth.endpoint, self.__PORT)

        self.__client.configureCredentials(auth.root_ca_file_path, auth.private_key_path, auth.certificate_path)

        self.__client.configureAutoReconnectBackoffTime(self.__RECONN_BASE, self.__RECONN_MAX, self.__RECONN_STABLE)

        self.__client.configureOfflinePublishQueueing(self.__QUEUE_SIZE)
        self.__client.configureDrainingFrequency(self.__QUEUE_DRAINING_FREQUENCY)

        self.__client.configureConnectDisconnectTimeout(self.__DISCONNECT_TIMEOUT)
        self.__client.configureMQTTOperationTimeout(self.__OPERATION_TIMEOUT)

        # subscriptions...
        for subscriber in self.__subscribers:
            self.__client.subscribe(subscriber.topic, self.__SUB_QOS, subscriber.handler)

        # connect...
        try:
            return self.__client.connect(self.__KEEP_ALIVE_INTERVAL)

        except (connectError, connectTimeoutException) as ex:
            raise OSError(ex.__class__.__name__)

    def disconnect(self):
        if self.__client is None:
            return

        try:
            self.__client.disconnect()

        except (disconnectError, disconnectTimeoutException):
            pass

        self.__client = None

    # --------------------------------------------------------------------------------------------------------------------

    def publish(self, publication):
        if not self.__client:
            raise IOError("publish: no client")

        payload = JSONify.dumps(publication.payload)

        return self.__client.publish(publication.topic, payload, self.__PUB_QOS)

    # ----------------------------------------------------------------------------------------------------------------

    def __str__(self, *args, **kwargs):
        return "MQTTClient:{subscribers:%s}" % Str.collection(self.__subscribers)

# --------------------------------------------------------------------------------------------------------------------

class MQTTSubscriber(object):
    """
    classdocs
    """

    # ----------------------------------------------------------------------------------------------------------------

    def __init__(self, topic, handler):
        """
        Constructor
        """
        self.__topic = topic
        self.__handler = handler

    # ----------------------------------------------------------------------------------------------------------------

    @property
    def topic(self):
        return self.__topic

    @property
    def handler(self):
        return self.__handler

    # ----------------------------------------------------------------------------------------------------------------

    def __str__(self, *args, **kwargs):
        return "MQTTSubscriber:{topic:%s, handler:%s}" % (self.topic, self.handler.__self__)
<commit_msg>Changed MQTTClient KeepAlive interval<commit_after>"""
Created on 6 Oct 2017

@author: Bruno Beloff (bruno.beloff@southcoastscience.com)

https://github.com/aws/aws-iot-device-sdk-python
https://s3.amazonaws.com/aws-iot-device-sdk-python-docs/sphinx/html/index.html

https://github.com/aws/aws-iot-device-sdk-python/issues/57

https://stackoverflow.com/questions/20083858/how-to-extract-value-from-bound-method-in-python

https://github.com/aws/aws-iot-device-sdk-python-V2
"""

import logging

from AWSIoTPythonSDK.exception.AWSIoTExceptions import connectError, connectTimeoutException
from AWSIoTPythonSDK.exception.AWSIoTExceptions import disconnectError, disconnectTimeoutException

import AWSIoTPythonSDK.MQTTLib as MQTTLib

from scs_core.data.json import JSONify
from scs_core.data.str import Str

# --------------------------------------------------------------------------------------------------------------------

class MQTTClient(object):
    """
    classdocs
    """
    __KEEP_ALIVE_INTERVAL =         60                     # recommended: 30 default: 600 (sec)

    __PORT =                        8883

    __QUEUE_SIZE =                  -1                      # recommended: infinite
    __QUEUE_DROP_BEHAVIOUR =        MQTTLib.DROP_OLDEST     # not required for infinite queue
    __QUEUE_DRAINING_FREQUENCY =    2                       # recommended: 2 (Hz)

    __RECONN_BASE =                 1                       # recommended: 1 (sec)
    __RECONN_MAX =                  32                      # recommended: 32 or 128 (sec), was 10
    __RECONN_STABLE =               20                      # recommended: 20 (sec), was 10

    __DISCONNECT_TIMEOUT =          10                      # recommended: 10 (sec), was 40
    __OPERATION_TIMEOUT =           5                       # recommended: 5 (sec), was 20

    __PUB_QOS =                     1
    __SUB_QOS =                     1

    # ----------------------------------------------------------------------------------------------------------------

    @staticmethod
    def __assert_logger(level):
        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')

        stream_handler = logging.StreamHandler()
        stream_handler.setFormatter(formatter)

        logger = logging.getLogger("AWSIoTPythonSDK.core")
        logger.setLevel(level)
        logger.addHandler(stream_handler)

    # ----------------------------------------------------------------------------------------------------------------

    def __init__(self, *subscribers):
        """
        Constructor
        """
        self.__client = None
        self.__subscribers = subscribers

    # ----------------------------------------------------------------------------------------------------------------

    def connect(self, auth, debug):
        # logging...
        if debug:
            self.__assert_logger(logging.DEBUG)

        # client...
        self.__client = MQTTLib.AWSIoTMQTTClient(auth.client_id)

        # configuration...
        self.__client.configureEndpoint(auth.endpoint, self.__PORT)

        self.__client.configureCredentials(auth.root_ca_file_path, auth.private_key_path, auth.certificate_path)

        self.__client.configureAutoReconnectBackoffTime(self.__RECONN_BASE, self.__RECONN_MAX, self.__RECONN_STABLE)

        self.__client.configureOfflinePublishQueueing(self.__QUEUE_SIZE)
        self.__client.configureDrainingFrequency(self.__QUEUE_DRAINING_FREQUENCY)

        self.__client.configureConnectDisconnectTimeout(self.__DISCONNECT_TIMEOUT)
        self.__client.configureMQTTOperationTimeout(self.__OPERATION_TIMEOUT)

        # subscriptions...
        for subscriber in self.__subscribers:
            self.__client.subscribe(subscriber.topic, self.__SUB_QOS, subscriber.handler)

        # connect...
        try:
            return self.__client.connect(self.__KEEP_ALIVE_INTERVAL)

        except (connectError, connectTimeoutException) as ex:
            raise OSError(ex.__class__.__name__)

    def disconnect(self):
        if self.__client is None:
            return

        try:
            self.__client.disconnect()

        except (disconnectError, disconnectTimeoutException):
            pass

        self.__client = None

    # --------------------------------------------------------------------------------------------------------------------

    def publish(self, publication):
        if not self.__client:
            raise IOError("publish: no client")

        payload = JSONify.dumps(publication.payload)

        return self.__client.publish(publication.topic, payload, self.__PUB_QOS)

    # ----------------------------------------------------------------------------------------------------------------

    def __str__(self, *args, **kwargs):
        return "MQTTClient:{subscribers:%s}" % Str.collection(self.__subscribers)

# --------------------------------------------------------------------------------------------------------------------

class MQTTSubscriber(object):
    """
    classdocs
    """

    # ----------------------------------------------------------------------------------------------------------------

    def __init__(self, topic, handler):
        """
        Constructor
        """
        self.__topic = topic
        self.__handler = handler

    # ----------------------------------------------------------------------------------------------------------------

    @property
    def topic(self):
        return self.__topic

    @property
    def handler(self):
        return self.__handler

    # ----------------------------------------------------------------------------------------------------------------

    def __str__(self, *args, **kwargs):
        return "MQTTSubscriber:{topic:%s, handler:%s}" % (self.topic, self.handler.__self__)
<|endoftext|>


<commit_before>from __future__ import absolute_import

from rest_framework.response import Response
from rest_framework import serializers

from sentry.api.base import DocSection
from sentry.api.bases.project import ProjectEndpoint, ProjectReleasePermission
from sentry.api.content_negotiation import ConditionalContentNegotiation
from sentry.api.serializers import serialize
from sentry.api.serializers.rest_framework import ListField
from sentry.models import ProjectDSymFile, create_files_from_dsym_zip, \
    VersionDSymFile, DSymApp, DSYM_PLATFORMS

ERR_FILE_EXISTS = 'A file matching this uuid already exists'

class AssociateDsymSerializer(serializers.Serializer):
    checksums = ListField(child=serializers.CharField(max_length=40))
    platform = serializers.ChoiceField(choices=zip(
        DSYM_PLATFORMS.keys(),
        DSYM_PLATFORMS.keys(),
    ))
    name = serializers.CharField(max_length=250)
    appId = serializers.CharField(max_length=250)
    version = serializers.CharField(max_length=40)
    build = serializers.CharField(max_length=40)

def upload_from_request(request, project=None):
    if 'file' not in request.FILES:
        return Response({'detail': 'Missing uploaded file'}, status=400)
    fileobj = request.FILES['file']
    files = create_files_from_dsym_zip(fileobj, project=project)
    return Response(serialize(files, request.user), status=201)

class DSymFilesEndpoint(ProjectEndpoint):
    doc_section = DocSection.PROJECTS
    permission_classes = (ProjectReleasePermission, )

    content_negotiation_class = ConditionalContentNegotiation

    def get(self, request, project):
        """
        List a Project's DSym Files
        ```````````````````````````

        Retrieve a list of dsym files for a given project.

        :pparam string organization_slug: the slug of the organization the
                                          release belongs to.
        :pparam string project_slug: the slug of the project to list the
                                     dsym files of.
        :auth: required
        """

        apps = DSymApp.objects.filter(project=project)
        dsym_files = VersionDSymFile.objects.filter(
            dsym_app=apps
        ).select_related('projectdsymfile').order_by('-build', 'version')

        file_list = ProjectDSymFile.objects.filter(
            project=project,
            versiondsymfile__isnull=True,
        ).select_related('file')[:100]

        return Response(
            {
                'apps': serialize(list(apps)),
                'debugSymbols': serialize(list(dsym_files)),
                'unreferencedDebugSymbols': serialize(list(file_list)),
            }
        )

    def post(self, request, project):
        """
        Upload a New Files
        ``````````````````

        Upload a new dsym file for the given release.

        Unlike other API requests, files must be uploaded using the
        traditional multipart/form-data content-type.

        The file uploaded is a zip archive of a Apple .dSYM folder which
        contains the individual debug images.  Uploading through this endpoint
        will create different files for the contained images.

        :pparam string organization_slug: the slug of the organization the
                                          release belongs to.
        :pparam string project_slug: the slug of the project to change the
                                     release of.
        :param file file: the multipart encoded file.
        :auth: required
        """
        return upload_from_request(request, project=project)

class UnknownDSymFilesEndpoint(ProjectEndpoint):
    doc_section = DocSection.PROJECTS
    permission_classes = (ProjectReleasePermission, )

    def get(self, request, project):
        checksums = request.GET.getlist('checksums')
        missing = ProjectDSymFile.objects.find_missing(checksums, project=project)
        return Response({'missing': missing})

class AssociateDSymFilesEndpoint(ProjectEndpoint):
    doc_section = DocSection.PROJECTS
    permission_classes = (ProjectReleasePermission, )

    def post(self, request, project):
        serializer = AssociateDsymSerializer(data=request.DATA)
        if not serializer.is_valid():
            return Response(serializer.errors, status=400)

        data = serializer.object

        associated = []
        dsym_app = DSymApp.objects.create_or_update_app(
            sync_id=None,
            app_id=data['appId'],
            project=project,
            data={'name': data['name']},
            platform=DSYM_PLATFORMS[data['platform']],
        )
        dsym_files = ProjectDSymFile.objects.find_by_checksums(data['checksums'], project)

        for dsym_file in dsym_files:
            version_dsym_file, created = VersionDSymFile.objects.get_or_create(
                dsym_file=dsym_file,
                version=data['version'],
                build=data['build'],
                defaults=dict(dsym_app=dsym_app),
            )
            if created:
                associated.append(dsym_file)

        return Response({
            'associatedDsymFiles': serialize(associated, request.user),
        })
<commit_msg>Make build optional for proguard associations<commit_after>from __future__ import absolute_import

from rest_framework.response import Response
from rest_framework import serializers

from sentry.api.base import DocSection
from sentry.api.bases.project import ProjectEndpoint, ProjectReleasePermission
from sentry.api.content_negotiation import ConditionalContentNegotiation
from sentry.api.serializers import serialize
from sentry.api.serializers.rest_framework import ListField
from sentry.models import ProjectDSymFile, create_files_from_dsym_zip, \
    VersionDSymFile, DSymApp, DSYM_PLATFORMS

ERR_FILE_EXISTS = 'A file matching this uuid already exists'

class AssociateDsymSerializer(serializers.Serializer):
    checksums = ListField(child=serializers.CharField(max_length=40))
    platform = serializers.ChoiceField(choices=zip(
        DSYM_PLATFORMS.keys(),
        DSYM_PLATFORMS.keys(),
    ))
    name = serializers.CharField(max_length=250)
    appId = serializers.CharField(max_length=250)
    version = serializers.CharField(max_length=40)
    build = serializers.CharField(max_length=40, null=True)

def upload_from_request(request, project=None):
    if 'file' not in request.FILES:
        return Response({'detail': 'Missing uploaded file'}, status=400)
    fileobj = request.FILES['file']
    files = create_files_from_dsym_zip(fileobj, project=project)
    return Response(serialize(files, request.user), status=201)

class DSymFilesEndpoint(ProjectEndpoint):
    doc_section = DocSection.PROJECTS
    permission_classes = (ProjectReleasePermission, )

    content_negotiation_class = ConditionalContentNegotiation

    def get(self, request, project):
        """
        List a Project's DSym Files
        ```````````````````````````

        Retrieve a list of dsym files for a given project.

        :pparam string organization_slug: the slug of the organization the
                                          release belongs to.
        :pparam string project_slug: the slug of the project to list the
                                     dsym files of.
        :auth: required
        """

        apps = DSymApp.objects.filter(project=project)
        dsym_files = VersionDSymFile.objects.filter(
            dsym_app=apps
        ).select_related('projectdsymfile').order_by('-build', 'version')

        file_list = ProjectDSymFile.objects.filter(
            project=project,
            versiondsymfile__isnull=True,
        ).select_related('file')[:100]

        return Response(
            {
                'apps': serialize(list(apps)),
                'debugSymbols': serialize(list(dsym_files)),
                'unreferencedDebugSymbols': serialize(list(file_list)),
            }
        )

    def post(self, request, project):
        """
        Upload a New Files
        ``````````````````

        Upload a new dsym file for the given release.

        Unlike other API requests, files must be uploaded using the
        traditional multipart/form-data content-type.

        The file uploaded is a zip archive of a Apple .dSYM folder which
        contains the individual debug images.  Uploading through this endpoint
        will create different files for the contained images.

        :pparam string organization_slug: the slug of the organization the
                                          release belongs to.
        :pparam string project_slug: the slug of the project to change the
                                     release of.
        :param file file: the multipart encoded file.
        :auth: required
        """
        return upload_from_request(request, project=project)

class UnknownDSymFilesEndpoint(ProjectEndpoint):
    doc_section = DocSection.PROJECTS
    permission_classes = (ProjectReleasePermission, )

    def get(self, request, project):
        checksums = request.GET.getlist('checksums')
        missing = ProjectDSymFile.objects.find_missing(checksums, project=project)
        return Response({'missing': missing})

class AssociateDSymFilesEndpoint(ProjectEndpoint):
    doc_section = DocSection.PROJECTS
    permission_classes = (ProjectReleasePermission, )

    def post(self, request, project):
        serializer = AssociateDsymSerializer(data=request.DATA)
        if not serializer.is_valid():
            return Response(serializer.errors, status=400)

        data = serializer.object

        associated = []
        dsym_app = DSymApp.objects.create_or_update_app(
            sync_id=None,
            app_id=data['appId'],
            project=project,
            data={'name': data['name']},
            platform=DSYM_PLATFORMS[data['platform']],
        )
        dsym_files = ProjectDSymFile.objects.find_by_checksums(data['checksums'], project)

        for dsym_file in dsym_files:
            version_dsym_file, created = VersionDSymFile.objects.get_or_create(
                dsym_file=dsym_file,
                version=data['version'],
                build=data['build'],
                defaults=dict(dsym_app=dsym_app),
            )
            if created:
                associated.append(dsym_file)

        return Response({
            'associatedDsymFiles': serialize(associated, request.user),
        })
<|endoftext|>


<commit_before>from ...decorators import syft_decorator
from ...common.id import UID
from ...interfaces import Serializable
from .storeable_object import StorableObject
from typing import Iterable
from abc import ABC

class ObjectStore(Serializable, ABC):
    """
    ObjectStore is the common interface for all the stores that a Node can handle. This should
    provide a dict-like interface on handling data on a worker. Indexing should be always done
    by using UID objects, while de indexed value should always be a SerizableObject.
    """

    @syft_decorator(typechecking=True)
    def __sizeof__(self) -> int:
        """
        Method to return the memory size of the object if possible, if not, it will return __len__.

        Returns:
            int: memory size of the store.
        """
        raise NotImplementedError

    @syft_decorator(typechecking=True)
    def __str__(self) -> str:
        """
        Method to print to first 5 entries of the ObjectStore in a database-like format.

        Returns:
            str: header and first 5 entries of store in a formatted manner.
        """
        raise NotImplementedError

    @syft_decorator(typechecking=True)
    def __len__(self) -> int:
        """
        Method to return the number of elements in the store.

        Returns:
            int: the number of objects in the store.
        """
        raise NotImplementedError

    @syft_decorator(typechecking=True)
    def keys(self) -> Iterable[UID]:
        """
        Method to return all indexes from the store.

        Returns:
            Iterable[UID]: an iterable containing the keys of the store.
        """
        raise NotImplementedError

    @syft_decorator(typechecking=True)
    def values(self) -> Iterable[StorableObject]:
        """
        Method to return all values from the store.

        Returns:
            Iterable[StorableObject]: an iterable containing the keys of the store.
        """
        raise NotImplementedError

    @syft_decorator(typechecking=True)
    def store(self, obj: StorableObject) -> None:
        """
        Method to store an object based on its own UID in the store.

        Args:
            obj (StorableObject): the object to be stored.
        """
        raise NotImplementedError

    @syft_decorator(typechecking=True)
    def __contains__(self, key: UID) -> bool:
        """
        Method to check if an UID is present in the store.

        Args:
            key (UID): the UID to be searched.

        Returns:
            bool: if the object is present or not.
        """
        raise NotImplementedError

    @syft_decorator(typechecking=True)
    def __getitem__(self, key: UID) -> StorableObject:
        """
        Method to retrieve an object from the store.

        Args:
            key (UID): The key to be searched for in the store.

        Returns:
            StorableObject: The object associated with the input key.

        Raises:
            ValueError: If the key is not present in the store.
        """
        raise NotImplementedError

    @syft_decorator(typechecking=True)
    def __setitem__(self, key: UID, value: StorableObject) -> None:
        """
        Method to store an object in the store. The difference between this and store is that you
        can you use self.store to store objects at a different key than the one specified in
        the StorableObject.

        This updates the value at the input UID if it exists already.

        Args:
            key (UID): the UID at which to store the StorableObject.
            value (StorableObject): the StorableObject to be stored.
        """
        raise NotImplementedError

    @syft_decorator(typechecking=True)
    def __delitem__(self, key: UID) -> None:
        """
        Method to remove an object from the store based on its UID.

        Args:
            key (UID): the key at which to delete the object.

        Raises:
            ValueError: if the key is not present in the store.
        """
        raise NotImplementedError

    @syft_decorator(typechecking=True)
    def clear(self) -> None:
        """
        Clears all storage owned by the store.
        """
        raise NotImplementedError
<commit_msg>Fix broken imports<commit_after>from ...decorators import syft_decorator
from ..common.uid import UID
from ..common.serializable import Serializable
from .storeable_object import StorableObject
from typing import Iterable
from abc import ABC

class ObjectStore(Serializable, ABC):
    """
    ObjectStore is the common interface for all the stores that a Node can handle. This should
    provide a dict-like interface on handling data on a worker. Indexing should be always done
    by using UID objects, while de indexed value should always be a SerizableObject.
    """

    @syft_decorator(typechecking=True)
    def __sizeof__(self) -> int:
        """
        Method to return the memory size of the object if possible, if not, it will return __len__.

        Returns:
            int: memory size of the store.
        """
        raise NotImplementedError

    @syft_decorator(typechecking=True)
    def __str__(self) -> str:
        """
        Method to print to first 5 entries of the ObjectStore in a database-like format.

        Returns:
            str: header and first 5 entries of store in a formatted manner.
        """
        raise NotImplementedError

    @syft_decorator(typechecking=True)
    def __len__(self) -> int:
        """
        Method to return the number of elements in the store.

        Returns:
            int: the number of objects in the store.
        """
        raise NotImplementedError

    @syft_decorator(typechecking=True)
    def keys(self) -> Iterable[UID]:
        """
        Method to return all indexes from the store.

        Returns:
            Iterable[UID]: an iterable containing the keys of the store.
        """
        raise NotImplementedError

    @syft_decorator(typechecking=True)
    def values(self) -> Iterable[StorableObject]:
        """
        Method to return all values from the store.

        Returns:
            Iterable[StorableObject]: an iterable containing the keys of the store.
        """
        raise NotImplementedError

    @syft_decorator(typechecking=True)
    def store(self, obj: StorableObject) -> None:
        """
        Method to store an object based on its own UID in the store.

        Args:
            obj (StorableObject): the object to be stored.
        """
        raise NotImplementedError

    @syft_decorator(typechecking=True)
    def __contains__(self, key: UID) -> bool:
        """
        Method to check if an UID is present in the store.

        Args:
            key (UID): the UID to be searched.

        Returns:
            bool: if the object is present or not.
        """
        raise NotImplementedError

    @syft_decorator(typechecking=True)
    def __getitem__(self, key: UID) -> StorableObject:
        """
        Method to retrieve an object from the store.

        Args:
            key (UID): The key to be searched for in the store.

        Returns:
            StorableObject: The object associated with the input key.

        Raises:
            ValueError: If the key is not present in the store.
        """
        raise NotImplementedError

    @syft_decorator(typechecking=True)
    def __setitem__(self, key: UID, value: StorableObject) -> None:
        """
        Method to store an object in the store. The difference between this and store is that you
        can you use self.store to store objects at a different key than the one specified in
        the StorableObject.

        This updates the value at the input UID if it exists already.

        Args:
            key (UID): the UID at which to store the StorableObject.
            value (StorableObject): the StorableObject to be stored.
        """
        raise NotImplementedError

    @syft_decorator(typechecking=True)
    def __delitem__(self, key: UID) -> None:
        """
        Method to remove an object from the store based on its UID.

        Args:
            key (UID): the key at which to delete the object.

        Raises:
            ValueError: if the key is not present in the store.
        """
        raise NotImplementedError

    @syft_decorator(typechecking=True)
    def clear(self) -> None:
        """
        Clears all storage owned by the store.
        """
        raise NotImplementedError
<|endoftext|>