<commit_before>"""Implements the serial interface protocol of RaspiBot's Attiny firmware."""

from collections import namedtuple

EncoderValues = namedtuple('EncoderValues', 'left right')

# Named byte values for the protocol commands

# Requests
ALIVE = b'\x01'

ENCODERS_RIGHT = b'\x02'
ENCODERS_LEFT = b'\x03'
ENCODERS_BOTH = b'\x04'

ENCODERS_RESET_RIGHT = b'\x05'
ENCODERS_RESET_LEFT = b'\x06'
ENCODERS_RESET_BOTH = b'\x07'

ECHO = b'\x0C'

STOP_MOTORS = b'\x21'
SET_LEFT_MOTOR = b'\x29'
SET_RIGHT_MOTOR = b'\x25'
SET_BOTH_MOTORS = b'\x2D'

SET_PI_PARAMETERS = b'\x2F'

SET_BUZZER = b'\x40'

# Responses
ACK = b'\x10'
NAK = b'\x14'

MOTOR_MIN = -127
MOTOR_MAX = 127

def _clamp(value, minimum, maximum):
    return min(maximum, max(minimum, value))

class InvalidResponseException(Exception):
    """
    Raised when the protocol implementation reads an unexpected byte.

    This is typically something that is neither an ACK nor a NAK.
    """

    pass

class InvalidLengthException(Exception):
    """Raised when the "Echo" command receives none or more than one byte."""

    pass

class AttinyProtocol(object):
    """Provides methods for protocol transactions."""

    def __init__(self, serial):
        """Create a protocol instance on a serial interface."""
        super(AttinyProtocol, self).__init__()
        self._serial = serial

    def get_encoders(self):
        """Request both left and right encoder values."""
        self._serial.write(ENCODERS_BOTH)
        response = self._serial.read(4)
        left = int.from_bytes(response[:2], 'big')
        right = int.from_bytes(response[2:], 'big')
        return EncoderValues(left, right)

    def get_left_encoder(self):
        """Request the left encoder value."""
        self._serial.write(ENCODERS_LEFT)
        response = self._serial.read(2)
        value = int.from_bytes(response, 'big')
        return value

    def get_right_encoder(self):
        """Request the right encoder value."""
        self._serial.write(ENCODERS_RIGHT)
        response = self._serial.read(2)
        value = int.from_bytes(response, 'big')
        return value

    def reset_encoders(self):
        """Reset both left and right encoder counters to zero."""
        self._serial.write(ENCODERS_RESET_BOTH)
        response = self._serial.read(1)
        if response == ACK:
            return True
        # empty response can happen on a timeout, which we interpret as
        # "not alive"
        elif response == NAK or response == b'':
            return False
        else:
            raise InvalidResponseException()

    def reset_left_encoder(self):
        """Reset the left encoder counter to zero."""
        self._serial.write(ENCODERS_RESET_LEFT)
        response = self._serial.read(1)
        if response == ACK:
            return True
        # empty response can happen on a timeout, which we interpret as
        # "not alive"
        elif response == NAK or response == b'':
            return False
        else:
            raise InvalidResponseException()

    def reset_right_encoder(self):
        """Reset the right encoder counter to zero."""
        self._serial.write(ENCODERS_RESET_RIGHT)
        response = self._serial.read(1)
        if response == ACK:
            return True
        # empty response can happen on a timeout, which we interpret as
        # "not alive"
        elif response == NAK or response == b'':
            return False
        else:
            raise InvalidResponseException()

    def alive(self):
        """Request an 'alive' signal from the microcontroller."""
        self._serial.write(ALIVE)
        response = self._serial.read(1)
        if response == ACK:
            return True
        # empty response can happen on a timeout, which we interpret as
        # "not alive"
        elif response == NAK or response == b'':
            return False
        else:
            raise InvalidResponseException()

    def echo(self, byte):
        """Receive back the same byte that was sent to the ATtiny."""
        if len(byte) != 1:
            raise InvalidLengthException()
        self._serial.write(ECHO + byte)
        response = self._serial.read(1)
        if response != byte:
            raise InvalidResponseException()
        return response

    def stop_motors(self):
        """Immediately stops both motors."""
        self._serial.write(STOP_MOTORS)
        response = self._serial.read(1)
        if response == ACK:
            return True
        # empty response can happen on a timeout, which we interpret as
        # "not alive"
        elif response == NAK or response == b'':
            return False
        else:
            raise InvalidResponseException()

    def set_motors(self, left, right):
        """
        Set the speed of both motors.

        Values can be in the range [-127, 127]. Speeds higher than that will be
        set to their respective extreme. Negative values turn the wheel
        backwards, positive values turn it forwards. Zero stops the motor.
        Higher absolute values mean higher speed.
        """
        left = _clamp(left, MOTOR_MIN, MOTOR_MAX)
        right = _clamp(right, MOTOR_MIN, MOTOR_MAX)
        left_bytes = left.to_bytes(1, 'big', signed=True)
        right_bytes = right.to_bytes(1, 'big', signed=True)

        self._serial.write(SET_BOTH_MOTORS + left_bytes + right_bytes)

        response = self._serial.read(1)
        if response == ACK:
            return True
        elif response == NAK or response == b'':
            return False
        else:
            raise InvalidResponseException()

    def set_left_motor(self, speed):
        """
        Set the speed of the left motor.

        Values can be in the range [-127, 127]. Speeds higher than that will be
        set to their respective extreme. Negative values turn the wheel
        backwards, positive values turn it forwards. Zero stops the motor.
        Higher absolute values mean higher speed.
        """
        speed = _clamp(speed, MOTOR_MIN, MOTOR_MAX)
        speed_bytes = speed.to_bytes(1, 'big', signed=True)

        self._serial.write(SET_LEFT_MOTOR + speed_bytes)

        response = self._serial.read(1)
        if response == ACK:
            return True
        elif response == NAK or response == b'':
            return False
        else:
            raise InvalidResponseException()

    def set_right_motor(self, speed):
        """
        Set the speed of the right motor.

        Values can be in the range [-127, 127]. Speeds higher than that will be
        set to their respective extreme. Negative values turn the wheel
        backwards, positive values turn it forwards. Zero stops the motor.
        Higher absolute values mean higher speed.
        """
        speed = _clamp(speed, MOTOR_MIN, MOTOR_MAX)
        speed_bytes = speed.to_bytes(1, 'big', signed=True)

        self._serial.write(SET_RIGHT_MOTOR + speed_bytes)

        response = self._serial.read(1)
        if response == ACK:
            return True
        elif response == NAK or response == b'':
            return False
        else:
            raise InvalidResponseException()

    def set_pi_parameters(self, p, i, encoder_scale):
        """
        Set the parameters of the PI controller.

        `p` and `i` can be in the range of signed 16-bit integers
        (-32768, 32767). encoder_scale can be in the range of an unsigned 8-bit
        integer.
        """
        p = _clamp(p, -32768, 32767)
        i = _clamp(i, -32768, 32767)
        encoder_scale = _clamp(encoder_scale, 0, 255)

        p_encoded = p.to_bytes(2, 'big', signed=True)
        i_encoded = i.to_bytes(2, 'big', signed=True)
        scale_encoded = encoder_scale.to_bytes(1, 'big')

        message = SET_PI_PARAMETERS + p_encoded + i_encoded + scale_encoded

        self._serial.write(message)

        response = self._serial.read(1)
        if response == ACK:
            return True
        elif response == NAK or response == b'':
            return False
        else:
            raise InvalidResponseException()

    FREQUENCY_MIN = 0
    FREQUENCY_MAX = 2 ** 16 - 1

    DURATION_MIN = 0
    DURATION_MAX = 2 ** 16 - 1

    VOLUME_MIN = 0
    VOLUME_MAX = 15

    def set_buzzer(self, frequency, duration, volume):
        """
        Play a tone on the buzzer.

        Plays a tone on the buzzer for the given duration with a given volume.
        frequency is in the range (0, 65535), excessive values will be clipped.
        duration is in milliseconds and in the range (0, 65535), excessive
        values will be clipped.
        volume is in the range (0, 15), excessive values will be clipped.
        """
        frequency = _clamp(frequency, self.FREQUENCY_MIN, self.FREQUENCY_MAX)
        duration = _clamp(duration, self.DURATION_MIN, self.DURATION_MAX)
        volume = _clamp(volume, self.VOLUME_MIN, self.VOLUME_MAX)

        frequency_encoded = frequency.to_bytes(2, 'big')
        duration_encoded = duration.to_bytes(2, 'big')
        volume_encoded = volume.to_bytes(1, 'big')

        message = SET_BUZZER + frequency_encoded + duration_encoded + volume_encoded

        self._serial.write(message)

        response = self._serial.read(1)
        if response == ACK:
            return True
        elif response == NAK or response == b'':
            return False
        else:
            raise InvalidResponseException()
<commit_msg>implement serial command 'stop buzzer'<commit_after>"""Implements the serial interface protocol of RaspiBot's Attiny firmware."""

from collections import namedtuple

EncoderValues = namedtuple('EncoderValues', 'left right')

# Named byte values for the protocol commands

# Requests
ALIVE = b'\x01'

ENCODERS_RIGHT = b'\x02'
ENCODERS_LEFT = b'\x03'
ENCODERS_BOTH = b'\x04'

ENCODERS_RESET_RIGHT = b'\x05'
ENCODERS_RESET_LEFT = b'\x06'
ENCODERS_RESET_BOTH = b'\x07'

ECHO = b'\x0C'

STOP_MOTORS = b'\x21'
SET_LEFT_MOTOR = b'\x29'
SET_RIGHT_MOTOR = b'\x25'
SET_BOTH_MOTORS = b'\x2D'

SET_PI_PARAMETERS = b'\x2F'

SET_BUZZER = b'\x40'
STOP_BUZZER = b'\x43'

# Responses
ACK = b'\x10'
NAK = b'\x14'

MOTOR_MIN = -127
MOTOR_MAX = 127

def _clamp(value, minimum, maximum):
    return min(maximum, max(minimum, value))

class InvalidResponseException(Exception):
    """
    Raised when the protocol implementation reads an unexpected byte.

    This is typically something that is neither an ACK nor a NAK.
    """

    pass

class InvalidLengthException(Exception):
    """Raised when the "Echo" command receives none or more than one byte."""

    pass

class AttinyProtocol(object):
    """Provides methods for protocol transactions."""

    def __init__(self, serial):
        """Create a protocol instance on a serial interface."""
        super(AttinyProtocol, self).__init__()
        self._serial = serial

    def get_encoders(self):
        """Request both left and right encoder values."""
        self._serial.write(ENCODERS_BOTH)
        response = self._serial.read(4)
        left = int.from_bytes(response[:2], 'big')
        right = int.from_bytes(response[2:], 'big')
        return EncoderValues(left, right)

    def get_left_encoder(self):
        """Request the left encoder value."""
        self._serial.write(ENCODERS_LEFT)
        response = self._serial.read(2)
        value = int.from_bytes(response, 'big')
        return value

    def get_right_encoder(self):
        """Request the right encoder value."""
        self._serial.write(ENCODERS_RIGHT)
        response = self._serial.read(2)
        value = int.from_bytes(response, 'big')
        return value

    def reset_encoders(self):
        """Reset both left and right encoder counters to zero."""
        self._serial.write(ENCODERS_RESET_BOTH)
        response = self._serial.read(1)
        if response == ACK:
            return True
        # empty response can happen on a timeout, which we interpret as
        # "not alive"
        elif response == NAK or response == b'':
            return False
        else:
            raise InvalidResponseException()

    def reset_left_encoder(self):
        """Reset the left encoder counter to zero."""
        self._serial.write(ENCODERS_RESET_LEFT)
        response = self._serial.read(1)
        if response == ACK:
            return True
        # empty response can happen on a timeout, which we interpret as
        # "not alive"
        elif response == NAK or response == b'':
            return False
        else:
            raise InvalidResponseException()

    def reset_right_encoder(self):
        """Reset the right encoder counter to zero."""
        self._serial.write(ENCODERS_RESET_RIGHT)
        response = self._serial.read(1)
        if response == ACK:
            return True
        # empty response can happen on a timeout, which we interpret as
        # "not alive"
        elif response == NAK or response == b'':
            return False
        else:
            raise InvalidResponseException()

    def alive(self):
        """Request an 'alive' signal from the microcontroller."""
        self._serial.write(ALIVE)
        response = self._serial.read(1)
        if response == ACK:
            return True
        # empty response can happen on a timeout, which we interpret as
        # "not alive"
        elif response == NAK or response == b'':
            return False
        else:
            raise InvalidResponseException()

    def echo(self, byte):
        """Receive back the same byte that was sent to the ATtiny."""
        if len(byte) != 1:
            raise InvalidLengthException()
        self._serial.write(ECHO + byte)
        response = self._serial.read(1)
        if response != byte:
            raise InvalidResponseException()
        return response

    def stop_motors(self):
        """Immediately stops both motors."""
        self._serial.write(STOP_MOTORS)
        response = self._serial.read(1)
        if response == ACK:
            return True
        # empty response can happen on a timeout, which we interpret as
        # "not alive"
        elif response == NAK or response == b'':
            return False
        else:
            raise InvalidResponseException()

    def set_motors(self, left, right):
        """
        Set the speed of both motors.

        Values can be in the range [-127, 127]. Speeds higher than that will be
        set to their respective extreme. Negative values turn the wheel
        backwards, positive values turn it forwards. Zero stops the motor.
        Higher absolute values mean higher speed.
        """
        left = _clamp(left, MOTOR_MIN, MOTOR_MAX)
        right = _clamp(right, MOTOR_MIN, MOTOR_MAX)
        left_bytes = left.to_bytes(1, 'big', signed=True)
        right_bytes = right.to_bytes(1, 'big', signed=True)

        self._serial.write(SET_BOTH_MOTORS + left_bytes + right_bytes)

        response = self._serial.read(1)
        if response == ACK:
            return True
        elif response == NAK or response == b'':
            return False
        else:
            raise InvalidResponseException()

    def set_left_motor(self, speed):
        """
        Set the speed of the left motor.

        Values can be in the range [-127, 127]. Speeds higher than that will be
        set to their respective extreme. Negative values turn the wheel
        backwards, positive values turn it forwards. Zero stops the motor.
        Higher absolute values mean higher speed.
        """
        speed = _clamp(speed, MOTOR_MIN, MOTOR_MAX)
        speed_bytes = speed.to_bytes(1, 'big', signed=True)

        self._serial.write(SET_LEFT_MOTOR + speed_bytes)

        response = self._serial.read(1)
        if response == ACK:
            return True
        elif response == NAK or response == b'':
            return False
        else:
            raise InvalidResponseException()

    def set_right_motor(self, speed):
        """
        Set the speed of the right motor.

        Values can be in the range [-127, 127]. Speeds higher than that will be
        set to their respective extreme. Negative values turn the wheel
        backwards, positive values turn it forwards. Zero stops the motor.
        Higher absolute values mean higher speed.
        """
        speed = _clamp(speed, MOTOR_MIN, MOTOR_MAX)
        speed_bytes = speed.to_bytes(1, 'big', signed=True)

        self._serial.write(SET_RIGHT_MOTOR + speed_bytes)

        response = self._serial.read(1)
        if response == ACK:
            return True
        elif response == NAK or response == b'':
            return False
        else:
            raise InvalidResponseException()

    def set_pi_parameters(self, p, i, encoder_scale):
        """
        Set the parameters of the PI controller.

        `p` and `i` can be in the range of signed 16-bit integers
        (-32768, 32767). encoder_scale can be in the range of an unsigned 8-bit
        integer.
        """
        p = _clamp(p, -32768, 32767)
        i = _clamp(i, -32768, 32767)
        encoder_scale = _clamp(encoder_scale, 0, 255)

        p_encoded = p.to_bytes(2, 'big', signed=True)
        i_encoded = i.to_bytes(2, 'big', signed=True)
        scale_encoded = encoder_scale.to_bytes(1, 'big')

        message = SET_PI_PARAMETERS + p_encoded + i_encoded + scale_encoded

        self._serial.write(message)

        response = self._serial.read(1)
        if response == ACK:
            return True
        elif response == NAK or response == b'':
            return False
        else:
            raise InvalidResponseException()

    FREQUENCY_MIN = 0
    FREQUENCY_MAX = 2 ** 16 - 1

    DURATION_MIN = 0
    DURATION_MAX = 2 ** 16 - 1

    VOLUME_MIN = 0
    VOLUME_MAX = 15

    def set_buzzer(self, frequency, duration, volume):
        """
        Play a tone on the buzzer.

        Plays a tone on the buzzer for the given duration with a given volume.
        frequency is in the range (0, 65535), excessive values will be clipped.
        duration is in milliseconds and in the range (0, 65535), excessive
        values will be clipped.
        volume is in the range (0, 15), excessive values will be clipped.
        """
        frequency = _clamp(frequency, self.FREQUENCY_MIN, self.FREQUENCY_MAX)
        duration = _clamp(duration, self.DURATION_MIN, self.DURATION_MAX)
        volume = _clamp(volume, self.VOLUME_MIN, self.VOLUME_MAX)

        frequency_encoded = frequency.to_bytes(2, 'big')
        duration_encoded = duration.to_bytes(2, 'big')
        volume_encoded = volume.to_bytes(1, 'big')

        message = SET_BUZZER + frequency_encoded + duration_encoded + volume_encoded

        self._serial.write(message)

        response = self._serial.read(1)
        if response == ACK:
            return True
        elif response == NAK or response == b'':
            return False
        else:
            raise InvalidResponseException()

    def stop_buzzer(self):
        """Stop any buzzer sound."""
        self._serial.write(STOP_BUZZER)

        response = self._serial.read(1)
        if response == ACK:
            return True
        elif response == NAK or response == b'':
            return False
        else:
            raise InvalidResponseException()
<|endoftext|>


<commit_before># coding: utf-8
import re
import logging
import time
import functools

from . import httpbroker
from . import exceptions

logger = logging.getLogger(__name__)

ITEMS_PER_REQUEST = 20
API_VERSIONS = ('v1',)
RESOURCE_PATH_PATTERN = re.compile(r'/api/(\w+)/(\w+)/(\d+)/')

class Connector(object):
    """
    Encapsulates the HTTP requests layer.

    :param username: (optional) valid username that has access to ratchet.scielo.org. Mandatory for POST actions.
    :param api_key: (optional) its respective api key. Mandatory for POST actions.
    :param api_uri: (optional) if connecting to a non official instance of `Ratchet SciELO Manager <https://github.com/scieloorg/ratchet>`_
    :param version: (optional) by default the newest version is used.
    :param http_broker: (optional) a module to deal with http stuff. The reference API is implemented at :mod:`scieloapi.httpbroker`.
    :param check_ca: (optional) if certification authority should be checked during ssl sessions. Defaults to `False`.
    """
    # caches endpoints definitions
    _cache = {}

    def __init__(self, username=None, api_key=None, api_uri=None,
                 version=None, http_broker=None, check_ca=False):
        # dependencies
        self._time = time

        if http_broker:
            _httpbroker = http_broker
        else:
            _httpbroker = httpbroker  # module

        # setup
        self.check_ca = check_ca
        self.api_uri = api_uri if api_uri else r'http://ratchet.scielo.org/api/'

        if version:
            if version in API_VERSIONS:
                self.version = version
            else:
                raise ValueError('unsupported api version. supported are: %s' % ', '.join(API_VERSIONS))
        else:
            self.version = sorted(API_VERSIONS)[-1]

        self.username = username
        self.api_uri = self.api_uri + self.version + '/'

        # dynamic http methods creation
        self._create_http_methods(_httpbroker, username, api_key)

    def _create_http_methods(self, broker, username, api_key):
        """
        Dynamically adds http methods bound to user credentials.

        :param broker: reference to the module to be used as http broker.
        :param username: valid username that has access to ratchet.scielo.org. Mandatory for POST actions.
        :param api_key: its respective api key. Mandatory for POST actions.
        """
        bound_get = functools.partial(broker.get, auth=(username, api_key),
            check_ca=self.check_ca)

        setattr(self, '_http_get', bound_get)

    def fetch_data(self, endpoint,
                         resource_id=None,
                         **kwargs):
        """
        Fetches the specified resource from the SciELO Manager API.

        :param endpoint: a valid endpoint at http://ratchet.scielo.org/api/v1/
        :param resource_id: (optional) an int representing the document.
        :param \*\*kwargs: (optional) params to be passed as query string.
        """
        err_count = 0

        while True:
            try:
                response = self._http_get(self.api_uri,
                                          endpoint=endpoint,
                                          resource_id=resource_id,
                                          params=kwargs)

            except (exceptions.ConnectionError, exceptions.ServiceUnavailable) as e:
                if err_count < 10:
                    wait_secs = err_count * 5
                    logger.info('%s. Waiting %ss to retry.' % (e, wait_secs))
                    self._time.sleep(wait_secs)
                    err_count += 1
                    continue
                else:
                    logger.error('%s. Unable to connect to resource.' % e)
                    raise
            else:
                # restart error count
                err_count = 0
                return response

    def iter_docs(self, endpoint, **kwargs):
        """
        Iterates over all documents of a given endpoint and collection.

        :param endpoint: must be a valid endpoint at http://ratchet.scielo.org/api/v1/
        :param \*\*kwargs: are passed thru the request as query string params
        """
        offset = 0
        limit = ITEMS_PER_REQUEST

        qry_params = {'limit': limit}
        qry_params.update(kwargs)

        while True:
            qry_params.update({'offset': offset})
            doc = self.fetch_data(endpoint, **qry_params)

            for obj in doc['objects']:
                # we are interested only in non-trashed items.
                if obj.get('is_trashed'):
                    continue

                yield obj

            if not doc['meta']['next']:
                raise StopIteration()
            else:
                offset += ITEMS_PER_REQUEST

    def get_endpoints(self):
        """
        Get all endpoints available for the given API version.
        """
        cls = self.__class__

        if self.version not in cls._cache:
            cls._cache[self.version] = self._http_get(self.api_uri)

        return cls._cache[self.version]

class Endpoint(object):
    """
    Represents an API endpoint.

    :param name: the endpoint name.
    :param connector: instance of :class:`Connector`.
    """
    def __init__(self, name, connector):
        self.name = name
        self.connector = connector

    def get(self, resource_id):
        """
        Gets a specific document of the endpoint.

        :param resource_id: an int representing the document.
        """
        res = self.connector.fetch_data(self.name, resource_id=resource_id)
        return res

    def all(self):
        """
        Gets all documents of the endpoint.
        """
        return self.connector.iter_docs(self.name)

class Client(object):
    """
    Collection of :class:`Endpoint` made available in an object oriented fashion.

    An instance of Client tries to figure out the available endpoints
    for the version of the API the Client is instantiated for, and
    automatically instantiates :class:`Endpoint` for each one.
    If ``version`` is missing, the newest available will be used.

    :param username: (optional) valid username that has access to ratchet.scielo.org. Mandatory for POST actions.
    :param api_key: (optional) its respective api key. Mandatory for POST actions.
    :param api_uri: (optional) if connecting to a non official instance of `Ratchet SciELO <https://github.com/scieloorg/ratchet>`_
    :param version: (optional) by default the newest version is used.
    :param check_ca: (optional) if certification authority should be checked during ssl sessions. Defaults to `False`.

    Usage::

        >>> import scieloapi
        >>> cli = scieloapi.Client('some.user', 'some.apikey')
        <scieloapi.scieloapi.Client object at 0x10726f9d0>
        >>> cli.query('journals').all()
        <generator object iter_docs at 0x10fd59730>
    """
    def __init__(self, username=None, api_key=None, api_uri=None,
                 version=None, connector_dep=Connector, check_ca=False):

        self._connector = connector_dep(username,
                                        api_key,
                                        api_uri=api_uri,
                                        version=version,
                                        check_ca=check_ca)
        self._endpoints = {}
        for ep in self._introspect_endpoints():
            self._endpoints[ep] = Endpoint(ep, self._connector)

    def _introspect_endpoints(self):
        """
        Contact the API server to discover the available endpoints.
        """
        return self._connector.get_endpoints().keys()

    def __getattr__(self, name):
        """
        Missing attributes are assumed to be endpoint lookups.
        i.e. Client.journals.all()
        """
        if name in self._endpoints:
            logger.warning('DEPRECATION WARNING! Use the `query` method for endpoint lookups.')
            return self._endpoints[name]
        else:
            raise AttributeError()

    @property
    def endpoints(self):
        """
        Lists all available endpoints for the api version
        the instance of :class:`Client` was created to interact.
        """
        return self._endpoints.keys()

    @property
    def version(self):
        """
        The API version the Client instance is interfacing with.
        """
        return self._connector.version

    def fetch_relations(self, dataset, only=None):
        """
        Fetches all records that relates to `dataset`.

        Its important to note that only first-level relations will be fetched
        in order to avoid massive data retrieval.

        :param dataset: datastructure representing a record. Tipically a `dict` instance.
        :param only: (optional) a collection of relations to fetch. By default, all relations are retrieved.

        Usage::

            >>> import scieloapi
            >>> cli = scieloapi.Client('some.user', 'some.apikey')
            >>> cli.fetch_relations(cli.journals.get(70))
        """
        new_dataset = {}

        for attr_name, attr_value in dataset.items():
            # skip fetching itself and undesired fields
            if attr_name == 'resource_uri' or (only and attr_name not in only):
                new_dataset[attr_name] = attr_value
            elif isinstance(attr_value, basestring):
                try:
                    new_dataset[attr_name] = self.get(attr_value)
                except ValueError as e:
                    new_dataset[attr_name] = attr_value

            elif isinstance(attr_value, list):
                new_elems = []
                for elem in attr_value:
                    try:
                        new_elems.append(self.get(elem))
                    except (TypeError, ValueError) as e:
                        new_elems.append(elem)

                new_dataset[attr_name] = new_elems
            else:
                new_dataset[attr_name] = attr_value

        return new_dataset

    def get(self, resource_uri):
        """
        Gets resource_uri.

        Gets the given resource in a opinionated fashion in terms of the
        `version` passed during client's instantiation. The `endpoint` must also
        be available for the version the client is bound to.

        :param resource_uri: text string in the form `/api/<version>/<endpoint>/<resource_id>/`.
        """
        match = RESOURCE_PATH_PATTERN.match(resource_uri)
        if match:
            version, endpoint, resource_id = match.groups()

            if version != self.version:
                raise ValueError('Resource and Client version must match')

            return self.query(endpoint).get(resource_id)
        else:
            raise ValueError('Invalid resource_uri')

    def query(self, endpoint):
        """
        Query an endpoint.

        :param endpoint: string of the endpoint's name. A complete list of
        valid endpoints can be got at :attr:`Client.endpoints`.
        """
        if endpoint in self._endpoints:
            return self._endpoints[endpoint]
        else:
            raise ValueError('Unknown endpoint %s.' % endpoint)
<commit_msg>Incluindo metodo filter<commit_after># coding: utf-8
import re
import logging
import time
import functools

from . import httpbroker
from . import exceptions

logger = logging.getLogger(__name__)

ITEMS_PER_REQUEST = 20
API_VERSIONS = ('v1',)
RESOURCE_PATH_PATTERN = re.compile(r'/api/(\w+)/(\w+)/(\d+)/')

class Connector(object):
    """
    Encapsulates the HTTP requests layer.

    :param username: (optional) valid username that has access to ratchet.scielo.org. Mandatory for POST actions.
    :param api_key: (optional) its respective api key. Mandatory for POST actions.
    :param api_uri: (optional) if connecting to a non official instance of `Ratchet SciELO Manager <https://github.com/scieloorg/ratchet>`_
    :param version: (optional) by default the newest version is used.
    :param http_broker: (optional) a module to deal with http stuff. The reference API is implemented at :mod:`scieloapi.httpbroker`.
    :param check_ca: (optional) if certification authority should be checked during ssl sessions. Defaults to `False`.
    """
    # caches endpoints definitions
    _cache = {}

    def __init__(self, username=None, api_key=None, api_uri=None,
                 version=None, http_broker=None, check_ca=False):
        # dependencies
        self._time = time

        if http_broker:
            _httpbroker = http_broker
        else:
            _httpbroker = httpbroker  # module

        # setup
        self.check_ca = check_ca
        self.api_uri = api_uri if api_uri else r'http://ratchet.scielo.org/api/'

        if version:
            if version in API_VERSIONS:
                self.version = version
            else:
                raise ValueError('unsupported api version. supported are: %s' % ', '.join(API_VERSIONS))
        else:
            self.version = sorted(API_VERSIONS)[-1]

        self.username = username
        self.api_uri = self.api_uri + self.version + '/'

        # dynamic http methods creation
        self._create_http_methods(_httpbroker, username, api_key)

    def _create_http_methods(self, broker, username, api_key):
        """
        Dynamically adds http methods bound to user credentials.

        :param broker: reference to the module to be used as http broker.
        :param username: valid username that has access to ratchet.scielo.org. Mandatory for POST actions.
        :param api_key: its respective api key. Mandatory for POST actions.
        """
        bound_get = functools.partial(broker.get, auth=(username, api_key),
            check_ca=self.check_ca)

        setattr(self, '_http_get', bound_get)

    def fetch_data(self, endpoint,
                         resource_id=None,
                         **kwargs):
        """
        Fetches the specified resource from the SciELO Manager API.

        :param endpoint: a valid endpoint at http://ratchet.scielo.org/api/v1/
        :param resource_id: (optional) an int representing the document.
        :param \*\*kwargs: (optional) params to be passed as query string.
        """
        err_count = 0

        while True:
            try:
                response = self._http_get(self.api_uri,
                                          endpoint=endpoint,
                                          resource_id=resource_id,
                                          params=kwargs)

            except (exceptions.ConnectionError, exceptions.ServiceUnavailable) as e:
                if err_count < 10:
                    wait_secs = err_count * 5
                    logger.info('%s. Waiting %ss to retry.' % (e, wait_secs))
                    self._time.sleep(wait_secs)
                    err_count += 1
                    continue
                else:
                    logger.error('%s. Unable to connect to resource.' % e)
                    raise
            else:
                # restart error count
                err_count = 0
                return response

    def iter_docs(self, endpoint, **kwargs):
        """
        Iterates over all documents of a given endpoint and collection.

        :param endpoint: must be a valid endpoint at http://ratchet.scielo.org/api/v1/
        :param \*\*kwargs: are passed thru the request as query string params
        """
        offset = 0
        limit = ITEMS_PER_REQUEST

        qry_params = {'limit': limit}
        qry_params.update(kwargs)

        while True:
            qry_params.update({'offset': offset})
            doc = self.fetch_data(endpoint, **qry_params)

            for obj in doc['objects']:
                # we are interested only in non-trashed items.
                if obj.get('is_trashed'):
                    continue

                yield obj

            if not doc['meta']['next']:
                raise StopIteration()
            else:
                offset += ITEMS_PER_REQUEST

    def get_endpoints(self):
        """
        Get all endpoints available for the given API version.
        """
        cls = self.__class__

        if self.version not in cls._cache:
            cls._cache[self.version] = self._http_get(self.api_uri)

        return cls._cache[self.version]

class Endpoint(object):
    """
    Represents an API endpoint.

    :param name: the endpoint name.
    :param connector: instance of :class:`Connector`.
    """
    def __init__(self, name, connector):
        self.name = name
        self.connector = connector

    def get(self, resource_id):
        """
        Gets a specific document of the endpoint.

        :param resource_id: an int representing the document.
        """
        res = self.connector.fetch_data(self.name, resource_id=resource_id)
        return res

    def all(self):
        """
        Gets all documents of the endpoint.
        """
        return self.connector.iter_docs(self.name)

    def filter(self, **kwargs):
        """
        Gets all documents of the endpoint that satisfies some criteria.

        :param \*\*kwargs: filtering criteria as documented at `docs.scielo.org <http://ref.scielo.org/ph6gvk>`_
        """
        return self.connector.iter_docs(self.name, **kwargs)

class Client(object):
    """
    Collection of :class:`Endpoint` made available in an object oriented fashion.

    An instance of Client tries to figure out the available endpoints
    for the version of the API the Client is instantiated for, and
    automatically instantiates :class:`Endpoint` for each one.
    If ``version`` is missing, the newest available will be used.

    :param username: (optional) valid username that has access to ratchet.scielo.org. Mandatory for POST actions.
    :param api_key: (optional) its respective api key. Mandatory for POST actions.
    :param api_uri: (optional) if connecting to a non official instance of `Ratchet SciELO <https://github.com/scieloorg/ratchet>`_
    :param version: (optional) by default the newest version is used.
    :param check_ca: (optional) if certification authority should be checked during ssl sessions. Defaults to `False`.

    Usage::

        >>> import scieloapi
        >>> cli = scieloapi.Client('some.user', 'some.apikey')
        <scieloapi.scieloapi.Client object at 0x10726f9d0>
        >>> cli.query('journals').all()
        <generator object iter_docs at 0x10fd59730>
    """
    def __init__(self, username=None, api_key=None, api_uri=None,
                 version=None, connector_dep=Connector, check_ca=False):

        self._connector = connector_dep(username,
                                        api_key,
                                        api_uri=api_uri,
                                        version=version,
                                        check_ca=check_ca)
        self._endpoints = {}
        for ep in self._introspect_endpoints():
            self._endpoints[ep] = Endpoint(ep, self._connector)

    def _introspect_endpoints(self):
        """
        Contact the API server to discover the available endpoints.
        """
        return self._connector.get_endpoints().keys()

    def __getattr__(self, name):
        """
        Missing attributes are assumed to be endpoint lookups.
        i.e. Client.journals.all()
        """
        if name in self._endpoints:
            logger.warning('DEPRECATION WARNING! Use the `query` method for endpoint lookups.')
            return self._endpoints[name]
        else:
            raise AttributeError()

    @property
    def endpoints(self):
        """
        Lists all available endpoints for the api version
        the instance of :class:`Client` was created to interact.
        """
        return self._endpoints.keys()

    @property
    def version(self):
        """
        The API version the Client instance is interfacing with.
        """
        return self._connector.version

    def fetch_relations(self, dataset, only=None):
        """
        Fetches all records that relates to `dataset`.

        Its important to note that only first-level relations will be fetched
        in order to avoid massive data retrieval.

        :param dataset: datastructure representing a record. Tipically a `dict` instance.
        :param only: (optional) a collection of relations to fetch. By default, all relations are retrieved.

        Usage::

            >>> import scieloapi
            >>> cli = scieloapi.Client('some.user', 'some.apikey')
            >>> cli.fetch_relations(cli.journals.get(70))
        """
        new_dataset = {}

        for attr_name, attr_value in dataset.items():
            # skip fetching itself and undesired fields
            if attr_name == 'resource_uri' or (only and attr_name not in only):
                new_dataset[attr_name] = attr_value
            elif isinstance(attr_value, basestring):
                try:
                    new_dataset[attr_name] = self.get(attr_value)
                except ValueError as e:
                    new_dataset[attr_name] = attr_value

            elif isinstance(attr_value, list):
                new_elems = []
                for elem in attr_value:
                    try:
                        new_elems.append(self.get(elem))
                    except (TypeError, ValueError) as e:
                        new_elems.append(elem)

                new_dataset[attr_name] = new_elems
            else:
                new_dataset[attr_name] = attr_value

        return new_dataset

    def get(self, resource_uri):
        """
        Gets resource_uri.

        Gets the given resource in a opinionated fashion in terms of the
        `version` passed during client's instantiation. The `endpoint` must also
        be available for the version the client is bound to.

        :param resource_uri: text string in the form `/api/<version>/<endpoint>/<resource_id>/`.
        """
        match = RESOURCE_PATH_PATTERN.match(resource_uri)
        if match:
            version, endpoint, resource_id = match.groups()

            if version != self.version:
                raise ValueError('Resource and Client version must match')

            return self.query(endpoint).get(resource_id)
        else:
            raise ValueError('Invalid resource_uri')

    def query(self, endpoint):
        """
        Query an endpoint.

        :param endpoint: string of the endpoint's name. A complete list of
        valid endpoints can be got at :attr:`Client.endpoints`.
        """
        if endpoint in self._endpoints:
            return self._endpoints[endpoint]
        else:
            raise ValueError('Unknown endpoint %s.' % endpoint)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# this file is part of 'RAX-AutoScaler'
#
# Copyright 2014 Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import common
import pyrax
import argparse
import time
import os
import sys
import logging.config
import random
from colouredconsolehandler import ColouredConsoleHandler
from auth import Auth
import cloudmonitor
import subprocess
from version import return_version

# CHECK logging.conf
logging_config = common.check_file('logging.conf')

if logging_config is None:
    logging.handlers.ColouredConsoleHandler = ColouredConsoleHandler
    logging.basicConfig(level=logging.INFO)
    logger = logging.getLogger(__name__)

else:

    logging_conf_file = logging_config
    logging.handlers.ColouredConsoleHandler = ColouredConsoleHandler
    logging.config.fileConfig(logging_conf_file)
    logger = logging.getLogger(__name__)

def exit_with_error(msg):
    """This function prints error message and exit with error.

    :param msg: error message
    :type name: str
    :returns: 1 (int) -- the return code

    """
    if msg is None:
        try:
            log_file = logger.root.handlers[0].baseFilename
            logger.info('completed with an error: %s' % log_file)
        except:
            print ('(info) rax-autoscale completed with an error')
    else:
        try:
            logger.error(msg)
            log_file = logger.root.handlers[0].baseFilename
            logger.info('completed with an error: %s' % log_file)
        except:
            print ('(error) %s' % msg)
            print ('(info) rax-autoscale completed with an error')

    exit(1)

def is_node_master(scalingGroup):
    """This function checks scaling group state and determines if node is a master.

    :param scalingGroup: data about servers in scaling group retrieve from
                         cloudmonitor
    :returns: 1 : if cluster state is unknown
              2 : node is a master
              3 : node is not a master

    """
    masters = []
    systemUUID = None
    systemUUID_file = None
    node_id = None
    logger.debug('executing dmidecode to get system uuid')
    try:
        sysout = subprocess.Popen(['dmidecode', '-s', 'system-uuid'],
                                  stdout=subprocess.PIPE)
        systemUUID = sysout.communicate()[0]
        systemUUID = systemUUID.strip()
    except Exception, e:
        logger.debug('Failed to execute dmidecode: %s' % str(e))
        pass

    if systemUUID is not None:
        logger.debug("Checking if file '%s' exists " % systemUUID)
        systemUUID_file = common.check_file(systemUUID)
        if systemUUID_file is not None:
            logger.debug("Reading file '%s' to get node id" % systemUUID_file)
            try:
                rfh = open(systemUUID_file, 'r').read()
                node_id = rfh.strip()
                logger.info("Using machine uuid available in '%s'"
                            % systemUUID_file)
            except:
                logger.warning("Unable to read a file '%s' in '%s'"
                               % (systemUUID, '/etc/rax-autoscaler'))
                pass

    if node_id is None:
        logger.info('Launching xenstore query to get machine uuid')
        node_id = common.get_machine_uuid()

        if systemUUID_file is None and systemUUID is not None:
            logger.debug("Creating file '%s' to save node id"
                         % systemUUID_file)
            try:
                newFile = '/etc/rax-autoscaler/' + systemUUID
                wfh = open(newFile, 'w')
                wfh.write(node_id)
                wfh.close()
            except Exception, e:
                logger.warning("Unable to create a file '%s': '%s'"
                               % (systemUUID, str(e)))
                pass

    sg_state = scalingGroup.get_state()
    if len(sg_state['active']) == 1:
        masters.append(sg_state['active'][0])
    elif len(sg_state['active']) > 1:
        masters.append(sg_state['active'][0])
        masters.append(sg_state['active'][1])
    else:
        logger.error('Unknown cluster state')
        return 1

    if node_id in masters:
        logger.info('Node is a master, continuing')
        return 2
    else:
        logger.info('Node is not a master, nothing to do. Exiting')
        return

def get_scaling_group(group, config_data):
    """This function checks and gets active servers in scaling group

    :param group: group name
    :param config_data: json configuration data
    :returns: scalingGroup if server state is active else null

    """
    group_id = common.get_group_value(config_data, group, 'group_id')
    if group_id is None:
        logger.error('Unable to get group_id from json file')
        return

    scalingGroup = cloudmonitor.scaling_group_servers(group_id)
    if scalingGroup is None:
        return
    # Check active server(s) in scaling group
    if len(scalingGroup.get_state()['active']) == 0:
        return
    else:
        logger.info('Server(s) in scaling group: %s' %
                    ', '.join(['(%s, %s)'
                              % (cloudmonitor.get_server_name(s_id), s_id)
                              for s_id in scalingGroup.get_state()['active']]))
    logger.info('Current Active Servers: ' +
                str(scalingGroup.get_state()['active_capacity']))
    return scalingGroup

def autoscale(group, config_data, args):
    """This function executes scale up or scale down policy

    :param group: group name
    :param config_data: json configuration data
    :param args: user provided arguments

    """
    au = pyrax.autoscale

    scalingGroup = get_scaling_group(group, config_data)
    if scalingGroup is None:
        return 1
    check_type = common.get_group_value(config_data, group, 'check_type')
    if check_type is None:
        return 1
    check_config = common.get_group_value(config_data, group, 'check_config')
    if check_config is None:
        return 1

    for s_id in scalingGroup.get_state()['active']:
        rv = cloudmonitor.add_cm_check(s_id, check_type, check_config)

    logger.info('Cluster Mode Enabled: %s' % str(args['cluster']))

    if args['cluster']:
        rv = is_node_master(scalingGroup)
        if rv is None:
            # Not a master, no need to proceed further
            return
        if rv == 1:
            # Cluster state unknown return error.
            return 1

    # Gather cluster statistics
    check_type = common.get_group_value(config_data, group, 'check_type')
    if check_type is None:
        check_type = 'agent.load_average'

    metric_name = common.get_group_value(config_data, group, 'metric_name')
    if metric_name is None:
        metric_name = '1m'

    logger.info('Gathering Monitoring Data')

    results = []
    cm = pyrax.cloud_monitoring
    # Get all CloudMonitoring entities on the account
    entities = cm.list_entities()

    # Shuffle entities so the sample uses different servers
    entities = random.sample(entities, len(entities))

    for ent in entities:
        # Check if the entity is also in the scaling group
        if ent.agent_id in scalingGroup.get_state()['active']:
            ent_checks = ent.list_checks()
            # Loop through checks to find checks of the correct type
            for check in ent_checks:
                if check.type == check_type:
                    data = check.get_metric_data_points(metric_name,
                                                        int(time.time())-300,
                                                        int(time.time()),
                                                        points=2)
                    if len(data) > 0:
                        point = len(data)-1
                        logger.info('Found metric for: ' + ent.name +
                                    ', value: ' + str(data[point]['average']))
                        results.append(float(data[point]['average']))
                        break

        # Restrict number of data points to save on API calls
        if len(results) >= args['max_sample']:
            logger.info('--max-sample value of ' + str(args['max_sample']) +
                        ' reached, not gathering any more statistics')
            break

    if len(results) == 0:
        logger.error('No data available')
        return 1
    else:
        average = sum(results)/len(results)
        scale_up_threshold = common.get_group_value(config_data, group,
                                                    'scale_up_threshold')
        if scale_up_threshold is None:
            scale_up_threshold = 0.6

    scale_down_threshold = common.get_group_value(config_data, group,
                                                  'scale_down_threshold')
    if scale_down_threshold is None:
        scale_down_threshold = 0.4

    logger.info('Cluster average for ' + check_type +
                '(' + metric_name + ') at: ' + str(average))

    if average > scale_up_threshold:
        try:
            logger.info('Above Threshold - Scaling Up')
            scale_policy_id = common.get_group_value(config_data, group,
                                                     'scale_up_policy')
            scale_policy = scalingGroup.get_policy(scale_policy_id)
            if not args['dry_run']:
                common.webhook_call(config_data, group, 'scale_up', 'pre')
                scale_policy.execute()
                common.webhook_call(config_data, group, 'scale_up', 'post')
            else:
                logger.info('Scale up prevented by --dry-run')
                logger.info('Scale up policy executed ('
                            + scale_policy_id + ')')
        except Exception, e:
            logger.warning('Scale up: %s' % str(e))
    elif average < scale_down_threshold:
        try:
            logger.info('Below Threshold - Scaling Down')
            scale_policy_id = common.get_group_value(config_data, group,
                                                     'scale_down_policy')
            scale_policy = scalingGroup.get_policy(scale_policy_id)
            if not args['dry_run']:
                common.webhook_call(config_data, group, 'scale_down', 'pre')
                scale_policy.execute()
                common.webhook_call(config_data, group, 'scale_down', 'post')
            else:
                logger.info('Scale down prevented by --dry-run')
                logger.info('Scale down policy executed (' +
                            scale_policy_id + ')')

        except Exception, e:
            logger.warning('Scale down: %s' % str(e))

    else:
        logger.info('Cluster within target paramters')

def parse_args():
    """This function validates user arguments and data in configuration file.

    """
    parser = argparse.ArgumentParser()
    parser.add_argument('--as-group', required=False,
                        help='The autoscale group config ID')
    parser.add_argument('--os-username', required=False,
                        help='Rackspace Cloud user name')
    parser.add_argument('--os-password', required=False,
                        help='Rackspace Cloud account API key')
    parser.add_argument('--config-file', required=False, default='config.json',
                        help='The autoscale configuration .ini file'
                             '(default: config.json)'),
    parser.add_argument('--os-region-name', required=False,
                        help='The region to build the servers',
                        choices=['SYD', 'HKG', 'DFW', 'ORD', 'IAD', 'LON'])
    parser.add_argument('--cluster', required=False,
                        default=False, action='store_true')
    parser.add_argument('--version', action='version',
                        help='Show version number', version=return_version())
    parser.add_argument('--dry-run', required=False, default=False,
                        action='store_true',
                        help='Do not actually perform any scaling operations '
                             'or call webhooks')
    parser.add_argument('--max-sample', required=False, default=10, type=int,
                        help='Maximum number of servers to obtain monitoring '
                             'samples from')
    args = vars(parser.parse_args())

    return args

def main():
    """This function calls auth class for authentication and autoscale to
       execute scaling policy

    """

    args = parse_args()

    # CONFIG.ini
    config_file = common.check_file(args['config_file'])
    if config_file is None:
        exit_with_error("Either file is missing or is not readable: '%s'"
                        % args['config_file'])

    # Show Version
    logger.info(return_version())

    for arg in args:
        logger.debug('argument provided by user ' + arg + ' : ' +
                     str(args[arg]))

    # Get data from config.json
    config_data = common.get_config(config_file)
    if config_data is None:
        exit_with_error('Failed to read config file: ' + config_file)

    # Get group
    if not args['as_group']:
        if len(config_data['autoscale_groups'].keys()) == 1:
            as_group = config_data['autoscale_groups'].keys()[0]
        else:
            logger.debug("Getting system hostname")
            try:
                sysout = subprocess.Popen(['hostname'], stdout=subprocess.PIPE)
                hostname = (sysout.communicate()[0]).strip()
                if '-' in hostname:
                    hostname = hostname.rsplit('-', 1)[0]

                group_value = config_data["autoscale_groups"][hostname]
                as_group = hostname
            except Exception, e:
                logger.debug("Failed to get hostname: %s" % str(e))
                logger.warning("Multiple group found in config file, "
                               "please use 'as-group' option")
                exit_with_error('Unable to identify targeted group')
    else:
        try:
            group_value = config_data["autoscale_groups"][args['as_group']]
            as_group = args['as_group']
        except:
            exit_with_error("Unable to find group '" + args['as_group'] +
                            "' in " + config_file)

    username = common.get_user_value(args, config_data, 'os_username')
    if username is None:
        exit_with_error(None)
    api_key = common.get_user_value(args, config_data, 'os_password')
    if api_key is None:
        exit_with_error(None)
    region = common.get_user_value(args, config_data, 'os_region_name')
    if region is None:
        exit_with_error(None)

    session = Auth(username, api_key, region)

    if session.authenticate() is True:
        rv = autoscale(as_group, config_data, args)
        if rv is None:
            log_file = None
            if hasattr(logger.root.handlers[0], 'baseFilename'):
                log_file = logger.root.handlers[0].baseFilename
            if log_file is None:
                logger.info('completed successfull')
            else:
                logger.info('completed successfully: %s' % log_file)
        else:
            exit_with_error(None)
    else:
        exit_with_error('Authentication failed')

if __name__ == '__main__':
    main()
<commit_msg>Bug: #83 update documentation to address latest code update.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# this file is part of 'RAX-AutoScaler'
#
# Copyright 2014 Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import common
import pyrax
import argparse
import time
import os
import sys
import logging.config
import random
from colouredconsolehandler import ColouredConsoleHandler
from auth import Auth
import cloudmonitor
import subprocess
from version import return_version

# CHECK logging.conf
logging_config = common.check_file('logging.conf')

if logging_config is None:
    logging.handlers.ColouredConsoleHandler = ColouredConsoleHandler
    logging.basicConfig(level=logging.INFO)
    logger = logging.getLogger(__name__)

else:

    logging_conf_file = logging_config
    logging.handlers.ColouredConsoleHandler = ColouredConsoleHandler
    logging.config.fileConfig(logging_conf_file)
    logger = logging.getLogger(__name__)

def exit_with_error(msg):
    """This function prints error message and exit with error.

    :param msg: error message
    :type name: str
    :returns: 1 (int) -- the return code

    """
    if msg is None:
        try:
            log_file = logger.root.handlers[0].baseFilename
            logger.info('completed with an error: %s' % log_file)
        except:
            print ('(info) rax-autoscale completed with an error')
    else:
        try:
            logger.error(msg)
            log_file = logger.root.handlers[0].baseFilename
            logger.info('completed with an error: %s' % log_file)
        except:
            print ('(error) %s' % msg)
            print ('(info) rax-autoscale completed with an error')

    exit(1)

def is_node_master(scalingGroup):
    """This function checks scaling group state and determines if node is a master.

    :param scalingGroup: data about servers in scaling group retrieve from
                         cloudmonitor
    :returns: 1     : if cluster state is unknown
              2     : node is a master
              None  : node is not a master

    """
    masters = []
    systemUUID = None
    systemUUID_file = None
    node_id = None
    logger.debug('executing dmidecode to get system uuid')
    try:
        sysout = subprocess.Popen(['dmidecode', '-s', 'system-uuid'],
                                  stdout=subprocess.PIPE)
        systemUUID = sysout.communicate()[0]
        systemUUID = systemUUID.strip()
    except Exception, e:
        logger.debug('Failed to execute dmidecode: %s' % str(e))
        pass

    if systemUUID is not None:
        logger.debug("Checking if file '%s' exists " % systemUUID)
        systemUUID_file = common.check_file(systemUUID)
        if systemUUID_file is not None:
            logger.debug("Reading file '%s' to get node id" % systemUUID_file)
            try:
                rfh = open(systemUUID_file, 'r').read()
                node_id = rfh.strip()
                logger.info("Using machine uuid available in '%s'"
                            % systemUUID_file)
            except:
                logger.warning("Unable to read a file '%s' in '%s'"
                               % (systemUUID, '/etc/rax-autoscaler'))
                pass

    if node_id is None:
        logger.info('Launching xenstore query to get machine uuid')
        node_id = common.get_machine_uuid()

        if systemUUID_file is None and systemUUID is not None:
            logger.debug("Creating file '%s' to save node id"
                         % systemUUID_file)
            try:
                newFile = '/etc/rax-autoscaler/' + systemUUID
                wfh = open(newFile, 'w')
                wfh.write(node_id)
                wfh.close()
            except Exception, e:
                logger.warning("Unable to create a file '%s': '%s'"
                               % (systemUUID, str(e)))
                pass

    sg_state = scalingGroup.get_state()
    if len(sg_state['active']) == 1:
        masters.append(sg_state['active'][0])
    elif len(sg_state['active']) > 1:
        masters.append(sg_state['active'][0])
        masters.append(sg_state['active'][1])
    else:
        logger.error('Unknown cluster state')
        return 1

    if node_id in masters:
        logger.info('Node is a master, continuing')
        return 2
    else:
        logger.info('Node is not a master, nothing to do. Exiting')
        return

def get_scaling_group(group, config_data):
    """This function checks and gets active servers in scaling group

    :param group: group name
    :param config_data: json configuration data
    :returns: scalingGroup if server state is active else null

    """
    group_id = common.get_group_value(config_data, group, 'group_id')
    if group_id is None:
        logger.error('Unable to get group_id from json file')
        return

    scalingGroup = cloudmonitor.scaling_group_servers(group_id)
    if scalingGroup is None:
        return
    # Check active server(s) in scaling group
    if len(scalingGroup.get_state()['active']) == 0:
        return
    else:
        logger.info('Server(s) in scaling group: %s' %
                    ', '.join(['(%s, %s)'
                              % (cloudmonitor.get_server_name(s_id), s_id)
                              for s_id in scalingGroup.get_state()['active']]))
    logger.info('Current Active Servers: ' +
                str(scalingGroup.get_state()['active_capacity']))
    return scalingGroup

def autoscale(group, config_data, args):
    """This function executes scale up or scale down policy

    :param group: group name
    :param config_data: json configuration data
    :param args: user provided arguments

    """
    au = pyrax.autoscale

    scalingGroup = get_scaling_group(group, config_data)
    if scalingGroup is None:
        return 1
    check_type = common.get_group_value(config_data, group, 'check_type')
    if check_type is None:
        return 1
    check_config = common.get_group_value(config_data, group, 'check_config')
    if check_config is None:
        return 1

    for s_id in scalingGroup.get_state()['active']:
        rv = cloudmonitor.add_cm_check(s_id, check_type, check_config)

    logger.info('Cluster Mode Enabled: %s' % str(args['cluster']))

    if args['cluster']:
        rv = is_node_master(scalingGroup)
        if rv is None:
            # Not a master, no need to proceed further
            return
        if rv == 1:
            # Cluster state unknown return error.
            return 1

    # Gather cluster statistics
    check_type = common.get_group_value(config_data, group, 'check_type')
    if check_type is None:
        check_type = 'agent.load_average'

    metric_name = common.get_group_value(config_data, group, 'metric_name')
    if metric_name is None:
        metric_name = '1m'

    logger.info('Gathering Monitoring Data')

    results = []
    cm = pyrax.cloud_monitoring
    # Get all CloudMonitoring entities on the account
    entities = cm.list_entities()

    # Shuffle entities so the sample uses different servers
    entities = random.sample(entities, len(entities))

    for ent in entities:
        # Check if the entity is also in the scaling group
        if ent.agent_id in scalingGroup.get_state()['active']:
            ent_checks = ent.list_checks()
            # Loop through checks to find checks of the correct type
            for check in ent_checks:
                if check.type == check_type:
                    data = check.get_metric_data_points(metric_name,
                                                        int(time.time())-300,
                                                        int(time.time()),
                                                        points=2)
                    if len(data) > 0:
                        point = len(data)-1
                        logger.info('Found metric for: ' + ent.name +
                                    ', value: ' + str(data[point]['average']))
                        results.append(float(data[point]['average']))
                        break

        # Restrict number of data points to save on API calls
        if len(results) >= args['max_sample']:
            logger.info('--max-sample value of ' + str(args['max_sample']) +
                        ' reached, not gathering any more statistics')
            break

    if len(results) == 0:
        logger.error('No data available')
        return 1
    else:
        average = sum(results)/len(results)
        scale_up_threshold = common.get_group_value(config_data, group,
                                                    'scale_up_threshold')
        if scale_up_threshold is None:
            scale_up_threshold = 0.6

    scale_down_threshold = common.get_group_value(config_data, group,
                                                  'scale_down_threshold')
    if scale_down_threshold is None:
        scale_down_threshold = 0.4

    logger.info('Cluster average for ' + check_type +
                '(' + metric_name + ') at: ' + str(average))

    if average > scale_up_threshold:
        try:
            logger.info('Above Threshold - Scaling Up')
            scale_policy_id = common.get_group_value(config_data, group,
                                                     'scale_up_policy')
            scale_policy = scalingGroup.get_policy(scale_policy_id)
            if not args['dry_run']:
                common.webhook_call(config_data, group, 'scale_up', 'pre')
                scale_policy.execute()
                common.webhook_call(config_data, group, 'scale_up', 'post')
            else:
                logger.info('Scale up prevented by --dry-run')
                logger.info('Scale up policy executed ('
                            + scale_policy_id + ')')
        except Exception, e:
            logger.warning('Scale up: %s' % str(e))
    elif average < scale_down_threshold:
        try:
            logger.info('Below Threshold - Scaling Down')
            scale_policy_id = common.get_group_value(config_data, group,
                                                     'scale_down_policy')
            scale_policy = scalingGroup.get_policy(scale_policy_id)
            if not args['dry_run']:
                common.webhook_call(config_data, group, 'scale_down', 'pre')
                scale_policy.execute()
                common.webhook_call(config_data, group, 'scale_down', 'post')
            else:
                logger.info('Scale down prevented by --dry-run')
                logger.info('Scale down policy executed (' +
                            scale_policy_id + ')')

        except Exception, e:
            logger.warning('Scale down: %s' % str(e))

    else:
        logger.info('Cluster within target paramters')

def parse_args():
    """This function validates user arguments and data in configuration file.

    """
    parser = argparse.ArgumentParser()
    parser.add_argument('--as-group', required=False,
                        help='The autoscale group config ID')
    parser.add_argument('--os-username', required=False,
                        help='Rackspace Cloud user name')
    parser.add_argument('--os-password', required=False,
                        help='Rackspace Cloud account API key')
    parser.add_argument('--config-file', required=False, default='config.json',
                        help='The autoscale configuration .ini file'
                             '(default: config.json)'),
    parser.add_argument('--os-region-name', required=False,
                        help='The region to build the servers',
                        choices=['SYD', 'HKG', 'DFW', 'ORD', 'IAD', 'LON'])
    parser.add_argument('--cluster', required=False,
                        default=False, action='store_true')
    parser.add_argument('--version', action='version',
                        help='Show version number', version=return_version())
    parser.add_argument('--dry-run', required=False, default=False,
                        action='store_true',
                        help='Do not actually perform any scaling operations '
                             'or call webhooks')
    parser.add_argument('--max-sample', required=False, default=10, type=int,
                        help='Maximum number of servers to obtain monitoring '
                             'samples from')
    args = vars(parser.parse_args())

    return args

def main():
    """This function calls auth class for authentication and autoscale to
       execute scaling policy

    """

    args = parse_args()

    # CONFIG.ini
    config_file = common.check_file(args['config_file'])
    if config_file is None:
        exit_with_error("Either file is missing or is not readable: '%s'"
                        % args['config_file'])

    # Show Version
    logger.info(return_version())

    for arg in args:
        logger.debug('argument provided by user ' + arg + ' : ' +
                     str(args[arg]))

    # Get data from config.json
    config_data = common.get_config(config_file)
    if config_data is None:
        exit_with_error('Failed to read config file: ' + config_file)

    # Get group
    if not args['as_group']:
        if len(config_data['autoscale_groups'].keys()) == 1:
            as_group = config_data['autoscale_groups'].keys()[0]
        else:
            logger.debug("Getting system hostname")
            try:
                sysout = subprocess.Popen(['hostname'], stdout=subprocess.PIPE)
                hostname = (sysout.communicate()[0]).strip()
                if '-' in hostname:
                    hostname = hostname.rsplit('-', 1)[0]

                group_value = config_data["autoscale_groups"][hostname]
                as_group = hostname
            except Exception, e:
                logger.debug("Failed to get hostname: %s" % str(e))
                logger.warning("Multiple group found in config file, "
                               "please use 'as-group' option")
                exit_with_error('Unable to identify targeted group')
    else:
        try:
            group_value = config_data["autoscale_groups"][args['as_group']]
            as_group = args['as_group']
        except:
            exit_with_error("Unable to find group '" + args['as_group'] +
                            "' in " + config_file)

    username = common.get_user_value(args, config_data, 'os_username')
    if username is None:
        exit_with_error(None)
    api_key = common.get_user_value(args, config_data, 'os_password')
    if api_key is None:
        exit_with_error(None)
    region = common.get_user_value(args, config_data, 'os_region_name')
    if region is None:
        exit_with_error(None)

    session = Auth(username, api_key, region)

    if session.authenticate() is True:
        rv = autoscale(as_group, config_data, args)
        if rv is None:
            log_file = None
            if hasattr(logger.root.handlers[0], 'baseFilename'):
                log_file = logger.root.handlers[0].baseFilename
            if log_file is None:
                logger.info('completed successfull')
            else:
                logger.info('completed successfully: %s' % log_file)
        else:
            exit_with_error(None)
    else:
        exit_with_error('Authentication failed')

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import os
import json
import requests
import pkg_resources

from pkg_resources import DistributionNotFound

from types import ModuleType

from . import resources, utility

from .errors import (BadRequestError, GatewayError,
                     ProcessingError, ServerError)

# Create a dict of resource classes
RESOURCE_CLASSES = {}
for name, module in resources.__dict__.items():
    if isinstance(module, ModuleType) and name.capitalize() in module.__dict__:
        RESOURCE_CLASSES[name] = module.__dict__[name.capitalize()]

UTILITY_CLASSES = {}
for name, module in utility.__dict__.items():
    if isinstance(module, ModuleType) and name.capitalize() in module.__dict__:
        UTILITY_CLASSES[name] = module.__dict__[name.capitalize()]

class Client:
    """Razorpay client class"""

    DEFAULTS = {
        'base_url': 'https://api.razorpay.com/v1'
    }

    def __init__(self, session=None, auth=None, **options):
        """
        Initialize a Client object with session,
        optional auth handler, and options
        """
        self.session = session or requests.Session()
        self.auth = auth
        file_dir = os.path.dirname(__file__)
        self.cert_path = file_dir + '/ca-bundle.crt'

        # intializes each resource
        # injecting this client object into the constructor
        for name, Klass in RESOURCE_CLASSES.items() + UTILITY_CLASSES.items():
            setattr(self, name, Klass(self))

    def _update_user_agent_header(self, options):
        user_agent = "{}{}".format('Razorpay-Python/',  self._get_version())

        if 'headers' in options:
            options['headers']['User-Agent'] = user_agent
        else:
            options['headers'] = {'User-Agent': user_agent}

        return options

    def _get_version(self):
        version = ""
        try:
            version = pkg_resources.require("razorpay")[0].version
        except DistributionNotFound:
            pass
        return version

    def request(self, method, path, **options):
        """
        Dispatches a request to the Razorpay HTTP API
        """
        base_url = self.DEFAULTS['base_url']

        if 'base_url' in options:
            base_url = options['base_url']
            del(options['base_url'])

        options = self._update_user_agent_header(options)

        url = "{}{}".format(base_url,  path)
        response = getattr(self.session, method)(url, auth=self.auth,
                                                 verify=self.cert_path,
                                                 **options)
        if ((response.status_code >= 200) and (response.status_code < 300)):
            return response.json()
        else:
            msg = ""
            code = ""
            json_response = response.json()
            if 'error' in json_response:
                if 'description' in json_response['error']:
                    msg = json_response['error']['description']
                if 'code' in json_response['error']:
                    code = json_response['error']['code']

            if code == 'BAD_REQUEST_ERROR':
                raise BadRequestError(msg)
            elif code == 'GATEWAY_ERROR':
                raise GatewayError(msg)
            elif code == 'SERVER_ERROR':
                raise ServerError(msg)
            elif code == 'PROCESSING_ERROR':
                raise ProcessingError(msg)
            else:
                raise ServerError(msg)

    def get(self, path, params, **options):
        """
        Parses GET request options and dispatches a request
        """
        return self.request('get', path, params=params, **options)

    def post(self, path, data, **options):
        """
        Parses POST request options and dispatches a request
        """
        data, options = self._update_request(data, options)
        return self.request('post', path, data=data, **options)

    def delete(self, path, data, **options):
        """
        Parses DELETE request options and dispatches a request
        """
        data, options = self._update_request(data, options)
        return self.request('delete', path, data=data, **options)

    def put(self, path, data, **options):
        """
        Parses PUT request options and dispatches a request
        """
        data, options = self._update_request(data, options)
        return self.request('put', path, data=data, **options)

    def _update_request(self, data, options):
        """
        Updates The resource data and header options
        """
        data = json.dumps(data)
        if 'headers' in options:
            options['headers']['Content-type'] = 'application/json'
        else:
            options['headers'] = {'Content-type': 'application/json'}

        return data, options
<commit_msg>[razorpay-python] fixed test cases for python 3 and above<commit_after>import os
import json
import requests
import pkg_resources

from pkg_resources import DistributionNotFound

from types import ModuleType

from . import resources, utility

from .errors import (BadRequestError, GatewayError,
                     ProcessingError, ServerError)

# Create a dict of resource classes
RESOURCE_CLASSES = {}
for name, module in resources.__dict__.items():
    if isinstance(module, ModuleType) and name.capitalize() in module.__dict__:
        RESOURCE_CLASSES[name] = module.__dict__[name.capitalize()]

UTILITY_CLASSES = {}
for name, module in utility.__dict__.items():
    if isinstance(module, ModuleType) and name.capitalize() in module.__dict__:
        UTILITY_CLASSES[name] = module.__dict__[name.capitalize()]

class Client:
    """Razorpay client class"""

    DEFAULTS = {
        'base_url': 'https://api.razorpay.com/v1'
    }

    def __init__(self, session=None, auth=None, **options):
        """
        Initialize a Client object with session,
        optional auth handler, and options
        """
        self.session = session or requests.Session()
        self.auth = auth
        file_dir = os.path.dirname(__file__)
        self.cert_path = file_dir + '/ca-bundle.crt'

        # intializes each resource
        # injecting this client object into the constructor
        for name, Klass in RESOURCE_CLASSES.items():
            setattr(self, name, Klass(self))

        for name, Klass in UTILITY_CLASSES.items():
            setattr(self, name, Klass(self))

    def _update_user_agent_header(self, options):
        user_agent = "{}{}".format('Razorpay-Python/',  self._get_version())

        if 'headers' in options:
            options['headers']['User-Agent'] = user_agent
        else:
            options['headers'] = {'User-Agent': user_agent}

        return options

    def _get_version(self):
        version = ""
        try:
            version = pkg_resources.require("razorpay")[0].version
        except DistributionNotFound:
            pass
        return version

    def request(self, method, path, **options):
        """
        Dispatches a request to the Razorpay HTTP API
        """
        base_url = self.DEFAULTS['base_url']

        if 'base_url' in options:
            base_url = options['base_url']
            del(options['base_url'])

        options = self._update_user_agent_header(options)

        url = "{}{}".format(base_url,  path)
        response = getattr(self.session, method)(url, auth=self.auth,
                                                 verify=self.cert_path,
                                                 **options)
        if ((response.status_code >= 200) and (response.status_code < 300)):
            return response.json()
        else:
            msg = ""
            code = ""
            json_response = response.json()
            if 'error' in json_response:
                if 'description' in json_response['error']:
                    msg = json_response['error']['description']
                if 'code' in json_response['error']:
                    code = json_response['error']['code']

            if code == 'BAD_REQUEST_ERROR':
                raise BadRequestError(msg)
            elif code == 'GATEWAY_ERROR':
                raise GatewayError(msg)
            elif code == 'SERVER_ERROR':
                raise ServerError(msg)
            elif code == 'PROCESSING_ERROR':
                raise ProcessingError(msg)
            else:
                raise ServerError(msg)

    def get(self, path, params, **options):
        """
        Parses GET request options and dispatches a request
        """
        return self.request('get', path, params=params, **options)

    def post(self, path, data, **options):
        """
        Parses POST request options and dispatches a request
        """
        data, options = self._update_request(data, options)
        return self.request('post', path, data=data, **options)

    def delete(self, path, data, **options):
        """
        Parses DELETE request options and dispatches a request
        """
        data, options = self._update_request(data, options)
        return self.request('delete', path, data=data, **options)

    def put(self, path, data, **options):
        """
        Parses PUT request options and dispatches a request
        """
        data, options = self._update_request(data, options)
        return self.request('put', path, data=data, **options)

    def _update_request(self, data, options):
        """
        Updates The resource data and header options
        """
        data = json.dumps(data)
        if 'headers' in options:
            options['headers']['Content-type'] = 'application/json'
        else:
            options['headers'] = {'Content-type': 'application/json'}

        return data, options
<|endoftext|>