<commit_before>import logging
from operator import itemgetter
import asyncio
import time

from .streams import BinanceSocketManager

class DepthCache(object):

    def __init__(self, symbol, conv_type = float):
        """Initialise the DepthCache

        :param symbol: Symbol to create depth cache for
        :type symbol: string
        :param conv_type: Optional type to represent price, and amount, default is float.
        :type conv_type: function.

        """
        self.symbol = symbol
        self._bids = {}
        self._asks = {}
        self.update_time = None
        self.conv_type = conv_type

    def add_bid(self, bid):
        """Add a bid to the cache

        :param bid:
        :return:

        """
        self._bids[bid[0]] = self.conv_type(bid[1])
        if bid[1] == "0.00000000":
            del self._bids[bid[0]]

    def add_ask(self, ask):
        """Add an ask to the cache

        :param ask:
        :return:

        """
        self._asks[ask[0]] = self.conv_type(ask[1])
        if ask[1] == "0.00000000":
            del self._asks[ask[0]]

    def get_bids(self):
        """Get the current bids

        :return: list of bids with price and quantity as conv_type

        .. code-block:: python

            [
                [
                    0.0001946,  # Price
                    45.0        # Quantity
                ],
                [
                    0.00019459,
                    2384.0
                ],
                [
                    0.00019158,
                    5219.0
                ],
                [
                    0.00019157,
                    1180.0
                ],
                [
                    0.00019082,
                    287.0
                ]
            ]

        """
        return DepthCache.sort_depth(self._bids, reverse=True, conv_type=self.conv_type)

    def get_asks(self):
        """Get the current asks

        :return: list of asks with price and quantity as conv_type.

        .. code-block:: python

            [
                [
                    0.0001955,  # Price
                    57.0'       # Quantity
                ],
                [
                    0.00019699,
                    778.0
                ],
                [
                    0.000197,
                    64.0
                ],
                [
                    0.00019709,
                    1130.0
                ],
                [
                    0.0001971,
                    385.0
                ]
            ]

        """
        return DepthCache.sort_depth(self._asks, reverse=False, conv_type=self.conv_type)

    @staticmethod
    def sort_depth(vals, reverse=False, conv_type = float):
        """Sort bids or asks by price
        """
        lst = [[conv_type(price), quantity] for price, quantity in vals.items()]
        lst = sorted(lst, key=itemgetter(0), reverse=reverse)
        return lst

class BaseDepthCacheManager:
    DEFAULT_REFRESH = 60 * 30  # 30 minutes
    TIMEOUT = 60

    def __init__(self, client, loop, symbol, refresh_interval=None, bm=None, limit=10, conv_type=float):
        """Create a DepthCacheManager instance

        :param client: Binance API client
        :type client: binance.Client
        :param loop:
        :type loop:
        :param symbol: Symbol to create depth cache for
        :type symbol: string
        :param refresh_interval: Optional number of seconds between cache refresh, use 0 or None to disable
        :type refresh_interval: int
        :param bm: Optional BinanceSocketManager
        :type bm: BinanceSocketManager
        :param limit: Optional number of orders to get from orderbook
        :type limit: int
        :param conv_type: Optional type to represent price, and amount, default is float.
        :type conv_type: function.

        """

        self._client = client
        self._depth_cache = None
        self._loop = loop or asyncio.get_event_loop()
        self._symbol = symbol
        self._limit = limit
        self._last_update_id = None
        self._bm = bm or BinanceSocketManager(self._client, self._loop)
        self._refresh_interval = refresh_interval or self.DEFAULT_REFRESH
        self._conn_key = None
        self._conv_type = conv_type

    async def __aenter__(self):
        await asyncio.gather(
            self._init_cache(),
            self._start_socket()
        )
        await self._socket.__aenter__()
        return self

    async def __aexit__(self, *args, **kwargs):
        await self._socket.__aexit__(*args, **kwargs)

    async def recv(self):
        dc = None
        while not dc:
            try:
                res = await asyncio.wait_for(self._socket.recv(), timeout=self.TIMEOUT)
            except Exception as e:

                logging.warning(e)
                pass
            else:
                dc = await self._depth_event(res)
        return dc

    async def _init_cache(self):
        """Initialise the depth cache calling REST endpoint

        :return:
        """

        # initialise or clear depth cache
        self._depth_cache = DepthCache(self._symbol, conv_type=self._conv_type)

        # set a time to refresh the depth cache
        if self._refresh_interval:
            self._refresh_time = int(time.time()) + self._refresh_interval

    async def _start_socket(self):
        """Start the depth cache socket

        :return:
        """
        self._socket = self._get_socket()

    def _get_socket(self):
        raise NotImplementedError

    async def _depth_event(self, msg):
        """Handle a depth event

        :param msg:
        :return:

        """

        if not msg:
            return None

        if 'e' in msg and msg['e'] == 'error':
            # close the socket
            await self.close()

            # notify the user by returning a None value
            return None

        return await self._process_depth_message(msg)

    async def _process_depth_message(self, msg):
        """Process a depth event message.

        :param msg: Depth event message.
        :return:

        """

        # add any bid or ask values
        self._apply_orders(msg)

        # call the callback with the updated depth cache
        res = self._depth_cache

        # after processing event see if we need to refresh the depth cache
        if self._refresh_interval and int(time.time()) > self._refresh_time:
            await self._init_cache()

        return res

    def _apply_orders(self, msg):
        for bid in msg.get('b', []) + msg.get('bids', []):
            self._depth_cache.add_bid(bid)
        for ask in msg.get('a', []) + msg.get('asks', []):
            self._depth_cache.add_ask(ask)

        # keeping update time
        self._depth_cache.update_time = msg.get('E') or msg.get('lastUpdateId')

    def get_depth_cache(self):
        """Get the current depth cache

        :return: DepthCache object

        """
        return self._depth_cache

    async def close(self):
        """Close the open socket for this manager

        :return:
        """
        self._depth_cache = None

    def get_symbol(self):
        """Get the symbol

        :return: symbol
        """
        return self._symbol

class DepthCacheManager(BaseDepthCacheManager):

    @classmethod
    async def create(cls, client, loop, symbol, coro=None, refresh_interval=None, bm=None, limit=500, ws_interval=None):
        """Initialise the DepthCacheManager

        :param client: Binance API client
        :type client: binance.Client
        :param loop: asyncio loop
        :param symbol: Symbol to create depth cache for
        :type symbol: string
        :param coro: Optional function to receive depth cache updates
        :type coro: function
        :param refresh_interval: Optional number of seconds between cache refresh, use 0 or None to disable
        :type refresh_interval: int
        :param limit: Optional number of orders to get from orderbook
        :type limit: int
        :param ws_interval: Optional interval for updates on websocket, default None. If not set, updates happen every second. Must be 0, None (1s) or 100 (100ms).
        :type ws_interval: int

        """
        self = await super().create(client, loop, symbol, coro, refresh_interval, bm, limit)
        self._ws_interval = ws_interval
        return self

    async def _init_cache(self):
        """Initialise the depth cache calling REST endpoint

        :return:
        """
        self._last_update_id = None
        self._depth_message_buffer = []

        res = await self._client.get_order_book(symbol=self._symbol, limit=self._limit)

        # initialise or clear depth cache
        await super()._init_cache()

        # process bid and asks from the order book
        self._apply_orders(res)
        for bid in res['bids']:
            self._depth_cache.add_bid(bid)
        for ask in res['asks']:
            self._depth_cache.add_ask(ask)

        # set first update id
        self._last_update_id = res['lastUpdateId']

        # Apply any updates from the websocket
        for msg in self._depth_message_buffer:
            await self._process_depth_message(msg)

        # clear the depth buffer
        self._depth_message_buffer = []

    async def _start_socket(self):
        """Start the depth cache socket

        :return:
        """
        if not getattr(self, '_depth_message_buffer', None):
            self._depth_message_buffer = []

        await super()._start_socket()

    def _get_socket(self):
        return self._bm.depth_socket(self._symbol)

    async def _process_depth_message(self, msg):
        """Process a depth event message.

        :param msg: Depth event message.
        :return:

        """

        print(msg['u'])

        if self._last_update_id is None:
            # Initial depth snapshot fetch not yet performed, buffer messages
            self._depth_message_buffer.append(msg)
            return

        if msg['u'] <= self._last_update_id:
            # ignore any updates before the initial update id
            return
        elif msg['U'] != self._last_update_id + 1:
            # if not buffered check we get sequential updates
            # otherwise init cache again
            await self._init_cache()

        # add any bid or ask values
        self._apply_orders(msg)

        # call the callback with the updated depth cache
        res = self._depth_cache

        self._last_update_id = msg['u']

        # after processing event see if we need to refresh the depth cache
        if self._refresh_interval and int(time.time()) > self._refresh_time:
            await self._init_cache()

        return res

class OptionsDepthCacheManager(BaseDepthCacheManager):

    def _get_socket(self):
        return self._bm.options_depth_socket(self._symbol)
<commit_msg>fix formatting<commit_after>import logging
from operator import itemgetter
import asyncio
import time

from .streams import BinanceSocketManager

class DepthCache(object):

    def __init__(self, symbol, conv_type=float):
        """Initialise the DepthCache

        :param symbol: Symbol to create depth cache for
        :type symbol: string
        :param conv_type: Optional type to represent price, and amount, default is float.
        :type conv_type: function.

        """
        self.symbol = symbol
        self._bids = {}
        self._asks = {}
        self.update_time = None
        self.conv_type = conv_type

    def add_bid(self, bid):
        """Add a bid to the cache

        :param bid:
        :return:

        """
        self._bids[bid[0]] = self.conv_type(bid[1])
        if bid[1] == "0.00000000":
            del self._bids[bid[0]]

    def add_ask(self, ask):
        """Add an ask to the cache

        :param ask:
        :return:

        """
        self._asks[ask[0]] = self.conv_type(ask[1])
        if ask[1] == "0.00000000":
            del self._asks[ask[0]]

    def get_bids(self):
        """Get the current bids

        :return: list of bids with price and quantity as conv_type

        .. code-block:: python

            [
                [
                    0.0001946,  # Price
                    45.0        # Quantity
                ],
                [
                    0.00019459,
                    2384.0
                ],
                [
                    0.00019158,
                    5219.0
                ],
                [
                    0.00019157,
                    1180.0
                ],
                [
                    0.00019082,
                    287.0
                ]
            ]

        """
        return DepthCache.sort_depth(self._bids, reverse=True, conv_type=self.conv_type)

    def get_asks(self):
        """Get the current asks

        :return: list of asks with price and quantity as conv_type.

        .. code-block:: python

            [
                [
                    0.0001955,  # Price
                    57.0'       # Quantity
                ],
                [
                    0.00019699,
                    778.0
                ],
                [
                    0.000197,
                    64.0
                ],
                [
                    0.00019709,
                    1130.0
                ],
                [
                    0.0001971,
                    385.0
                ]
            ]

        """
        return DepthCache.sort_depth(self._asks, reverse=False, conv_type=self.conv_type)

    @staticmethod
    def sort_depth(vals, reverse=False, conv_type=float):
        """Sort bids or asks by price
        """
        lst = [[conv_type(price), quantity] for price, quantity in vals.items()]
        lst = sorted(lst, key=itemgetter(0), reverse=reverse)
        return lst

class BaseDepthCacheManager:
    DEFAULT_REFRESH = 60 * 30  # 30 minutes
    TIMEOUT = 60

    def __init__(self, client, loop, symbol, refresh_interval=None, bm=None, limit=10, conv_type=float):
        """Create a DepthCacheManager instance

        :param client: Binance API client
        :type client: binance.Client
        :param loop:
        :type loop:
        :param symbol: Symbol to create depth cache for
        :type symbol: string
        :param refresh_interval: Optional number of seconds between cache refresh, use 0 or None to disable
        :type refresh_interval: int
        :param bm: Optional BinanceSocketManager
        :type bm: BinanceSocketManager
        :param limit: Optional number of orders to get from orderbook
        :type limit: int
        :param conv_type: Optional type to represent price, and amount, default is float.
        :type conv_type: function.

        """

        self._client = client
        self._depth_cache = None
        self._loop = loop or asyncio.get_event_loop()
        self._symbol = symbol
        self._limit = limit
        self._last_update_id = None
        self._bm = bm or BinanceSocketManager(self._client, self._loop)
        self._refresh_interval = refresh_interval or self.DEFAULT_REFRESH
        self._conn_key = None
        self._conv_type = conv_type

    async def __aenter__(self):
        await asyncio.gather(
            self._init_cache(),
            self._start_socket()
        )
        await self._socket.__aenter__()
        return self

    async def __aexit__(self, *args, **kwargs):
        await self._socket.__aexit__(*args, **kwargs)

    async def recv(self):
        dc = None
        while not dc:
            try:
                res = await asyncio.wait_for(self._socket.recv(), timeout=self.TIMEOUT)
            except Exception as e:

                logging.warning(e)
                pass
            else:
                dc = await self._depth_event(res)
        return dc

    async def _init_cache(self):
        """Initialise the depth cache calling REST endpoint

        :return:
        """

        # initialise or clear depth cache
        self._depth_cache = DepthCache(self._symbol, conv_type=self._conv_type)

        # set a time to refresh the depth cache
        if self._refresh_interval:
            self._refresh_time = int(time.time()) + self._refresh_interval

    async def _start_socket(self):
        """Start the depth cache socket

        :return:
        """
        self._socket = self._get_socket()

    def _get_socket(self):
        raise NotImplementedError

    async def _depth_event(self, msg):
        """Handle a depth event

        :param msg:
        :return:

        """

        if not msg:
            return None

        if 'e' in msg and msg['e'] == 'error':
            # close the socket
            await self.close()

            # notify the user by returning a None value
            return None

        return await self._process_depth_message(msg)

    async def _process_depth_message(self, msg):
        """Process a depth event message.

        :param msg: Depth event message.
        :return:

        """

        # add any bid or ask values
        self._apply_orders(msg)

        # call the callback with the updated depth cache
        res = self._depth_cache

        # after processing event see if we need to refresh the depth cache
        if self._refresh_interval and int(time.time()) > self._refresh_time:
            await self._init_cache()

        return res

    def _apply_orders(self, msg):
        for bid in msg.get('b', []) + msg.get('bids', []):
            self._depth_cache.add_bid(bid)
        for ask in msg.get('a', []) + msg.get('asks', []):
            self._depth_cache.add_ask(ask)

        # keeping update time
        self._depth_cache.update_time = msg.get('E') or msg.get('lastUpdateId')

    def get_depth_cache(self):
        """Get the current depth cache

        :return: DepthCache object

        """
        return self._depth_cache

    async def close(self):
        """Close the open socket for this manager

        :return:
        """
        self._depth_cache = None

    def get_symbol(self):
        """Get the symbol

        :return: symbol
        """
        return self._symbol

class DepthCacheManager(BaseDepthCacheManager):

    @classmethod
    async def create(cls, client, loop, symbol, coro=None, refresh_interval=None, bm=None, limit=500, ws_interval=None):
        """Initialise the DepthCacheManager

        :param client: Binance API client
        :type client: binance.Client
        :param loop: asyncio loop
        :param symbol: Symbol to create depth cache for
        :type symbol: string
        :param coro: Optional function to receive depth cache updates
        :type coro: function
        :param refresh_interval: Optional number of seconds between cache refresh, use 0 or None to disable
        :type refresh_interval: int
        :param limit: Optional number of orders to get from orderbook
        :type limit: int
        :param ws_interval: Optional interval for updates on websocket, default None. If not set, updates happen every second. Must be 0, None (1s) or 100 (100ms).
        :type ws_interval: int

        """
        self = await super().create(client, loop, symbol, coro, refresh_interval, bm, limit)
        self._ws_interval = ws_interval
        return self

    async def _init_cache(self):
        """Initialise the depth cache calling REST endpoint

        :return:
        """
        self._last_update_id = None
        self._depth_message_buffer = []

        res = await self._client.get_order_book(symbol=self._symbol, limit=self._limit)

        # initialise or clear depth cache
        await super()._init_cache()

        # process bid and asks from the order book
        self._apply_orders(res)
        for bid in res['bids']:
            self._depth_cache.add_bid(bid)
        for ask in res['asks']:
            self._depth_cache.add_ask(ask)

        # set first update id
        self._last_update_id = res['lastUpdateId']

        # Apply any updates from the websocket
        for msg in self._depth_message_buffer:
            await self._process_depth_message(msg)

        # clear the depth buffer
        self._depth_message_buffer = []

    async def _start_socket(self):
        """Start the depth cache socket

        :return:
        """
        if not getattr(self, '_depth_message_buffer', None):
            self._depth_message_buffer = []

        await super()._start_socket()

    def _get_socket(self):
        return self._bm.depth_socket(self._symbol)

    async def _process_depth_message(self, msg):
        """Process a depth event message.

        :param msg: Depth event message.
        :return:

        """

        print(msg['u'])

        if self._last_update_id is None:
            # Initial depth snapshot fetch not yet performed, buffer messages
            self._depth_message_buffer.append(msg)
            return

        if msg['u'] <= self._last_update_id:
            # ignore any updates before the initial update id
            return
        elif msg['U'] != self._last_update_id + 1:
            # if not buffered check we get sequential updates
            # otherwise init cache again
            await self._init_cache()

        # add any bid or ask values
        self._apply_orders(msg)

        # call the callback with the updated depth cache
        res = self._depth_cache

        self._last_update_id = msg['u']

        # after processing event see if we need to refresh the depth cache
        if self._refresh_interval and int(time.time()) > self._refresh_time:
            await self._init_cache()

        return res

class OptionsDepthCacheManager(BaseDepthCacheManager):

    def _get_socket(self):
        return self._bm.options_depth_socket(self._symbol)
<|endoftext|>


<commit_before>"""Compatibility routines."""

from __future__ import absolute_import

import sys

try:
    import Image  # pylint: disable=import-error,unused-import
except ImportError:
    from PIL import Image  # pylint: disable=unused-import

try:
    import ImageOps  # pylint: disable=import-error,unused-import
except ImportError:
    from PIL import ImageOps  # pylint: disable=unused-import

if sys.version_info[0] > 2:
    basestring = (str, bytes)  # pylint: disable=redefined-builtin,invalid-name
else:
    # pylint: disable=redefined-variable-type
    basestring = basestring  # pylint: disable=invalid-name
<commit_msg>Centralize compat module pylint messages.<commit_after>"""Compatibility routines."""

from __future__ import absolute_import

import sys

# pylint: disable=import-error
# pylint: disable=invalid-name
# pylint: disable=redefined-builtin
# pylint: disable=redefined-variable-type
# pylint: disable=unused-import

try:
    import Image
except ImportError:
    from PIL import Image

try:
    import ImageOps
except ImportError:
    from PIL import ImageOps

if sys.version_info[0] > 2:
    basestring = (str, bytes)
else:
    basestring = basestring
<|endoftext|>


<commit_before># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import random

from oslo_config import cfg

from senlin.common import exception
from senlin.common.i18n import _
from senlin.common.i18n import _LE
from senlin.common.i18n import _LW
from senlin.db import api as db_api
from senlin.engine.actions import base
from senlin.engine import cluster as cluster_mod
from senlin.engine import dispatcher
from senlin.engine import node as node_mod
from senlin.engine import scheduler
from senlin.openstack.common import log as logging
from senlin.policies import base as policies

CONF = cfg.CONF
CONF.import_opt('lock_retry_times', 'senlin.common.config')
CONF.import_opt('lock_retry_interval', 'senlin.common.config')

LOG = logging.getLogger(__name__)

class ClusterAction(base.Action):
    '''An action performed on a cluster.'''

    ACTIONS = (
        CLUSTER_CREATE, CLUSTER_DELETE, CLUSTER_UPDATE,
        CLUSTER_ADD_NODES, CLUSTER_DEL_NODES,
        CLUSTER_SCALE_UP, CLUSTER_SCALE_DOWN,
        CLUSTER_ATTACH_POLICY, CLUSTER_DETACH_POLICY,
    ) = (
        'CLUSTER_CREATE', 'CLUSTER_DELETE', 'CLUSTER_UPDATE',
        'CLUSTER_ADD_NODES', 'CLUSTER_DEL_NODES',
        'CLUSTER_SCALE_UP', 'CLUSTER_SCALE_DOWN',
        'CLUSTER_ATTACH_POLICY', 'CLUSTER_DETACH_POLICY',
    )

    def __init__(self, context, action, **kwargs):
        super(ClusterAction, self).__init__(context, action, **kwargs)

    def _cluster_lock_acquire(self, cluster, steal_lock=False):
        '''Try to lock the cluster

        :param steal_lock: set to True to cancel current action that owns the
                           lock, if any.
        '''
        # Step 1: try lock the cluster
        action_id = db_api.cluster_lock_acquire(cluster.id, self.id)
        if action_id == self.id:
            return self.RES_OK

        # Step 2: retry
        retries = cfg.CONF.lock_retry_times
        interval = cfg.CONF.lock_retry_interval

        while retries > 0:
            scheduler.sleep(interval)
            action_id = db_api.cluster_lock_acquire(cluster.id, self.id)
            if action_id == self.id:
                return self.RES_OK
            retries = retries - 1

        # Step 3: Last resort is 'stealing', only needed when retry failed
        if steal_lock:
            # Cancel the action that owns the lock
            scheduler.cancel_action(self.context, action_id)

            action_id = db_api.cluster_lock_acquire(cluster.id, self.id)
            while action_id and action_id != self.id:
                if scheduler.action_timeout(self):
                    msg = _LE('Cluster lock timeout, action (%s)')
                    LOG.error(msg, self.id)
                    cluster.set_status(self.context, cluster.ERROR,
                                       reason='Cluster lock timeout')
                    return self.RES_TIMEOUT

                scheduler.reschedule(self)
                action_id = db_api.cluster_lock_acquire(cluster.id, self.id)

            return self.RES_OK

        # Return
        msg = _LW('Cluster is already locked by action %(old)s, '
                  'action %(new)s failed grabbing the lock') % {
                      'old': action_id, 'new': self.id}
        LOG.warn(msg)

        return self.RES_RETRY

    def _release_cluster_lock(self, cluster):
        db_api.cluster_lock_release(cluster.id, self.id)

    def _wait_for_action(self):
        while self.get_status() != self.READY:
            if scheduler.action_cancelled(self):
                # During this period, if cancel request come, cancel this
                # cluster operation immediately, then release the cluster
                # lock and return.
                LOG.debug(_('%(action)s %(id)s cancelled') % {
                    'action': self.action, 'id': self.id})
                return self.RES_CANCEL
            elif scheduler.action_timeout(self):
                # Action timeout, return
                LOG.debug(_('%(action)s %(id)s timeout') % {
                    'action': self.action, 'id': self.id})
                return self.RES_TIMEOUT

            # Continue waiting (with reschedule)
            scheduler.reschedule(self)

        return self.RES_OK

    def do_create(self, cluster):
        res = cluster.do_create(self.context)
        if not res:
            cluster.set_status(cluster.ERROR, 'Cluster creation failed')
            return self.RES_ERROR

        for m in range(cluster.size):
            name = 'node-%s-%003d' % (cluster.id[:8], m + 1)
            node = node_mod.Node(name, cluster.profile_id, cluster.id,
                                 context=self.context)
            node.store(self.context)
            kwargs = {
                'name': 'node_create_%s' % node.id[:8],
                'target': node.id,
                'cause': 'Cluster creation',
            }

            action = base.Action(self.context, 'NODE_CREATE', **kwargs)
            action.store(self.context)

            # Build dependency and make the new action ready
            db_api.action_add_dependency(self.context, action.id, self.id)
            action.set_status(self.READY)

            dispatcher.notify(self.context, dispatcher.Dispatcher.NEW_ACTION,
                              None, action_id=action.id)

        # Wait for cluster creating complete
        result = self.RES_OK
        if cluster.size > 0:
            result = self._wait_for_action()

        if result == self.RES_OK:
            cluster.set_status(self.context, cluster.ACTIVE,
                               'Cluster creation completed')
        elif result == self.RES_CANCEL:
            cluster.set_status(self.context, cluster.ERROR,
                               'Cluster creation cancelled')
        elif result == self.RES_TIMEOUT:
            cluster.set_status(self.context, cluster.ERROR,
                               'Cluster creation timeout')
        else:
            # RETRY
            pass

        return result

    def do_update(self, cluster, new_profile_id):
        res = cluster.do_update(self.context, profile_id=new_profile_id)
        if not res:
            cluster.set_status(cluster.ACTIVE,
                               'Cluster updating was not executed')
            return self.RES_ERROR

        # Create NodeActions for all nodes
        node_list = cluster.get_nodes()
        for node_id in node_list:
            kwargs = {
                'name': 'node_update_%s' % node_id[:8],
                'context': self.context,
                'target': node_id,
                'cause': 'Cluster update',
                'inputs': {
                    'new_profile_id': new_profile_id,
                }
            }
            action = base.Action(self.context, 'NODE_UPDATE', **kwargs)
            action.store(self.context)

            db_api.action_add_dependency(action, self)
            action.set_status(self.READY)
            dispatcher.notify(self.context, dispatcher.Dispatcher.NEW_ACTION,
                              None, action_id=action.id)

        # Wait for cluster updating complete
        result = self.RES_OK
        if cluster.size > 0:
            result = self._wait_for_action()

        if result == self.RES_OK:
            cluster.set_status(self.context, cluster.ACTIVE,
                               'Cluster update completed')

        return self.RES_OK

    def do_delete(self, cluster):
        cluster.set_status(self.context, cluster.DELETING)
        node_list = cluster.get_nodes()
        for node_id in node_list:
            kwargs = {
                'name': 'node_delete_%s' % node_id[:8],
                'context': self.context,
                'target': node_id,
                'cause': 'Cluster deletion',
            }
            action = base.Action(self.context, 'NODE_DELETE', **kwargs)
            action.store(self.context)

            # Build dependency and make the new action ready
            db_api.action_add_dependency(action.id, self)
            action.set_status(self.READY)

            dispatcher.notify(self.context, dispatcher.Dispatcher.NEW_ACTION,
                              None, action_id=action.id)

        result = self.RES_OK
        if cluster.size > 0:
            result = self._wait_for_action()

        if result == self.RES_OK:
            result = cluster.do_delete(self.context)

        if result == self.RES_OK:
            cluster.set_status(self.context, cluster.DELETED,
                               'Cluster deletion completed')
        elif result == self.RES_CANCEL:
            cluster.set_status(self.context, cluster.ERROR,
                               'Cluster deletion cancelled')
        elif result == self.RES_TIMEOUT:
            cluster.set_status(self.context, cluster.ERROR,
                               'Cluster deletion timeout')
        else:
            # RETRY
            pass

        return self.RES_OK

    def do_add_nodes(self, cluster):
        return self.RES_OK

    def do_del_nodes(self, cluster):
        return self.RES_OK

    def do_scale_up(self, cluster):
        return self.RES_OK

    def do_scale_down(self, cluster, count, candidates):
        # Go through all policies before scaling down.
        if len(candidates) == 0:
            # No candidates for scaling down op which means no DeletionPolicy
            # is attached to cluster, we just choose random nodes to
            # delete based on scaling policy result.
            nodes = db_api.node_get_all_by_cluster(self.context,
                                                   self.cluster_id)
            # TODO(anyone): add some warning here
            if count > len(nodes):
                count = len(nodes)

            i = count
            while i > 0:
                rand = random.randrange(i)
                candidates.append(nodes[rand].id)
                nodes.remove(nodes[rand])
                i = i - 1

        action_list = []
        for node_id in candidates:
            kwargs = {
                'name': 'node-delete-%s' % node_id,
                'context': self.context,
                'target': node_id,
                'cause': 'Cluster scale down',
            }
            action = base.Action(self.context, 'NODE_DELETE', **kwargs)
            action.store(self.context)

            action_list.append(action.id)
            db_api.action_add_dependency(action, self)
            action.set_status(self.READY)

        # Notify dispatcher
        for action_id in action_list:
            dispatcher.notify(self.context,
                              dispatcher.Dispatcher.NEW_ACTION,
                              None,
                              action_id=action_id)

        # Wait for cluster creating complete. If timeout,
        # set cluster status to error.
        # Note: we don't allow to cancel scaling operations.
        while self.get_status() != self.READY:
            if scheduler.action_timeout(self):
                # Action timeout, set cluster status to ERROR and return
                LOG.debug('Cluster scale_down action %s timeout' % self.id)
                cluster.set_status(cluster.ERROR,
                                   'Cluster scaling down timeout')
                return self.RES_TIMEOUT

            # Continue waiting (with sleep)
            scheduler.reschedule(self)

        cluster.delete_nodes(candidates)

        # set cluster status to OK
        return self.RES_OK

    def do_attach_policy(self, cluster):
        policy_id = self.inputs.get('policy_id', None)
        if not policy_id:
            raise exception.PolicyNotSpecified()

        policy = policies.Policy.load(self.context, policy_id)
        # Check if policy has already been attached
        all = db_api.cluster_get_policies(self.context, cluster.id)
        for existing in all:
            # Policy already attached
            if existing.policy_id == policy_id:
                return self.RES_OK

            # Detect policy type conflicts
            curr = policies.Policy.load(self.context, existing.policy_id)
            if curr.type == policy.type:
                raise exception.PolicyExists(policy_type=policy.type)

        values = {
            'cooldown': self.inputs.get('cooldown', policy.cooldown),
            'level': self.inputs.get('level', policy.level),
            'enabled': self.inputs.get('enabled', True),
        }

        db_api.cluster_attach_policy(self.context, cluster.id, policy_id,
                                     values)

        cluster.rt.policies.append(policy)
        return self.RES_OK

    def do_detach_policy(self, cluster):
        return self.RES_OK

    def _execute(self, cluster):
        # do pre-action policy checking
        check_result = self.policy_check(cluster.id, 'BEFORE')
        if not check_result:
            # Don't emit message here since policy_check should have done it
            return self.RES_ERROR

        res = self.RES_OK
        if self.action == self.CLUSTER_CREATE:
            res = self.do_create(cluster)
        elif self.action == self.CLUSTER_UPDATE:
            new_profile_id = self.inputs.get('new_profile_id')
            # TODO(anyone): check null?
            res = self.do_update(cluster, new_profile_id)
        elif self.action == self.CLUSTER_DELETE:
            res = self.do_delete(cluster)
        elif self.action == self.CLUSTER_ADD_NODES:
            res = self.do_add_nodes(cluster)
        elif self.action == self.CLUSTER_DEL_NODES:
            res = self.do_del_nodes(cluster)
        elif self.action == self.CLUSTER_SCALE_UP:
            res = self.do_scale_up(cluster)
        elif self.action == self.CLUSTER_SCALE_DOWN:
            count = check_result.get('count', 0)
            if count == 0:
                # We assume the policy has emit error/warnings
                return self.RES_ERROR
            candidates = check_result.get('candidates', [])
            res = self.do_scale_down(cluster, count, candidates)
        elif self.action == self.CLUSTER_ATTACH_POLICY:
            res = self.do_attach_policy(cluster)
        elif self.action == self.CLUSTER_DETACH_POLICY:
            res = self.do_detach_policy(cluster)

        # do post-action policy checking
        if res == self.RES_OK:
            check_result = self.policy_check(cluster.id, 'AFTER')
            res = self.RES_OK if check_result else self.ERROR

        return res

    def execute(self, **kwargs):
        try:
            cluster = cluster_mod.Cluster.load(self.context, self.target)
        except exception.NotFound:
            LOG.error(_LE('Cluster %(id)s not found') % {'id': self.target})
            return self.RES_ERROR

        steal_lock = kwargs.get('steal_lock', False)
        if self.action in [self.CLUSTER_DELETE]:
            steal_lock = True

        # Try to lock cluster before do real action
        res = self._cluster_lock_acquire(cluster, steal_lock)

        if res != self.RES_OK:
            return res

        # Do real operation here
        res = self._execute(cluster)

        # We've done, release cluster lock
        self._release_cluster_lock(cluster)
        return res

    def cancel(self):
        return self.RES_OK
<commit_msg>Revised log format<commit_after># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import random

from oslo_config import cfg

from senlin.common import exception
from senlin.common.i18n import _
from senlin.common.i18n import _LE
from senlin.common.i18n import _LW
from senlin.db import api as db_api
from senlin.engine.actions import base
from senlin.engine import cluster as cluster_mod
from senlin.engine import dispatcher
from senlin.engine import node as node_mod
from senlin.engine import scheduler
from senlin.openstack.common import log as logging
from senlin.policies import base as policies

CONF = cfg.CONF
CONF.import_opt('lock_retry_times', 'senlin.common.config')
CONF.import_opt('lock_retry_interval', 'senlin.common.config')

LOG = logging.getLogger(__name__)

class ClusterAction(base.Action):
    '''An action performed on a cluster.'''

    ACTIONS = (
        CLUSTER_CREATE, CLUSTER_DELETE, CLUSTER_UPDATE,
        CLUSTER_ADD_NODES, CLUSTER_DEL_NODES,
        CLUSTER_SCALE_UP, CLUSTER_SCALE_DOWN,
        CLUSTER_ATTACH_POLICY, CLUSTER_DETACH_POLICY,
    ) = (
        'CLUSTER_CREATE', 'CLUSTER_DELETE', 'CLUSTER_UPDATE',
        'CLUSTER_ADD_NODES', 'CLUSTER_DEL_NODES',
        'CLUSTER_SCALE_UP', 'CLUSTER_SCALE_DOWN',
        'CLUSTER_ATTACH_POLICY', 'CLUSTER_DETACH_POLICY',
    )

    def __init__(self, context, action, **kwargs):
        super(ClusterAction, self).__init__(context, action, **kwargs)

    def _cluster_lock_acquire(self, cluster, steal_lock=False):
        '''Try to lock the cluster

        :param steal_lock: set to True to cancel current action that owns the
                           lock, if any.
        '''
        # Step 1: try lock the cluster
        action_id = db_api.cluster_lock_acquire(cluster.id, self.id)
        if action_id == self.id:
            return self.RES_OK

        # Step 2: retry
        retries = cfg.CONF.lock_retry_times
        interval = cfg.CONF.lock_retry_interval

        while retries > 0:
            scheduler.sleep(interval)
            action_id = db_api.cluster_lock_acquire(cluster.id, self.id)
            if action_id == self.id:
                return self.RES_OK
            retries = retries - 1

        # Step 3: Last resort is 'stealing', only needed when retry failed
        if steal_lock:
            # Cancel the action that owns the lock
            scheduler.cancel_action(self.context, action_id)

            action_id = db_api.cluster_lock_acquire(cluster.id, self.id)
            while action_id and action_id != self.id:
                if scheduler.action_timeout(self):
                    LOG.error(_LE('Cluster lock timeout, action (%s)'),
                              self.id)
                    return self.RES_TIMEOUT

                scheduler.reschedule(self)
                action_id = db_api.cluster_lock_acquire(cluster.id, self.id)

            return self.RES_OK

        # Return
        LOG.warn(_LW('Cluster is already locked by action %(old)s, '
                     'action %(new)s failed grabbing the lock') % {
                        'old': action_id, 'new': self.id})

        return self.RES_RETRY

    def _release_cluster_lock(self, cluster):
        db_api.cluster_lock_release(cluster.id, self.id)

    def _wait_for_action(self):
        while self.get_status() != self.READY:
            if scheduler.action_cancelled(self):
                # During this period, if cancel request come, cancel this
                # cluster operation immediately, then release the cluster
                # lock and return.
                LOG.debug(_('%(action)s %(id)s cancelled') % {
                    'action': self.action, 'id': self.id})
                return self.RES_CANCEL
            elif scheduler.action_timeout(self):
                # Action timeout, return
                LOG.debug(_('%(action)s %(id)s timeout') % {
                    'action': self.action, 'id': self.id})
                return self.RES_TIMEOUT

            # Continue waiting (with reschedule)
            scheduler.reschedule(self)

        return self.RES_OK

    def do_create(self, cluster):
        res = cluster.do_create(self.context)
        if not res:
            cluster.set_status(cluster.ERROR, 'Cluster creation failed')
            return self.RES_ERROR

        for m in range(cluster.size):
            name = 'node-%s-%003d' % (cluster.id[:8], m + 1)
            node = node_mod.Node(name, cluster.profile_id, cluster.id,
                                 context=self.context)
            node.store(self.context)
            kwargs = {
                'name': 'node_create_%s' % node.id[:8],
                'target': node.id,
                'cause': 'Cluster creation',
            }

            action = base.Action(self.context, 'NODE_CREATE', **kwargs)
            action.store(self.context)

            # Build dependency and make the new action ready
            db_api.action_add_dependency(self.context, action.id, self.id)
            action.set_status(self.READY)

            dispatcher.notify(self.context, dispatcher.Dispatcher.NEW_ACTION,
                              None, action_id=action.id)

        # Wait for cluster creating complete
        result = self.RES_OK
        if cluster.size > 0:
            result = self._wait_for_action()

        if result == self.RES_OK:
            cluster.set_status(self.context, cluster.ACTIVE,
                               'Cluster creation completed')
        elif result == self.RES_CANCEL:
            cluster.set_status(self.context, cluster.ERROR,
                               'Cluster creation cancelled')
        elif result == self.RES_TIMEOUT:
            cluster.set_status(self.context, cluster.ERROR,
                               'Cluster creation timeout')
        else:
            # RETRY
            pass

        return result

    def do_update(self, cluster, new_profile_id):
        res = cluster.do_update(self.context, profile_id=new_profile_id)
        if not res:
            cluster.set_status(cluster.ACTIVE,
                               'Cluster updating was not executed')
            return self.RES_ERROR

        # Create NodeActions for all nodes
        node_list = cluster.get_nodes()
        for node_id in node_list:
            kwargs = {
                'name': 'node_update_%s' % node_id[:8],
                'context': self.context,
                'target': node_id,
                'cause': 'Cluster update',
                'inputs': {
                    'new_profile_id': new_profile_id,
                }
            }
            action = base.Action(self.context, 'NODE_UPDATE', **kwargs)
            action.store(self.context)

            db_api.action_add_dependency(action, self)
            action.set_status(self.READY)
            dispatcher.notify(self.context, dispatcher.Dispatcher.NEW_ACTION,
                              None, action_id=action.id)

        # Wait for cluster updating complete
        result = self.RES_OK
        if cluster.size > 0:
            result = self._wait_for_action()

        if result == self.RES_OK:
            cluster.set_status(self.context, cluster.ACTIVE,
                               'Cluster update completed')

        return self.RES_OK

    def do_delete(self, cluster):
        cluster.set_status(self.context, cluster.DELETING)
        node_list = cluster.get_nodes()
        for node_id in node_list:
            kwargs = {
                'name': 'node_delete_%s' % node_id[:8],
                'context': self.context,
                'target': node_id,
                'cause': 'Cluster deletion',
            }
            action = base.Action(self.context, 'NODE_DELETE', **kwargs)
            action.store(self.context)

            # Build dependency and make the new action ready
            db_api.action_add_dependency(action.id, self)
            action.set_status(self.READY)

            dispatcher.notify(self.context, dispatcher.Dispatcher.NEW_ACTION,
                              None, action_id=action.id)

        result = self.RES_OK
        if cluster.size > 0:
            result = self._wait_for_action()

        if result == self.RES_OK:
            result = cluster.do_delete(self.context)

        if result == self.RES_OK:
            cluster.set_status(self.context, cluster.DELETED,
                               'Cluster deletion completed')
        elif result == self.RES_CANCEL:
            cluster.set_status(self.context, cluster.ERROR,
                               'Cluster deletion cancelled')
        elif result == self.RES_TIMEOUT:
            cluster.set_status(self.context, cluster.ERROR,
                               'Cluster deletion timeout')
        else:
            # RETRY
            pass

        return self.RES_OK

    def do_add_nodes(self, cluster):
        return self.RES_OK

    def do_del_nodes(self, cluster):
        return self.RES_OK

    def do_scale_up(self, cluster):
        return self.RES_OK

    def do_scale_down(self, cluster, count, candidates):
        # Go through all policies before scaling down.
        if len(candidates) == 0:
            # No candidates for scaling down op which means no DeletionPolicy
            # is attached to cluster, we just choose random nodes to
            # delete based on scaling policy result.
            nodes = db_api.node_get_all_by_cluster(self.context,
                                                   self.cluster_id)
            # TODO(anyone): add some warning here
            if count > len(nodes):
                count = len(nodes)

            i = count
            while i > 0:
                rand = random.randrange(i)
                candidates.append(nodes[rand].id)
                nodes.remove(nodes[rand])
                i = i - 1

        action_list = []
        for node_id in candidates:
            kwargs = {
                'name': 'node-delete-%s' % node_id,
                'context': self.context,
                'target': node_id,
                'cause': 'Cluster scale down',
            }
            action = base.Action(self.context, 'NODE_DELETE', **kwargs)
            action.store(self.context)

            action_list.append(action.id)
            db_api.action_add_dependency(action, self)
            action.set_status(self.READY)

        # Notify dispatcher
        for action_id in action_list:
            dispatcher.notify(self.context,
                              dispatcher.Dispatcher.NEW_ACTION,
                              None,
                              action_id=action_id)

        # Wait for cluster creating complete. If timeout,
        # set cluster status to error.
        # Note: we don't allow to cancel scaling operations.
        while self.get_status() != self.READY:
            if scheduler.action_timeout(self):
                # Action timeout, set cluster status to ERROR and return
                LOG.debug('Cluster scale_down action %s timeout' % self.id)
                cluster.set_status(cluster.ERROR,
                                   'Cluster scaling down timeout')
                return self.RES_TIMEOUT

            # Continue waiting (with sleep)
            scheduler.reschedule(self)

        cluster.delete_nodes(candidates)

        # set cluster status to OK
        return self.RES_OK

    def do_attach_policy(self, cluster):
        policy_id = self.inputs.get('policy_id', None)
        if not policy_id:
            raise exception.PolicyNotSpecified()

        policy = policies.Policy.load(self.context, policy_id)
        # Check if policy has already been attached
        all = db_api.cluster_get_policies(self.context, cluster.id)
        for existing in all:
            # Policy already attached
            if existing.policy_id == policy_id:
                return self.RES_OK

            # Detect policy type conflicts
            curr = policies.Policy.load(self.context, existing.policy_id)
            if curr.type == policy.type:
                raise exception.PolicyExists(policy_type=policy.type)

        values = {
            'cooldown': self.inputs.get('cooldown', policy.cooldown),
            'level': self.inputs.get('level', policy.level),
            'enabled': self.inputs.get('enabled', True),
        }

        db_api.cluster_attach_policy(self.context, cluster.id, policy_id,
                                     values)

        cluster.rt.policies.append(policy)
        return self.RES_OK

    def do_detach_policy(self, cluster):
        return self.RES_OK

    def _execute(self, cluster):
        # do pre-action policy checking
        check_result = self.policy_check(cluster.id, 'BEFORE')
        if not check_result:
            # Don't emit message here since policy_check should have done it
            return self.RES_ERROR

        res = self.RES_OK
        if self.action == self.CLUSTER_CREATE:
            res = self.do_create(cluster)
        elif self.action == self.CLUSTER_UPDATE:
            new_profile_id = self.inputs.get('new_profile_id')
            # TODO(anyone): check null?
            res = self.do_update(cluster, new_profile_id)
        elif self.action == self.CLUSTER_DELETE:
            res = self.do_delete(cluster)
        elif self.action == self.CLUSTER_ADD_NODES:
            res = self.do_add_nodes(cluster)
        elif self.action == self.CLUSTER_DEL_NODES:
            res = self.do_del_nodes(cluster)
        elif self.action == self.CLUSTER_SCALE_UP:
            res = self.do_scale_up(cluster)
        elif self.action == self.CLUSTER_SCALE_DOWN:
            count = check_result.get('count', 0)
            if count == 0:
                # We assume the policy has emit error/warnings
                return self.RES_ERROR
            candidates = check_result.get('candidates', [])
            res = self.do_scale_down(cluster, count, candidates)
        elif self.action == self.CLUSTER_ATTACH_POLICY:
            res = self.do_attach_policy(cluster)
        elif self.action == self.CLUSTER_DETACH_POLICY:
            res = self.do_detach_policy(cluster)

        # do post-action policy checking
        if res == self.RES_OK:
            check_result = self.policy_check(cluster.id, 'AFTER')
            res = self.RES_OK if check_result else self.ERROR

        return res

    def execute(self, **kwargs):
        try:
            cluster = cluster_mod.Cluster.load(self.context, self.target)
        except exception.NotFound:
            LOG.error(_LE('Cluster %(id)s not found') % {'id': self.target})
            return self.RES_ERROR

        steal_lock = kwargs.get('steal_lock', False)
        if self.action in [self.CLUSTER_DELETE]:
            steal_lock = True

        # Try to lock cluster before do real action
        res = self._cluster_lock_acquire(cluster, steal_lock)

        if res != self.RES_OK:
            return res

        # Do real operation here
        res = self._execute(cluster)

        # We've done, release cluster lock
        self._release_cluster_lock(cluster)
        return res

    def cancel(self):
        return self.RES_OK
<|endoftext|>


<commit_before>"""
Helpers for Python package hosting at GitHub.

Copyright (c) 2009, Damien Lebrun
All rights reserved.

Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:

    * Redistributions of source code must retain the above COPYRIGHT notice,
      this list of conditions and the following disclaimer.

    * Redistributions in binary form must reproduce the above COPYRIGHT notice,
      this list of conditions and the following disclaimer in the documentation
      and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""

PROJECT = 'github-tools'
DESCRIPTION = __doc__.strip().splitlines()[0]
AUTHOR = 'Damien Lebrun'
AUTHOR_EMAIL = 'dinoboff@hotmail.com'
VERSION_TUPLE = (0, 1, 0, 'beta', 1)
VERSION = '%s.%s' % VERSION_TUPLE[0:2]
RELEASE = '%s.%s.%s%.1s%s' % VERSION_TUPLE[0:5]

COPYRIGHT = __doc__.strip().splitlines()[2]
LICENCE = 'BSD'
LICENCE_NOTICE = '\n'.join(__doc__.strip().splitlines()[2:])<commit_msg>First release<commit_after>"""
Helpers for Python package hosting at GitHub.

Copyright (c) 2009, Damien Lebrun
All rights reserved.

Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:

    * Redistributions of source code must retain the above COPYRIGHT notice,
      this list of conditions and the following disclaimer.

    * Redistributions in binary form must reproduce the above COPYRIGHT notice,
      this list of conditions and the following disclaimer in the documentation
      and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""

PROJECT = 'github-tools'
DESCRIPTION = __doc__.strip().splitlines()[0]
AUTHOR = 'Damien Lebrun'
AUTHOR_EMAIL = 'dinoboff@hotmail.com'
VERSION_TUPLE = (0, 1, 0, '', '')
VERSION = '%s.%s' % VERSION_TUPLE[0:2]
RELEASE = '%s.%s.%s%.1s%s' % VERSION_TUPLE[0:5]

COPYRIGHT = __doc__.strip().splitlines()[2]
LICENCE = 'BSD'
LICENCE_NOTICE = '\n'.join(__doc__.strip().splitlines()[2:])<|endoftext|>


<commit_before>#!/usr/bin/env python
"""Builds the documentaion. First it runs gendoc to create rst files for the source code. Then it runs sphinx make.
.. Warning:: This will delete the content of the output directory first! So you might loose data.
             You can use updatedoc.py -nod.
Usage, just call::

  updatedoc.py -h

This is automatically executed when building with sphinx. Check the bottom of docs/conf.py.
"""
import argparse
import os
import logging
import shutil
import sys

import gendoc

log = logging.getLogger(__name__)
thisdir = os.path.abspath(os.path.dirname(__file__))

def setup_argparse():
    """Sets up the argument parser and returns it

    :returns: the parser
    :rtype: :class:`argparse.ArgumentParser`
    :raises: None
    """
    log.debug("Setting up argparser.")
    parser = argparse.ArgumentParser(
        description="Generate rst files for python source files.\
        WARNING: this will delete the contents of the output dirs. You can use -nod.")
    parser.add_argument('src', help='Path to the source directory')
    parser.add_argument('out', action='store',
                        help='Directory to place all output')
    parser.add_argument('exclude_paths', help='Paths to exclude', nargs='*')
    parser.add_argument('-f', '--force', action='store_true', dest='force',
                        help='Overwrite existing files')
    parser.add_argument('-l', '--follow-links', action='store_true',
                        dest='followlinks', default=False,
                        help='Follow symbolic links. Powerful when combined '
                        'with collective.recipe.omelette.')
    parser.add_argument('-n', '--dry-run', action='store_true', dest='dryrun',
                        help='Run the script without creating files')
    parser.add_argument('-P', '--private', action='store_true',
                        dest='includeprivate',
                        help='Include "_private" modules')
    parser.add_argument('-s', '--suffix', action='store', dest='suffix',
                        help='file suffix (default: rst)', default='rst')
    parser.add_argument('-nod', '--nodelete', action='store_true',
                        help='Do not empty the output directories first.')
    parser.add_argument('-v', '--verbosity', help="Also log debug messages.",
                        action='store_true')
    return parser

def prepare_dir(directory, delete=True):
    """Create apidoc dir, delete contents if delete is True.

    :param directory: the apidoc directory. you can use relative paths here
    :type directory: str
    :param delete: if True, deletes the contents of apidoc. This acts like an override switch.
    :type delete: bool
    :returns: None
    :rtype: None
    :raises: None
    """
    log.debug("Preparing output directories")
    if os.path.exists(directory):
        if delete:
            assert directory != thisdir, 'Trying to delete docs! Specify other output dir!'
            log.debug("Deleting %s", directory)
            shutil.rmtree(directory)
            log.debug("Creating %s", directory)
            os.mkdir(directory)
    else:
        log.debug("Creating %s", directory)
        os.mkdir(directory)

def main(argv=None):
    """Parse commandline arguments and run the tool

    :param argv: the commandline arguments.
    :type argv: list
    :returns: None
    :rtype: None
    :raises: None
    """
    if argv is None:
        argv = sys.argv[1:]
    parser = setup_argparse()
    args = parser.parse_args(argv)
    if args.verbosity:
        l = logging.getLogger()
        l.setLevel(logging.DEBUG)
    prepare_dir(args.out, not args.nodelete)
    gendoc.generate(args.src, args.out,
                    exclude=args.exclude_paths,
                    force=args.force,
                    followlinks = args.followlinks,
                    dryrun=args.dryrun,
                    private=args.includeprivate,
                    suffix=args.suffix)

if __name__ == '__main__':
    main()
<commit_msg>instead of parsing commandline use sphinx api<commit_after>#!/usr/bin/env python
"""Builds the documentaion. First it runs gendoc to create rst files for the source code. Then it runs sphinx make.
.. Warning:: This will delete the content of the output directory first! So you might loose data.
             You can use updatedoc.py -nod.
Usage, just call::

  updatedoc.py -h

This is automatically executed when building with sphinx. Check the bottom of docs/conf.py.
"""
import os
import shutil

import gendoc

def prepare_dir(app, directory, delete=False):
    """Create apidoc dir, delete contents if delete is True.

    :param app: the sphinx app
    :type app: :class:`sphinx.application.Sphinx`
    :param directory: the apidoc directory. you can use relative paths here
    :type directory: str
    :param delete: if True, deletes the contents of apidoc. This acts like an override switch.
    :type delete: bool
    :returns: None
    :rtype: None
    :raises: None
    """
    app.info("Preparing output directories for jinjaapidoc.")
    if os.path.exists(directory):
        if delete:
            app.debug("Deleting dir %s", directory)
            shutil.rmtree(directory)
            app.debug("Creating dir %s", directory)
            os.mkdir(directory)
    else:
        app.debug("Creating %s", directory)
        os.mkdir(directory)

def main(app):
    """Parse the config of the app and initiate the generation process

    :param app: the sphinx app
    :type app: :class:`sphinx.application.Sphinx`
    :returns: None
    :rtype: None
    :raises: None
    """
    c = app.config
    src = c.jinjaapi_srcdir

    if not src:
        return

    out = c.jinjaapi_outputdir or app.env.srcdir

    prepare_dir(app, out, not c.jinjaapi_nodelete)
    gendoc.generate(app, out, src,
                    exclude=c.jinjaapi_exclude_paths,
                    force=c.jinjaapi_force,
                    followlinks=c.jinjaapi_followlinks,
                    dryrun=c.jinjaapi_dryrun,
                    private=c.jinjaapi_includeprivate,
                    suffix=c.source_suffix,
                    template_dirs=c.jinjaapi_templatedirs)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import time
import json
from flask import request
from flask_cors import *
import kuas_api.kuas.cache as cache

from kuas_api.modules.stateless_auth import auth
import kuas_api.modules.stateless_auth as stateless_auth
import kuas_api.modules.error as error
from kuas_api.modules.json import jsonify
from .doc import auto

# Nestable blueprints problem
# not sure isn't this a best practice now.
# https://github.com/mitsuhiko/flask/issues/593
#from kuas_api.views.v2 import api_v2
routes = []

def route(rule, **options):
    def decorator(f):
        url_rule = {
            "rule": rule,
            "view_func": f,
            "options": options if options else {}
        }

        routes.append(url_rule)
        return f

    return decorator

@route('/bus/timetables')
@auto.doc(groups=["public"])
#@cross_origin(supports_credentials=True)
@auth.login_required
def timetables():
    """Get KUAS school bus time table.

    :query string date: Specific date to query timetable. format: yyyy-mm-dd
    :query string from: The start station you want to query. (not impl yet)
    :statuscode 200: no error

    **Requires authentication?**

      Yes

    **Request**

    without date (default the date on server)

    .. sourcecode:: http

        GET /latest/bus/timetables HTTP/1.1
        Host: kuas.grd.idv.tw:14769
        Authorization: Basic xxxxxxxxxxxxx=

    .. sourcecode:: shell

        curl -i -X GET localhost:8001/v2/bus/timetables

    with date

    .. sourcecode:: http

        GET /latest/bus/timetables?date=2015-9-1 HTTP/1.1
        Host: kuas.grd.idv.tw:14769
        Authorization: Basic xxxxxxxxxxxxx=

    **Response**

    .. sourcecode:: http

        HTTP/1.0 200 OK
        Content-Type: application/json

        {
          "timetable":[
            {
              "endStation":"燕巢",
              "EndEnrollDateTime":"2015-08-31 17:20",
              "isReserve":-1,
              "Time":"08:20",
              "busId":"27034",
              "limitCount":"999",
              "reserveCount":"27",
              "runDateTime":"2015-09-01 08:20"
            },
            {
              "endStation":"燕巢",
              "EndEnrollDateTime":"2015-09-01 08:00",
              "isReserve":-1,
              "Time":"13:00",
              "busId":"27062",
              "limitCount":"999",
              "reserveCount":"1",
              "runDateTime":"2015-09-01 13:00"
            },
            {
              "endStation":"建工",
              "EndEnrollDateTime":"2015-09-01 07:15",
              "isReserve":-1,
              "Time":"12:15",
              "busId":"27090",
              "limitCount":"999",
              "reserveCount":"5",
              "runDateTime":"2015-09-01 12:15"
            },
            {
              "endStation":"建工",
              "EndEnrollDateTime":"2015-09-01 11:45",
              "isReserve":-1,
              "Time":"16:45",
              "busId":"27118",
              "limitCount":"999",
              "reserveCount":"24",
              "runDateTime":"2015-09-01 16:45"
            }
          ],
          "date":"2015-9-1"
        }

    """

    date = time.strftime("%Y-%m-%d", time.gmtime())
    if request.args.get("date"):
        date = request.args.get("date")

    # Restore cookies
    s = stateless_auth.get_requests_session_with_cookies()

    return jsonify(date=date, timetable=cache.bus_query(s, date))

@route("/bus/reservations", methods=["GET"])
@route("/bus/reservations/<int:bus_id>", methods=["PUT"])
@route("/bus/reservations/<string:end_time>", methods=["DELETE"])
@auth.login_required
def bus_reservations(bus_id=None, end_time=None):
    """
    .. http:get:: /bus/reservations

      Get user's bus reservation.

      **Requires authentication?**

        Yes

      **Request**

        .. sourcecode:: http

      **Response**

        .. sourcecode:: http

    .. http:put:: /bus/reservations

        put

    .. http:delete:: /bus/reservations

        delete
    """

    # Restore cookies
    s = stateless_auth.get_requests_session_with_cookies()

    if request.method == "GET":
        return jsonify(reservation=cache.bus_reserve_query(s))
    elif request.method == "PUT":
        return jsonify(cache.bus_booking(s, bus_id, ""))
    elif request.method == "DELETE":
        return jsonify(cache.bus_booking(s, end_time, "un"))

    return request.method
<commit_msg>Update docstring for warining<commit_after># -*- coding: utf-8 -*-
import time
import json
from flask import request
from flask_cors import *
import kuas_api.kuas.cache as cache

from kuas_api.modules.stateless_auth import auth
import kuas_api.modules.stateless_auth as stateless_auth
import kuas_api.modules.error as error
from kuas_api.modules.json import jsonify
from .doc import auto

# Nestable blueprints problem
# not sure isn't this a best practice now.
# https://github.com/mitsuhiko/flask/issues/593
#from kuas_api.views.v2 import api_v2
routes = []

def route(rule, **options):
    def decorator(f):
        url_rule = {
            "rule": rule,
            "view_func": f,
            "options": options if options else {}
        }

        routes.append(url_rule)
        return f

    return decorator

@route('/bus/timetables')
@auto.doc(groups=["public"])
#@cross_origin(supports_credentials=True)
@auth.login_required
def timetables():
    """Get KUAS school bus time table.

    :query string date: Specific date to query timetable. format: yyyy-mm-dd
    :query string from: The start station you want to query. (not impl yet)
    :statuscode 200: no error

    **Requires authentication?**

      Yes

    **Request**

    without date (default the date on server)

    .. sourcecode:: http

        GET /latest/bus/timetables HTTP/1.1
        Host: kuas.grd.idv.tw:14769
        Authorization: Basic xxxxxxxxxxxxx=

    .. sourcecode:: shell

        curl -i -X GET localhost:8001/v2/bus/timetables

    with date

    .. sourcecode:: http

        GET /latest/bus/timetables?date=2015-9-1 HTTP/1.1
        Host: kuas.grd.idv.tw:14769
        Authorization: Basic xxxxxxxxxxxxx=

    **Response**

    .. sourcecode:: http

        HTTP/1.0 200 OK
        Content-Type: application/json

        {
          "timetable":[
            {
              "endStation":"燕巢",
              "EndEnrollDateTime":"2015-08-31 17:20",
              "isReserve":-1,
              "Time":"08:20",
              "busId":"27034",
              "limitCount":"999",
              "reserveCount":"27",
              "runDateTime":"2015-09-01 08:20"
            },
            {
              "endStation":"燕巢",
              "EndEnrollDateTime":"2015-09-01 08:00",
              "isReserve":-1,
              "Time":"13:00",
              "busId":"27062",
              "limitCount":"999",
              "reserveCount":"1",
              "runDateTime":"2015-09-01 13:00"
            },
            {
              "endStation":"建工",
              "EndEnrollDateTime":"2015-09-01 07:15",
              "isReserve":-1,
              "Time":"12:15",
              "busId":"27090",
              "limitCount":"999",
              "reserveCount":"5",
              "runDateTime":"2015-09-01 12:15"
            },
            {
              "endStation":"建工",
              "EndEnrollDateTime":"2015-09-01 11:45",
              "isReserve":-1,
              "Time":"16:45",
              "busId":"27118",
              "limitCount":"999",
              "reserveCount":"24",
              "runDateTime":"2015-09-01 16:45"
            }
          ],
          "date":"2015-9-1"
        }

    """

    date = time.strftime("%Y-%m-%d", time.gmtime())
    if request.args.get("date"):
        date = request.args.get("date")

    # Restore cookies
    s = stateless_auth.get_requests_session_with_cookies()

    return jsonify(date=date, timetable=cache.bus_query(s, date))

@route("/bus/reservations", methods=["GET"])
@route("/bus/reservations/<int:bus_id>", methods=["PUT"])
@route("/bus/reservations/<string:end_time>", methods=["DELETE"])
@auth.login_required
def bus_reservations(bus_id=None, end_time=None):
    """
    .. warnings::

      This endpoints is still under construct, return value `may` change quickly.
      Please check this documentations for update.

    .. http:get:: /bus/reservations

      Get user's bus reservation.

      **Requires authentication?**

        Yes

      **Request**

        .. sourcecode:: http

      **Response**

        .. sourcecode:: http

    .. http:put:: /bus/reservations

        put

    .. http:delete:: /bus/reservations

        delete
    """

    # Restore cookies
    s = stateless_auth.get_requests_session_with_cookies()

    if request.method == "GET":
        return jsonify(reservation=cache.bus_reserve_query(s))
    elif request.method == "PUT":
        return jsonify(cache.bus_booking(s, bus_id, ""))
    elif request.method == "DELETE":
        return jsonify(cache.bus_booking(s, end_time, "un"))

    return request.method
<|endoftext|>


<commit_before># (The MIT License)
#
# Copyright (c) 2013 Kura
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the 'Software'), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

"""
Blackhole is an email MTA that pipes all mail to /dev/null

Blackhole is just any other MTA out there except it does not
actual do any disk I/O with the mail it receives. It is simply
accept or rejected based on configuration and pretends it's
actually done something.
"""

__author__ = "Kura"
__copyright__ = "None"
__credits__ = ["Kura", ]
__license__ = "MIT"
__version__ = "1.9.0"
__maintainer__ = "Kura"
__email__ = "kura@kura.io"
__status__ = "Stable"

__pname__ = "blackhole"
__desc__ = "blackhole.io MTA"

__fullname__ = "%s %s" % (__pname__, __version__)
<commit_msg>New release<commit_after># (The MIT License)
#
# Copyright (c) 2013 Kura
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the 'Software'), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

"""
Blackhole is an email MTA that pipes all mail to /dev/null

Blackhole is just any other MTA out there except it does not
actual do any disk I/O with the mail it receives. It is simply
accept or rejected based on configuration and pretends it's
actually done something.
"""

__author__ = "Kura"
__copyright__ = "None"
__credits__ = ["Kura", ]
__license__ = "MIT"
__version__ = ""
__maintainer__ = "Kura"
__email__ = "kura@kura.io"
__status__ = "Stable"

__pname__ = "blackhole"
__desc__ = "blackhole.io MTA"

__fullname__ = "%s %s" % (__pname__, __version__)
<|endoftext|>