<commit_before>from __future__ import absolute_import, division
import json
import workflows

class CommonTransport(object):
  '''A common transport class, containing e.g. the logic to connect clients
     to message subscriptions and transactions, so that these can be cleanly
     terminated when the client goes away.'''

  __clients = {}
  __client_id = 0
  __subscriptions = {}
  __subscription_id = 0
  __transactions = {}
  __transaction_id = 0
  __messages = {}

  #
  # -- High level communication calls ----------------------------------------
  #

  def connect(self):
    '''Connect the transport class. This function has to be overridden.
       :return: True-like value when connection successful,
                False-like value otherwise.'''
    return False

  def subscribe(self, channel, callback, **kwargs):
    '''Listen to a queue, notify via callback function.
       :param channel: Queue name to subscribe to
       :param callback: Function to be called when messages are received
       :param **kwargs: Further parameters for the transport layer. For example
              client_id: Value tying a subscription to one client. This allows
                         removing all subscriptions for a client simultaneously
                         when the client goes away.
              exclusive: Attempt to become exclusive subscriber to the queue.
              acknowledgement: If true receipt of each message needs to be
                               acknowledged.
       :return: A unique subscription ID
    '''
    self.__subscription_id += 1
    self.__subscriptions[self.__subscription_id] = {
      'channel': channel,
      'client': kwargs.get('client_id'),
      'callback': callback,
      'ack': kwargs.get('acknowledgement'),
    }
    self._debug('Subscribing to %s for %s with ID %d' % \
        (channel, kwargs.get('client_id'), self.__subscription_id))
    if 'client_id' in kwargs:
      self.__clients[kwargs['client_id']]['subscriptions'].add \
        (self.__subscription_id)
      del(kwargs['client_id'])
    self._subscribe(self.__subscription_id, channel, callback, **kwargs)
    return self.__subscription_id

  def unsubscribe(self, subscription, **kwargs):
    '''Stop listening to a queue or a broadcast
       :param subscription: Subscription ID to cancel
       :param **kwargs: Further parameters for the transport layer.
    '''
    if subscription not in self.__subscriptions:
      raise workflows.WorkflowsError \
            ("Attempting to unsubscribe unknown subscription")
    if self.__subscriptions[subscription]['client']:
      self.__clients[self.__subscriptions[subscription]['client']] \
        ['subscriptions'].remove(subscription)
    self._unsubscribe(subscription, **kwargs)
    del(self.__subscriptions[subscription])

  def subscribe_broadcast(self, channel, callback, **kwargs):
    '''Listen to a broadcast topic, notify via callback function.
       :param channel: Topic name to subscribe to
       :param callback: Function to be called when messages are received
       :param client_id: Value tying a subscription to one client. This allows
                         removing all subscriptions for a client simultaneously
                         when the client goes away.
       :param **kwargs: Further parameters for the transport layer. For example
              retroactive: Ask broker to send old messages if possible
       :return: A unique subscription ID
    '''
    self.__subscription_id += 1
    self.__subscriptions[self.__subscription_id] = {
      'channel': channel,
      'client': kwargs.get('client_id'),
      'callback': callback,
      'ack': False,
    }
    if 'client_id' in kwargs:
      self.__clients[kwargs['client_id']]['subscriptions'].add \
        (self.__subscription_id)
      del(kwargs['client_id'])
    self._subscribe_broadcast(self.__subscription_id, channel, callback, \
        **kwargs)
    return self.__subscription_id

  def subscription_callback(self, subscription):
    '''Retrieve the callback function for a subscription. Raise a
       WorkflowsError if the subscription does not exist.
       :param subscription: Subscription ID to look up
       :return: Callback function
    '''
    if subscription not in self.__subscriptions:
      raise workflows.WorkflowsError \
            ("Attempting to callback on unknown subscription")
    return self.__subscriptions[subscription]['callback']

  def send(self, destination, message, **kwargs):
    '''Send a message to a queue.
       :param destination: Queue name to send to
       :param message: Either a string or a serializable object to be sent
       :param **kwargs: Further parameters for the transport layer. For example
              headers: Optional dictionary of header entries
              expiration: Optional expiration time, relative to sending time
              transaction: Transaction ID if message should be part of a
                           transaction
    '''
    if not isinstance(message, basestring):
      message = json.dumps(message)
    self._send(destination, message, **kwargs)

  def broadcast(self, destination, message, **kwargs):
    '''Broadcast a message.
       :param destination: Topic name to send to
       :param message: Either a string or a serializable object to be sent
       :param **kwargs: Further parameters for the transport layer. For example
              headers: Optional dictionary of header entries
              expiration: Optional expiration time, relative to sending time
              transaction: Transaction ID if message should be part of a
                           transaction
    '''
    if not isinstance(message, basestring):
      message = json.dumps(message)
    self._broadcast(destination, message, **kwargs)

  def ack(self, message_id, **kwargs):
    '''Acknowledge receipt of a message. This only makes sense when the
       'acknowledgement' flag was set for the relevant subscription.
       :param message_id: ID of the message to be acknowledged
       :param **kwargs: Further parameters for the transport layer. For example
              transaction: Transaction ID if acknowledgement should be part of
                           a transaction
    '''
    if message_id not in self.__messages:
      raise workflows.WorkflowsError \
            ("Attempting to ACK unknown message")
    subscription, client = self.__messages[message_id]
    if client:
      self.__clients[client]['messages'].remove(message_id)
    del(self.__messages[message_id])
    self._ack(message_id, subscription, **kwargs)

  def nack(self, message_id, **kwargs):
    '''Reject receipt of a message. This only makes sense when the
       'acknowledgement' flag was set for the relevant subscription.
       :param message_id: ID of the message to be rejected
       :param **kwargs: Further parameters for the transport layer. For example
              transaction: Transaction ID if rejection should be part of a
                           transaction
    '''
    if message_id not in self.__messages:
      raise workflows.WorkflowsError \
            ("Attempting to NACK unknown message")
    subscription, client = self.__messages[message_id]
    if client:
      self.__clients[client]['messages'].remove(message_id)
    del(self.__messages[message_id])
    self._nack(message_id, subscription, **kwargs)

  def subscription_requires_ack(self, subscription):
    '''Report if messages belonging to a particular subscription require to be
       acknowledged manually.
       :param subscription: ID of the subscription to test
       :return: Boolean value, true if messages need to be ACKed/NACKed.
    '''
    if subscription not in self.__subscriptions:
      raise workflows.WorkflowsError \
            ("Unknown subscription")
    return self.__subscriptions[subscription]['ack']

  def register_message(self, subscription, message_id):
    '''Mark an incoming message to be ACKed/NACKed. This is only relevant when
       'acknowledgement' flag was set for the relevant subscription, and will
       raise a WorkflowsError otherwise. The message_id will be tracked, and
       if it is not ACKed/NACKed, the transport layer will NACK it once it is
       clear that the message can no longer be processed (ie. when the
       associated client goes away).
       :param subscription: ID of the subscription that the incoming message
                            belongs to.
       :param message_id: ID of the message to be rejected
    '''
    if not self.subscription_requires_ack(subscription):
      raise workflows.WorkflowsError \
            ("Attempting to register message for subscription without"
             "acknowledgement flag set")
    client = self.__subscriptions[subscription]['client']
    self.__messages[message_id] = (subscription, client)
    if client:
      self.__clients[client]['messages'].add(message_id)

  def transaction_begin(self, **kwargs):
    '''Start a new transaction.
       :param **kwargs: Further parameters for the transport layer. For example
              client_id: Value tying a transaction to one client. This allows
                         aborting all transactions for a client simultaneously
                         when the client goes away.
       :return: A transaction ID that can be passed to other functions.
    '''
    self.__transaction_id += 1
    self.__transactions[self.__transaction_id] = {
      'client': kwargs.get('client_id'),
    }
    if 'client_id' in kwargs:
      self.__clients[kwargs['client_id']]['transactions'].add \
        (self.__transaction_id)
      del(kwargs['client_id'])
    self._transaction_begin(self.__transaction_id, **kwargs)
    return self.__transaction_id

  def transaction_abort(self, transaction_id, **kwargs):
    '''Abort a transaction and roll back all operations.
       :param transaction_id: ID of transaction to be aborted.
       :param **kwargs: Further parameters for the transport layer.
    '''
    if transaction_id not in self.__transactions:
      raise workflows.WorkflowsError("Attempting to abort unknown transaction")
    if self.__transactions[transaction_id]['client']:
      self.__clients[self.__transactions[transaction_id]['client']] \
        ['transactions'].remove(transaction_id)
    del(self.__transactions[transaction_id])
    self._transaction_abort(transaction_id, **kwargs)

  def transaction_commit(self, transaction_id, **kwargs):
    '''Commit a transaction.
       :param transaction_id: ID of transaction to be committed.
       :param **kwargs: Further parameters for the transport layer.
    '''
    if transaction_id not in self.__transactions:
      raise workflows.WorkflowsError("Attempting to commit unknown transaction")
    if self.__transactions[transaction_id]['client']:
      self.__clients[self.__transactions[transaction_id]['client']] \
        ['transactions'].remove(transaction_id)
    del(self.__transactions[transaction_id])
    self._transaction_commit(transaction_id, **kwargs)

  #
  # -- Client management calls -----------------------------------------------
  #

  def register_client(self):
    '''Generates a new unique client ID. Subscriptions and transactions can be
       tied to client IDs, so that they can be collectively dropped when
       clients go away.'''
    self.__client_id += 1
    self.__clients[self.__client_id] = { 'subscriptions': set(),
                                         'transactions': set(),
                                         'messages': set() }
    return self.__client_id

  def drop_client(self, client_id):
    '''Remove a client ID and all connected subscriptions, transactions and
       unacknowledged messages.
       :param client_id: Client to be dropped.'''
    if client_id not in self.__clients:
      raise workflows.WorkflowsError("Attempting to drop unregistered client")
    channel_subscriptions = list(self.__clients[client_id]['subscriptions'])
    for subscription in channel_subscriptions:
      self.unsubscribe(subscription)
    transactions = list(self.__clients[client_id]['transactions'])
    for transaction in transactions:
      self.transaction_abort(transaction)
    messages = list(self.__clients[client_id]['messages'])
    for message_id in messages:
      self.nack(message_id)
    del(self.__clients[client_id])

  #
  # -- Debugging -------------------------------------------------------------
  #

  @staticmethod
  def _debug(debug_info):
    '''An overrideable central debugging function.'''
    print(debug_info)

  #
  # -- Low level communication calls to be implemented by subclass -----------
  #

  def _subscribe(self, sub_id, channel, callback, **kwargs):
    '''Listen to a queue, notify via callback function.
       :param sub_id: ID for this subscription in the transport layer
       :param channel: Queue name to subscribe to
       :param callback: Function to be called when messages are received
       :param **kwargs: Further parameters for the transport layer. For example
              exclusive: Attempt to become exclusive subscriber to the queue.
              acknowledgement: If true receipt of each message needs to be
                               acknowledged.
    '''
    raise workflows.WorkflowsError("Transport interface not implemented")

  def _subscribe_broadcast(self, sub_id, channel, callback, **kwargs):
    '''Listen to a broadcast topic, notify via callback function.
       :param sub_id: ID for this subscription in the transport layer
       :param channel: Topic name to subscribe to
       :param callback: Function to be called when messages are received
       :param **kwargs: Further parameters for the transport layer. For example
              retroactive: Ask broker to send old messages if possible
    '''
    raise workflows.WorkflowsError("Transport interface not implemented")

  def _unsubscribe(self, sub_id):
    '''Stop listening to a queue or a broadcast
       :param sub_id: ID for this subscription in the transport layer
    '''
    raise workflows.WorkflowsError("Transport interface not implemented")

  def _send(self, destination, message, **kwargs):
    '''Send a message to a queue.
       :param destination: Queue name to send to
       :param message: A string to be sent
       :param **kwargs: Further parameters for the transport layer. For example
              headers: Optional dictionary of header entries
              expiration: Optional expiration time, relative to sending time
              transaction: Transaction ID if message should be part of a
                           transaction
    '''
    raise workflows.WorkflowsError("Transport interface not implemented")

  def _broadcast(self, destination, message, **kwargs):
    '''Broadcast a message.
       :param destination: Topic name to send to
       :param message: A string to be broadcast
       :param **kwargs: Further parameters for the transport layer. For example
              headers: Optional dictionary of header entries
              expiration: Optional expiration time, relative to sending time
              transaction: Transaction ID if message should be part of a
                           transaction
    '''
    raise workflows.WorkflowsError("Transport interface not implemented")

  def _ack(self, message_id, subscription, **kwargs):
    '''Acknowledge receipt of a message. This only makes sense when the
       'acknowledgement' flag was set for the relevant subscription.
       :param message_id: ID of the message to be acknowledged
       :param subscription: ID of the relevant subscriptiong
       :param **kwargs: Further parameters for the transport layer. For example
              transaction: Transaction ID if acknowledgement should be part of
                           a transaction
    '''
    raise workflows.WorkflowsError("Transport interface not implemented")

  def _nack(self, message_id, subscription, **kwargs):
    '''Reject receipt of a message. This only makes sense when the
       'acknowledgement' flag was set for the relevant subscription.
       :param message_id: ID of the message to be rejected
       :param subscription: ID of the relevant subscriptiong
       :param **kwargs: Further parameters for the transport layer. For example
              transaction: Transaction ID if rejection should be part of a
                           transaction
    '''
    raise workflows.WorkflowsError("Transport interface not implemented")

  def _transaction_begin(self, transaction_id, **kwargs):
    '''Start a new transaction.
       :param transaction_id: ID for this transaction in the transport layer.
       :param **kwargs: Further parameters for the transport layer.
    '''
    raise workflows.WorkflowsError("Transport interface not implemented")

  def _transaction_abort(self, transaction_id, **kwargs):
    '''Abort a transaction and roll back all operations.
       :param transaction_id: ID of transaction to be aborted.
       :param **kwargs: Further parameters for the transport layer.
    '''
    raise workflows.WorkflowsError("Transport interface not implemented")

  def _transaction_commit(self, transaction_id, **kwargs):
    '''Commit a transaction.
       :param transaction_id: ID of transaction to be committed.
       :param **kwargs: Further parameters for the transport layer.
    '''
    raise workflows.WorkflowsError("Transport interface not implemented")

  #
  # -- Plugin-related function -----------------------------------------------
  #

  class __metaclass__(type):
    '''Define metaclass function to keep a list of all subclasses. This enables
       looking up transport mechanisms by name.'''
    def __init__(cls, name, base, attrs):
      '''Add new subclass of CommonTransport to list of all known subclasses.'''
      if not hasattr(cls, 'transport_register'):
        cls.transport_register = {}
      else:
        cls.transport_register[name] = cls
<commit_msg>Clarify documentation<commit_after>from __future__ import absolute_import, division
import json
import workflows

class CommonTransport(object):
  '''A common transport class, containing e.g. the logic to connect clients
     to message subscriptions and transactions, so that these can be cleanly
     terminated when the client goes away.'''

  __clients = {}
  __client_id = 0
  __subscriptions = {}
  __subscription_id = 0
  __transactions = {}
  __transaction_id = 0
  __messages = {}

  #
  # -- High level communication calls ----------------------------------------
  #

  def connect(self):
    '''Connect the transport class. This function has to be overridden.
       :return: True-like value when connection successful,
                False-like value otherwise.'''
    return False

  def subscribe(self, channel, callback, **kwargs):
    '''Listen to a queue, notify via callback function.
       :param channel: Queue name to subscribe to
       :param callback: Function to be called when messages are received.
                        The callback will pass two arguments, the header as a
                        dictionary structure, and the message.
       :param **kwargs: Further parameters for the transport layer. For example
              client_id: Value tying a subscription to one client. This allows
                         removing all subscriptions for a client simultaneously
                         when the client goes away.
              exclusive: Attempt to become exclusive subscriber to the queue.
              acknowledgement: If true receipt of each message needs to be
                               acknowledged.
       :return: A unique subscription ID
    '''
    self.__subscription_id += 1
    self.__subscriptions[self.__subscription_id] = {
      'channel': channel,
      'client': kwargs.get('client_id'),
      'callback': callback,
      'ack': kwargs.get('acknowledgement'),
    }
    self._debug('Subscribing to %s for %s with ID %d' % \
        (channel, kwargs.get('client_id'), self.__subscription_id))
    if 'client_id' in kwargs:
      self.__clients[kwargs['client_id']]['subscriptions'].add \
        (self.__subscription_id)
      del(kwargs['client_id'])
    self._subscribe(self.__subscription_id, channel, callback, **kwargs)
    return self.__subscription_id

  def unsubscribe(self, subscription, **kwargs):
    '''Stop listening to a queue or a broadcast
       :param subscription: Subscription ID to cancel
       :param **kwargs: Further parameters for the transport layer.
    '''
    if subscription not in self.__subscriptions:
      raise workflows.WorkflowsError \
            ("Attempting to unsubscribe unknown subscription")
    if self.__subscriptions[subscription]['client']:
      self.__clients[self.__subscriptions[subscription]['client']] \
        ['subscriptions'].remove(subscription)
    self._unsubscribe(subscription, **kwargs)
    del(self.__subscriptions[subscription])

  def subscribe_broadcast(self, channel, callback, **kwargs):
    '''Listen to a broadcast topic, notify via callback function.
       :param channel: Topic name to subscribe to
       :param callback: Function to be called when messages are received.
                        The callback will pass two arguments, the header as a
                        dictionary structure, and the message.
       :param client_id: Value tying a subscription to one client. This allows
                         removing all subscriptions for a client simultaneously
                         when the client goes away.
       :param **kwargs: Further parameters for the transport layer. For example
              retroactive: Ask broker to send old messages if possible
       :return: A unique subscription ID
    '''
    self.__subscription_id += 1
    self.__subscriptions[self.__subscription_id] = {
      'channel': channel,
      'client': kwargs.get('client_id'),
      'callback': callback,
      'ack': False,
    }
    if 'client_id' in kwargs:
      self.__clients[kwargs['client_id']]['subscriptions'].add \
        (self.__subscription_id)
      del(kwargs['client_id'])
    self._subscribe_broadcast(self.__subscription_id, channel, callback, \
        **kwargs)
    return self.__subscription_id

  def subscription_callback(self, subscription):
    '''Retrieve the callback function for a subscription. Raise a
       WorkflowsError if the subscription does not exist.
       :param subscription: Subscription ID to look up
       :return: Callback function
    '''
    if subscription not in self.__subscriptions:
      raise workflows.WorkflowsError \
            ("Attempting to callback on unknown subscription")
    return self.__subscriptions[subscription]['callback']

  def send(self, destination, message, **kwargs):
    '''Send a message to a queue.
       :param destination: Queue name to send to
       :param message: Either a string or a serializable object to be sent
       :param **kwargs: Further parameters for the transport layer. For example
              headers: Optional dictionary of header entries
              expiration: Optional expiration time, relative to sending time
              transaction: Transaction ID if message should be part of a
                           transaction
    '''
    if not isinstance(message, basestring):
      message = json.dumps(message)
    self._send(destination, message, **kwargs)

  def broadcast(self, destination, message, **kwargs):
    '''Broadcast a message.
       :param destination: Topic name to send to
       :param message: Either a string or a serializable object to be sent
       :param **kwargs: Further parameters for the transport layer. For example
              headers: Optional dictionary of header entries
              expiration: Optional expiration time, relative to sending time
              transaction: Transaction ID if message should be part of a
                           transaction
    '''
    if not isinstance(message, basestring):
      message = json.dumps(message)
    self._broadcast(destination, message, **kwargs)

  def ack(self, message_id, **kwargs):
    '''Acknowledge receipt of a message. This only makes sense when the
       'acknowledgement' flag was set for the relevant subscription.
       :param message_id: ID of the message to be acknowledged
       :param **kwargs: Further parameters for the transport layer. For example
              transaction: Transaction ID if acknowledgement should be part of
                           a transaction
    '''
    if message_id not in self.__messages:
      raise workflows.WorkflowsError \
            ("Attempting to ACK unknown message")
    subscription, client = self.__messages[message_id]
    if client:
      self.__clients[client]['messages'].remove(message_id)
    del(self.__messages[message_id])
    self._ack(message_id, subscription, **kwargs)

  def nack(self, message_id, **kwargs):
    '''Reject receipt of a message. This only makes sense when the
       'acknowledgement' flag was set for the relevant subscription.
       :param message_id: ID of the message to be rejected
       :param **kwargs: Further parameters for the transport layer. For example
              transaction: Transaction ID if rejection should be part of a
                           transaction
    '''
    if message_id not in self.__messages:
      raise workflows.WorkflowsError \
            ("Attempting to NACK unknown message")
    subscription, client = self.__messages[message_id]
    if client:
      self.__clients[client]['messages'].remove(message_id)
    del(self.__messages[message_id])
    self._nack(message_id, subscription, **kwargs)

  def subscription_requires_ack(self, subscription):
    '''Report if messages belonging to a particular subscription require to be
       acknowledged manually.
       :param subscription: ID of the subscription to test
       :return: Boolean value, true if messages need to be ACKed/NACKed.
    '''
    if subscription not in self.__subscriptions:
      raise workflows.WorkflowsError \
            ("Unknown subscription")
    return self.__subscriptions[subscription]['ack']

  def register_message(self, subscription, message_id):
    '''Mark an incoming message to be ACKed/NACKed. This is only relevant when
       'acknowledgement' flag was set for the relevant subscription, and will
       raise a WorkflowsError otherwise. The message_id will be tracked, and
       if it is not ACKed/NACKed, the transport layer will NACK it once it is
       clear that the message can no longer be processed (ie. when the
       associated client goes away).
       :param subscription: ID of the subscription that the incoming message
                            belongs to.
       :param message_id: ID of the message to be rejected
    '''
    if not self.subscription_requires_ack(subscription):
      raise workflows.WorkflowsError \
            ("Attempting to register message for subscription without"
             "acknowledgement flag set")
    client = self.__subscriptions[subscription]['client']
    self.__messages[message_id] = (subscription, client)
    if client:
      self.__clients[client]['messages'].add(message_id)

  def transaction_begin(self, **kwargs):
    '''Start a new transaction.
       :param **kwargs: Further parameters for the transport layer. For example
              client_id: Value tying a transaction to one client. This allows
                         aborting all transactions for a client simultaneously
                         when the client goes away.
       :return: A transaction ID that can be passed to other functions.
    '''
    self.__transaction_id += 1
    self.__transactions[self.__transaction_id] = {
      'client': kwargs.get('client_id'),
    }
    if 'client_id' in kwargs:
      self.__clients[kwargs['client_id']]['transactions'].add \
        (self.__transaction_id)
      del(kwargs['client_id'])
    self._transaction_begin(self.__transaction_id, **kwargs)
    return self.__transaction_id

  def transaction_abort(self, transaction_id, **kwargs):
    '''Abort a transaction and roll back all operations.
       :param transaction_id: ID of transaction to be aborted.
       :param **kwargs: Further parameters for the transport layer.
    '''
    if transaction_id not in self.__transactions:
      raise workflows.WorkflowsError("Attempting to abort unknown transaction")
    if self.__transactions[transaction_id]['client']:
      self.__clients[self.__transactions[transaction_id]['client']] \
        ['transactions'].remove(transaction_id)
    del(self.__transactions[transaction_id])
    self._transaction_abort(transaction_id, **kwargs)

  def transaction_commit(self, transaction_id, **kwargs):
    '''Commit a transaction.
       :param transaction_id: ID of transaction to be committed.
       :param **kwargs: Further parameters for the transport layer.
    '''
    if transaction_id not in self.__transactions:
      raise workflows.WorkflowsError("Attempting to commit unknown transaction")
    if self.__transactions[transaction_id]['client']:
      self.__clients[self.__transactions[transaction_id]['client']] \
        ['transactions'].remove(transaction_id)
    del(self.__transactions[transaction_id])
    self._transaction_commit(transaction_id, **kwargs)

  #
  # -- Client management calls -----------------------------------------------
  #

  def register_client(self):
    '''Generates a new unique client ID. Subscriptions and transactions can be
       tied to client IDs, so that they can be collectively dropped when
       clients go away.'''
    self.__client_id += 1
    self.__clients[self.__client_id] = { 'subscriptions': set(),
                                         'transactions': set(),
                                         'messages': set() }
    return self.__client_id

  def drop_client(self, client_id):
    '''Remove a client ID and all connected subscriptions, transactions and
       unacknowledged messages.
       :param client_id: Client to be dropped.'''
    if client_id not in self.__clients:
      raise workflows.WorkflowsError("Attempting to drop unregistered client")
    channel_subscriptions = list(self.__clients[client_id]['subscriptions'])
    for subscription in channel_subscriptions:
      self.unsubscribe(subscription)
    transactions = list(self.__clients[client_id]['transactions'])
    for transaction in transactions:
      self.transaction_abort(transaction)
    messages = list(self.__clients[client_id]['messages'])
    for message_id in messages:
      self.nack(message_id)
    del(self.__clients[client_id])

  #
  # -- Debugging -------------------------------------------------------------
  #

  @staticmethod
  def _debug(debug_info):
    '''An overrideable central debugging function.'''
    print(debug_info)

  #
  # -- Low level communication calls to be implemented by subclass -----------
  #

  def _subscribe(self, sub_id, channel, callback, **kwargs):
    '''Listen to a queue, notify via callback function.
       :param sub_id: ID for this subscription in the transport layer
       :param channel: Queue name to subscribe to
       :param callback: Function to be called when messages are received
       :param **kwargs: Further parameters for the transport layer. For example
              exclusive: Attempt to become exclusive subscriber to the queue.
              acknowledgement: If true receipt of each message needs to be
                               acknowledged.
    '''
    raise workflows.WorkflowsError("Transport interface not implemented")

  def _subscribe_broadcast(self, sub_id, channel, callback, **kwargs):
    '''Listen to a broadcast topic, notify via callback function.
       :param sub_id: ID for this subscription in the transport layer
       :param channel: Topic name to subscribe to
       :param callback: Function to be called when messages are received
       :param **kwargs: Further parameters for the transport layer. For example
              retroactive: Ask broker to send old messages if possible
    '''
    raise workflows.WorkflowsError("Transport interface not implemented")

  def _unsubscribe(self, sub_id):
    '''Stop listening to a queue or a broadcast
       :param sub_id: ID for this subscription in the transport layer
    '''
    raise workflows.WorkflowsError("Transport interface not implemented")

  def _send(self, destination, message, **kwargs):
    '''Send a message to a queue.
       :param destination: Queue name to send to
       :param message: A string to be sent
       :param **kwargs: Further parameters for the transport layer. For example
              headers: Optional dictionary of header entries
              expiration: Optional expiration time, relative to sending time
              transaction: Transaction ID if message should be part of a
                           transaction
    '''
    raise workflows.WorkflowsError("Transport interface not implemented")

  def _broadcast(self, destination, message, **kwargs):
    '''Broadcast a message.
       :param destination: Topic name to send to
       :param message: A string to be broadcast
       :param **kwargs: Further parameters for the transport layer. For example
              headers: Optional dictionary of header entries
              expiration: Optional expiration time, relative to sending time
              transaction: Transaction ID if message should be part of a
                           transaction
    '''
    raise workflows.WorkflowsError("Transport interface not implemented")

  def _ack(self, message_id, subscription, **kwargs):
    '''Acknowledge receipt of a message. This only makes sense when the
       'acknowledgement' flag was set for the relevant subscription.
       :param message_id: ID of the message to be acknowledged
       :param subscription: ID of the relevant subscriptiong
       :param **kwargs: Further parameters for the transport layer. For example
              transaction: Transaction ID if acknowledgement should be part of
                           a transaction
    '''
    raise workflows.WorkflowsError("Transport interface not implemented")

  def _nack(self, message_id, subscription, **kwargs):
    '''Reject receipt of a message. This only makes sense when the
       'acknowledgement' flag was set for the relevant subscription.
       :param message_id: ID of the message to be rejected
       :param subscription: ID of the relevant subscriptiong
       :param **kwargs: Further parameters for the transport layer. For example
              transaction: Transaction ID if rejection should be part of a
                           transaction
    '''
    raise workflows.WorkflowsError("Transport interface not implemented")

  def _transaction_begin(self, transaction_id, **kwargs):
    '''Start a new transaction.
       :param transaction_id: ID for this transaction in the transport layer.
       :param **kwargs: Further parameters for the transport layer.
    '''
    raise workflows.WorkflowsError("Transport interface not implemented")

  def _transaction_abort(self, transaction_id, **kwargs):
    '''Abort a transaction and roll back all operations.
       :param transaction_id: ID of transaction to be aborted.
       :param **kwargs: Further parameters for the transport layer.
    '''
    raise workflows.WorkflowsError("Transport interface not implemented")

  def _transaction_commit(self, transaction_id, **kwargs):
    '''Commit a transaction.
       :param transaction_id: ID of transaction to be committed.
       :param **kwargs: Further parameters for the transport layer.
    '''
    raise workflows.WorkflowsError("Transport interface not implemented")

  #
  # -- Plugin-related function -----------------------------------------------
  #

  class __metaclass__(type):
    '''Define metaclass function to keep a list of all subclasses. This enables
       looking up transport mechanisms by name.'''
    def __init__(cls, name, base, attrs):
      '''Add new subclass of CommonTransport to list of all known subclasses.'''
      if not hasattr(cls, 'transport_register'):
        cls.transport_register = {}
      else:
        cls.transport_register[name] = cls
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals
import socket
import email
import os
import sys
import mimetypes

ADDR = ("127.0.0.1", 8000)
_CRLF = b"\r\n"
_RESPONSE_TEMPLATE = _CRLF.join([
    b"HTTP/1.1 {status_code} {reason_phrase}",
    b"{date}",
    b"Content-Type: {content_type}",
    # b"Content-Length: {content_length}",
    _CRLF,
    b"{content_body}",
    _CRLF,
])

def setup():
    """
    Create new socket, and bind localhost to socket.
    Set socket to listen, and return socket information.
    """
    sock = socket.socket(
        socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_IP
    )
    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    sock.bind(ADDR)
    sock.listen(4)
    return sock

def response_ok(body_plugin, content_type):
    """Zip together arguments for returning an OK response"""

    _date = email.Utils.formatdate(usegmt=True)
    content_body = b"{body_plugin}"
    content_body = content_body.format(body_plugin=body_plugin)

    return _RESPONSE_TEMPLATE.format(
        status_code=b"200",
        reason_phrase=b"OK",
        date=_date,
        content_type=content_type,
        # content_length=sys.getsizeof(content_body),
        content_body=content_body
    )

def response_error(status_code, reason_phrase, body_plugin):
    """Zip together arguments for returning an error response"""

    date = email.Utils.formatdate(usegmt=True)
    content_body = b"{body_plugin}"
    content_body = content_body.format(body_plugin=body_plugin)

    return _RESPONSE_TEMPLATE.format(
        status_code=status_code,
        reason_phrase=reason_phrase,
        date=date,
        content_type=b'text/html',
        # content_length=sys.getsizeof(content_body),
        content_body=content_body
    )

def parse_request(request):
    client_req = request.split((2 * _CRLF), 1)
    head_chunk = client_req[0].split(_CRLF)
    first_line = head_chunk[0].split()
    host = ''

    for item in head_chunk:
        if "Host: " in item:
            host = item
    # import pdb; pdb.set_trace()
    if 'GET' != first_line[0]:
        raise NotImplementedError('That is not a valid GET request')
    elif 'HTTP/1.1' != first_line[2]:
        raise NameError('That is not a valid HTTP/1.1 request')
    elif 'Host: ' not in host:
        raise ValueError('The required Host header is not present')
    else:
        return first_line[1]

def resolve_uri(parse):
    if '..' in parse:
        raise IOError
    # import pdb; pdb.set_trace()
    here = os.path.dirname(os.path.abspath(__file__))
    root = os.path.join(here, 'webroot')
    body = ''
    content_type = ''
    uri = os.path.join(root, parse)
    if os.path.isdir(uri):
        body = '<DOCTYPE html>
'
        for file_ in os.listdir(root + parse):
            body += '
' + file_ + '
'
        body += '
'
        content_type = 'text/html'
    elif os.path.isfile(root + parse):
        with open((root + parse), 'rb') as file_:
            body = file_.read()
        content_type, encoding = mimetypes.guess_type(parse)
    else:
        raise IOError
    body = bytes(body)
    content_type = bytes(content_type)
    return (body, content_type)

def run_server():
    """
    Create new instance of server, and begin accepting, logging,
    and returning response.
    """
    server = setup()
    while True:
        try:
            conn, addr = server.accept()
            msg = ''
            while True:
                """
                If response in msg, can use this to return Ok or Error
                """
                msg_recv = conn.recv(4096)
                msg += msg_recv
                if len(msg_recv) < 4096:
                    try:
                        parsed_response = parse_request(msg)
                        body, content_type = resolve_uri(parsed_response)
                        server_response = response_ok(body, content_type)
                    except NotImplementedError:
                        server_response = response_error(
                            b"405",
                            b"Method Not Allowed",
                            b"GET method required."
                        )
                    except NameError:
                        server_response = response_error(
                            b"400",
                            b"Bad Request",
                            b"Not a valid HTTP/1.1 request."
                        )
                    except ValueError:
                        server_response = response_error(
                            b"406",
                            b"Not Acceptable",
                            b"'Host' header required."
                        )
                    except IOError:
                        server_response = response_error(
                            b"404",
                            b"Not Found",
                            b"Requested resource does not exist."
                        )
                    conn.sendall(server_response)
                    conn.close()
                    break
            print(msg)
        except KeyboardInterrupt:
            break

if __name__ == "__main__":
    run_server()
<commit_msg>Image dir in response, but not files.<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals
import socket
import email
import os
from pathlib import Path
# import sys
import mimetypes

ADDR = ("127.0.0.1", 8000)
_CRLF = b"\r\n"
_RESPONSE_TEMPLATE = _CRLF.join([
    b"HTTP/1.1 {status_code} {reason_phrase}",
    b"{date}",
    b"Content-Type: {content_type}",
    # b"Content-Length: {content_length}",
    _CRLF,
    b"{content_body}",
    _CRLF,
])

def setup():
    """
    Create new socket, and bind localhost to socket.
    Set socket to listen, and return socket information.
    """
    sock = socket.socket(
        socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_IP
    )
    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    sock.bind(ADDR)
    sock.listen(4)
    return sock

def response_ok(body_plugin, content_type):
    """Zip together arguments for returning an OK response"""

    _date = email.Utils.formatdate(usegmt=True)
    content_body = b"{body_plugin}"
    content_body = content_body.format(body_plugin=body_plugin)

    return _RESPONSE_TEMPLATE.format(
        status_code=b"200",
        reason_phrase=b"OK",
        date=_date,
        content_type=content_type,
        # content_length=sys.getsizeof(content_body),
        content_body=content_body
    )

def response_error(status_code, reason_phrase, body_plugin):
    """Zip together arguments for returning an error response"""

    date = email.Utils.formatdate(usegmt=True)
    content_body = b"{body_plugin}"
    content_body = content_body.format(body_plugin=body_plugin)

    return _RESPONSE_TEMPLATE.format(
        status_code=status_code,
        reason_phrase=reason_phrase,
        date=date,
        content_type=b'text/html',
        # content_length=sys.getsizeof(content_body),
        content_body=content_body
    )

def parse_request(request):
    client_req = request.split((2 * _CRLF), 1)
    head_chunk = client_req[0].split(_CRLF)
    first_line = head_chunk[0].split()
    host = ''

    for item in head_chunk:
        if "Host: " in item:
            host = item
    # import pdb; pdb.set_trace()
    if 'GET' != first_line[0]:
        raise NotImplementedError('That is not a valid GET request')
    elif 'HTTP/1.1' != first_line[2]:
        raise NameError('That is not a valid HTTP/1.1 request')
    elif 'Host: ' not in host:
        raise ValueError('The required Host header is not present')
    else:
        return first_line[1]

def resolve_uri(parse):
    if '..' in parse:
        raise IOError
    here = os.path.dirname(os.path.abspath(__file__))
    root = os.path.join(here, 'webroot')

    path = Path(root + parse)
    if not path.exists():
        raise LookupError(b'Not Found')

    body = b''
    content_type = b''
    if path.is_dir():
        # import pdb; pdb.set_trace()
        gen = path.iterdir()
        html_start = b"<!DOCTYPE html>
"
        body = _CRLF.join(
            [b"
{item}
".format(item=item) for item in gen]
        )
        html_end = b"
"
        body = "".join([html_start, body, html_end])
        content_type = b'text/html'

    elif os.path.isfile(root + parse):
        with path.open('rb') as f:
            body = f.read()
            content_type, encoding = mimetypes.guess_type(parse)[0]
    else:
        raise IOError
    return (body, content_type)

# def resolve_uri(parse):
#     if '..' in parse:
#         raise IOError
#     here = os.path.dirname(os.path.abspath(__file__))
#     root = os.path.join(here, 'webroot')
#     import pdb; pdb.set_trace()
#     body = ''
#     content_type = ''
#     uri = os.path.join(root, parse)
#     if os.path.isdir(uri):
#         body = '<DOCTYPE html>
'
#         for file_ in os.listdir(root + parse):
#             body += '
' + file_ + '
'
#         body += '
'
#         content_type = 'text/html'
#     elif os.path.isfile(root + parse):
#         with open((root + parse), 'rb') as file_:
#             body = file_.read()
#         content_type, encoding = mimetypes.guess_type(parse)
#     else:
#         raise IOError
#     body = bytes(body)
#     content_type = bytes(content_type)
#     return (body, content_type)

def run_server():
    """
    Create new instance of server, and begin accepting, logging,
    and returning response.
    """
    server = setup()
    while True:
        try:
            conn, addr = server.accept()
            msg = ''
            while True:
                """
                If response in msg, can use this to return Ok or Error
                """
                msg_recv = conn.recv(4096)
                msg += msg_recv
                if len(msg_recv) < 4096:
                    try:
                        parsed_response = parse_request(msg)
                        body, content_type = resolve_uri(parsed_response)
                        server_response = response_ok(body, content_type)
                    except NotImplementedError:
                        server_response = response_error(
                            b"405",
                            b"Method Not Allowed",
                            b"GET method required."
                        )
                    except NameError:
                        server_response = response_error(
                            b"400",
                            b"Bad Request",
                            b"Not a valid HTTP/1.1 request."
                        )
                    except ValueError:
                        server_response = response_error(
                            b"406",
                            b"Not Acceptable",
                            b"'Host' header required."
                        )
                    except IOError:
                        server_response = response_error(
                            b"404",
                            b"Not Found",
                            b"Requested resource does not exist."
                        )
                    conn.sendall(server_response)
                    conn.close()
                    break
            print(msg)
        except KeyboardInterrupt:
            break

if __name__ == "__main__":
    run_server()
<|endoftext|>


<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Endpoints for the executor.

The endpoints are:
1) '/': Endpoint for executing imports on GitHub commits.
2) '/update': Endpoint for updating imports.
3) '/schedule': Endpoint for scheduling cron jobs for updating imports upon
                GitHub commits.
"""

import dataclasses

import flask
from google.cloud import scheduler

from app import configs
from app.executor import validation
from app.executor import update_scheduler
from app.executor import import_executor
from app.service import file_uploader
from app.service import dashboard_api
from app.service import github_api
from app.service import email_notifier
from app.service import import_service

def create_app():
    """Creates the Flask app."""
    return flask.Flask(__name__)

FLASK_APP = create_app()

@FLASK_APP.route('/', methods=['POST'])
def execute_imports():
    """Endpoint for executing imports on GitHub commits.

    Logs to the import progress dashboard.
    """
    task_info = flask.request.get_json(force=True)
    if 'COMMIT_SHA' not in task_info:
        return {'error': 'COMMIT_SHA not found'}
    commit_sha = task_info['COMMIT_SHA']
    repo_name = task_info.get('REPO_NAME')
    branch_name = task_info.get('BRANCH_NAME')
    pr_number = task_info.get('PR_NUMBER')

    task_configs = task_info.get('configs', {})
    config = configs.ExecutorConfig(**task_configs)
    executor = import_executor.ImportExecutor(
        uploader=file_uploader.GCSFileUploader(
            project_id=config.gcs_project_id,
            bucket_name=config.storage_dev_bucket_name,
            path_prefix=config.storage_executor_output_prefix),
        github=github_api.GitHubRepoAPI(
            repo_owner_username=config.github_repo_owner_username,
            repo_name=config.github_repo_name,
            auth_username=config.github_auth_username,
            auth_access_token=config.github_auth_access_token),
        config=config,
        dashboard=dashboard_api.DashboardAPI(config.dashboard_oauth_client_id),
        notifier=email_notifier.EmailNotifier(config.email_account,
                                              config.email_token),
        importer=import_service.ImportServiceClient(
            project_id=config.gcs_project_id,
            executor_output_prefix=config.storage_executor_output_prefix,
            importer_output_prefix=config.storage_importer_output_prefix,
            unresolved_mcf_bucket_name=config.storage_dev_bucket_name,
            resolved_mcf_bucket_name=config.storage_importer_bucket_name,
            client_id=config.importer_oauth_client_id))
    result = executor.execute_imports_on_commit(commit_sha=commit_sha,
                                                repo_name=repo_name,
                                                branch_name=branch_name,
                                                pr_number=pr_number)
    return dataclasses.asdict(result)

@FLASK_APP.route('/update', methods=['POST'])
def scheduled_updates():
    """Endpoint for updating imports.

    TODO(intrepiditee): Log to the import progress dashboard.
    """
    task_info = flask.request.get_json(force=True)
    if 'absolute_import_name' not in task_info:
        return {'error': 'absolute_import_name not found'}
    task_configs = task_info.get('configs', {})
    config = configs.ExecutorConfig(**task_configs)
    executor = import_executor.ImportExecutor(
        uploader=file_uploader.GCSFileUploader(
            project_id=config.gcs_project_id,
            bucket_name=config.storage_prod_bucket_name),
        github=github_api.GitHubRepoAPI(
            repo_owner_username=config.github_repo_owner_username,
            repo_name=config.github_repo_name,
            auth_username=config.github_auth_username,
            auth_access_token=config.github_auth_access_token),
        config=config)
    result = executor.execute_imports_on_update(
        task_info['absolute_import_name'])
    return dataclasses.asdict(result)

@FLASK_APP.route('/schedule', methods=['POST'])
def schedule_crons():
    """Endpoint for scheduling cron jobs for updating imports upon
    GitHub commits."""
    task_info = flask.request.get_json(force=True)
    if 'COMMIT_SHA' not in task_info:
        return 'COMMIT_SHA not found'
    task_configs = task_info.get('configs', {})
    config = configs.ExecutorConfig(**task_configs)
    import_scheduler = update_scheduler.UpdateScheduler(
        client=scheduler.CloudSchedulerClient(),
        github=github_api.GitHubRepoAPI(
            repo_owner_username=config.github_repo_owner_username,
            repo_name=config.github_repo_name,
            auth_username=config.github_auth_username,
            auth_access_token=config.github_auth_access_token),
        config=config)
    return dataclasses.asdict(
        import_scheduler.schedule_on_commit(task_info['COMMIT_SHA']))

@FLASK_APP.route('/_ah/start')
def start():
    """Handles start up calls from App Engine."""
    return ''

def main():
    """Runs the app locally."""
    FLASK_APP.run(host='127.0.0.1', port=8080, debug=True)
<commit_msg>enable dashboard (#259)<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Endpoints for the executor.

The endpoints are:
1) '/': Endpoint for executing imports on GitHub commits.
2) '/update': Endpoint for updating imports.
3) '/schedule': Endpoint for scheduling cron jobs for updating imports upon
                GitHub commits.
"""

import dataclasses

import flask
from google.cloud import scheduler

from app import configs
from app.executor import validation
from app.executor import update_scheduler
from app.executor import import_executor
from app.service import file_uploader
from app.service import dashboard_api
from app.service import github_api
from app.service import email_notifier
from app.service import import_service

def create_app():
    """Creates the Flask app."""
    return flask.Flask(__name__)

FLASK_APP = create_app()

@FLASK_APP.route('/', methods=['POST'])
def execute_imports():
    """Endpoint for executing imports on GitHub commits."""
    task_info = flask.request.get_json(force=True)
    if 'COMMIT_SHA' not in task_info:
        return {'error': 'COMMIT_SHA not found'}
    commit_sha = task_info['COMMIT_SHA']
    repo_name = task_info.get('REPO_NAME')
    branch_name = task_info.get('BRANCH_NAME')
    pr_number = task_info.get('PR_NUMBER')

    task_configs = task_info.get('configs', {})
    config = configs.ExecutorConfig(**task_configs)
    executor = import_executor.ImportExecutor(
        uploader=file_uploader.GCSFileUploader(
            project_id=config.gcs_project_id,
            bucket_name=config.storage_dev_bucket_name,
            path_prefix=config.storage_executor_output_prefix),
        github=github_api.GitHubRepoAPI(
            repo_owner_username=config.github_repo_owner_username,
            repo_name=config.github_repo_name,
            auth_username=config.github_auth_username,
            auth_access_token=config.github_auth_access_token),
        config=config,
        dashboard=dashboard_api.DashboardAPI(config.dashboard_oauth_client_id),
        notifier=email_notifier.EmailNotifier(config.email_account,
                                              config.email_token),
        importer=import_service.ImportServiceClient(
            project_id=config.gcs_project_id,
            executor_output_prefix=config.storage_executor_output_prefix,
            importer_output_prefix=config.storage_importer_output_prefix,
            unresolved_mcf_bucket_name=config.storage_dev_bucket_name,
            resolved_mcf_bucket_name=config.storage_importer_bucket_name,
            client_id=config.importer_oauth_client_id))
    result = executor.execute_imports_on_commit(commit_sha=commit_sha,
                                                repo_name=repo_name,
                                                branch_name=branch_name,
                                                pr_number=pr_number)
    return dataclasses.asdict(result)

@FLASK_APP.route('/update', methods=['POST'])
def scheduled_updates():
    """Endpoint for updating imports."""
    task_info = flask.request.get_json(force=True)
    if 'absolute_import_name' not in task_info:
        return {'error': 'absolute_import_name not found'}
    task_configs = task_info.get('configs', {})
    config = configs.ExecutorConfig(**task_configs)
    executor = import_executor.ImportExecutor(
        uploader=file_uploader.GCSFileUploader(
            project_id=config.gcs_project_id,
            bucket_name=config.storage_prod_bucket_name),
        github=github_api.GitHubRepoAPI(
            repo_owner_username=config.github_repo_owner_username,
            repo_name=config.github_repo_name,
            auth_username=config.github_auth_username,
            auth_access_token=config.github_auth_access_token),
        dashboard=dashboard_api.DashboardAPI(config.dashboard_oauth_client_id),
        config=config)
    result = executor.execute_imports_on_update(
        task_info['absolute_import_name'])
    return dataclasses.asdict(result)

@FLASK_APP.route('/schedule', methods=['POST'])
def schedule_crons():
    """Endpoint for scheduling cron jobs for updating imports upon
    GitHub commits."""
    task_info = flask.request.get_json(force=True)
    if 'COMMIT_SHA' not in task_info:
        return 'COMMIT_SHA not found'
    task_configs = task_info.get('configs', {})
    config = configs.ExecutorConfig(**task_configs)
    import_scheduler = update_scheduler.UpdateScheduler(
        client=scheduler.CloudSchedulerClient(),
        github=github_api.GitHubRepoAPI(
            repo_owner_username=config.github_repo_owner_username,
            repo_name=config.github_repo_name,
            auth_username=config.github_auth_username,
            auth_access_token=config.github_auth_access_token),
        config=config)
    return dataclasses.asdict(
        import_scheduler.schedule_on_commit(task_info['COMMIT_SHA']))

@FLASK_APP.route('/_ah/start')
def start():
    """Handles start up calls from App Engine."""
    return ''

def main():
    """Runs the app locally."""
    FLASK_APP.run(host='127.0.0.1', port=8080, debug=True)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
from two1.lib.wallet import Wallet
from two1.lib.bitserv.flask import Payment
from flask import Flask, jsonify

import urllib.request as request
import json
import requests

app = Flask(__name__)
wallet = Wallet()
payment = Payment(app, wallet)

def bad_request(message):
    response = jsonify({'message': message})
    response.status_code = 400
    return response

def get_price(request):
    data_to_send = {
        "customer": {
            "firstName": "Alex",
            "lastName": "Bitcoins",
            "address": {
                "Street": "532 Tyrella Ave #39",
                "City": "Mountain View",
                "Region": "CA",
                "PostalCode": "94043"
            },
            "phone": "8023564779",
            "email": "habs7707@gmail.com"
        },

        "item": {
            "code": "W40PHOTW",
            "options": [],
            "quantity": 1
        },

        "storeID": "7931"
    }
    r = requests.post(url='http://localhost:3000/validateAndPrice', json=request.data)
    response_status = json.loads(r.text)["result"]["Status"]

    if int(response_status) == 1 or int(response_status) == 0:
        price_in_usd = json.loads(r.text)["result"]["Order"]["Amounts"]["Payment"]

        get_bitpay_btc_usd_rate = request.urlopen(url="https://bitpay.com/api/rates/usd").read().decode("utf-8")
        usd_per_btc = json.loads(get_bitpay_btc_usd_rate)["rate"]

        # price = int(price_in_usd * 10**8 / usd_per_btc)
        price = 100
    else:
        setattr(request, 'error_validate', 'error_validate')
        price = 0

    return price

@app.route('/order', methods=['POST'])
@payment.required(get_price)
def order():
    if hasattr(request, 'error_validate'):
        return bad_request('There is a problem with your order details.')

    return 'Pizza ordered!'

@app.route('/validate', methods=['POST'])
def validate():
    price = get_price(request)
    return price

@app.route('/findNearbyStore', methods=['GET'])
def findNearbyStore():
  zip_code = request.args.get('zipCode')
  print(zip_code)
  some_url = EXPRESS_SERVER + 'findStores/' + zip_code
  r = request.get(url=some_url)
  # print(r)
  # print(r.result)

  return 'awesome'

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=5000)
<commit_msg>server<->order<commit_after>#!/usr/bin/env python3
from two1.lib.wallet import Wallet
from two1.lib.bitserv.flask import Payment
from flask import Flask, jsonify

import urllib.request as request
import json
import requests

app = Flask(__name__)
wallet = Wallet()
payment = Payment(app, wallet)

def bad_request(message):
    response = jsonify({'message': message})
    response.status_code = 400
    return response

def get_price(request):
    data_to_send = {
        "customer": {
            "firstName": "Alex",
            "lastName": "Bitcoins",
            "address": {
                "Street": "532 Tyrella Ave #39",
                "City": "Mountain View",
                "Region": "CA",
                "PostalCode": "94043"
            },
            "phone": "8023564779",
            "email": "habs7707@gmail.com"
        },

        "item": {
            "code": "W40PHOTW",
            "options": [],
            "quantity": 1
        },

        "storeID": "7931"
    }
    r = requests.post(url='http://localhost:3000/validateAndPrice', json=request.data)
    response_status = json.loads(r.text)["result"]["Status"]

    if int(response_status) == 1 or int(response_status) == 0:
        price_in_usd = json.loads(r.text)["result"]["Order"]["Amounts"]["Payment"]

        get_bitpay_btc_usd_rate = request.urlopen(url="https://bitpay.com/api/rates/usd").read().decode("utf-8")
        usd_per_btc = json.loads(get_bitpay_btc_usd_rate)["rate"]

        # price = int(price_in_usd * 10**8 / usd_per_btc)
        price = 100
    else:
        setattr(request, 'error_validate', 'error_validate')
        price = 0

    return price

@app.route('/order', methods=['POST'])
@payment.required(get_price)
def order():
    if hasattr(request, 'error_validate'):
        return bad_request('There is a problem with your order details.')

    return 'Pizza ordered!'

@app.route('/validate', methods=['POST'])
def validate():
    price = get_price(request)
    return price

@app.route('/findNearbyStore', methods=['GET'])
def findNearbyStore():
  zip_code = request.args.get('zipCode')
  print(zip_code)
  some_url = 'http://localhost:3000/findStores/' + zip_code
  r = request.get(url=some_url)
  # print(r)
  # print(r.result)

  return 'awesome'

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=5000)
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""A bot which will respond to various Die Hard character name commands and
mentions and respond with a random line spoken by that character in the film.
"""

from errbot import botcmd
from errbot.botplugin import BotPlugin
from errbot.utils import get_sender_username
import config
import logging

from dieHard import DieHard

def generate(character):
    f = lambda self, mess, args: "(%s) %s" % (character,
            self.diehard.get_random(character))
    f.__name__ = character
    f.__doc__ = "Get a random quote from %s." % character.title()
    return f

class DieHardBotBuilder(type):
    def __new__(mcs, name, bases, classDict):
        newClassDict = dict(classDict.items() +
                            [(character, botcmd(generate(character)))
                            for character in DieHard.CHARACTERS])
        return super(DieHardBotBuilder, mcs).__new__(mcs, name, bases,
                                                     newClassDict)

class DieHardBot(BotPlugin):
    __metaclass__ = DieHardBotBuilder
    min_err_version = "1.6.0"

    def __init__(self):
        super(BotPlugin, self).__init__()
        self.diehard = DieHard()

    def callback_message(self, conn, mess):
        """Listen for Die Hard mentions and interject random lines from those
        characters who were mentioned.
        """
        logging.debug("mess.getFrom().getStripped(): " + mess.getFrom().getStripped())
        logging.debug("config.BOT_IDENTITY['username']: " + config.BOT_IDENTITY["username"])
        logging.debug("get_sender_username(mess): " + get_sender_username(mess))
        logging.debug("config.CHATROOM_FN: " + config.CHATROOM_FN)
        if (mess.getFrom().getStripped() == config.BOT_IDENTITY["username"]) or (get_sender_username(mess) == config.CHATROOM_FN):
            logging.debug("Ignore a message from myself")
            return False

        message = ""
        for character in DieHard.CHARACTERS:
            if mess.getBody().find("(%s)" % character) != -1:
                message = "(%s) %s" % (character,
                                       self.diehard.get_random(character))
                break
        if message:
            self.send(mess.getFrom(), message, message_type=mess.getType())
<commit_msg>Append random quotes from multiple characters if mentioned multiple times.<commit_after>#!/usr/bin/env python

"""A bot which will respond to various Die Hard character name commands and
mentions and respond with a random line spoken by that character in the film.
"""

from errbot import botcmd
from errbot.botplugin import BotPlugin
from errbot.utils import get_sender_username
import config
import logging

from dieHard import DieHard

def generate(character):
    f = lambda self, mess, args: "(%s) %s" % (character,
            self.diehard.get_random(character))
    f.__name__ = character
    f.__doc__ = "Get a random quote from %s." % character.title()
    return f

class DieHardBotBuilder(type):
    def __new__(mcs, name, bases, classDict):
        newClassDict = dict(classDict.items() +
                            [(character, botcmd(generate(character)))
                            for character in DieHard.CHARACTERS])
        return super(DieHardBotBuilder, mcs).__new__(mcs, name, bases,
                                                     newClassDict)

class DieHardBot(BotPlugin):
    __metaclass__ = DieHardBotBuilder
    min_err_version = "1.6.0"

    def __init__(self):
        super(BotPlugin, self).__init__()
        self.diehard = DieHard()

    def callback_message(self, conn, mess):
        """Listen for Die Hard mentions and interject random lines from those
        characters who were mentioned.
        """
        logging.debug("mess.getFrom().getStripped(): " + mess.getFrom().getStripped())
        logging.debug("config.BOT_IDENTITY['username']: " + config.BOT_IDENTITY["username"])
        logging.debug("get_sender_username(mess): " + get_sender_username(mess))
        logging.debug("config.CHATROOM_FN: " + config.CHATROOM_FN)
        if (mess.getFrom().getStripped() == config.BOT_IDENTITY["username"]) or (get_sender_username(mess) == config.CHATROOM_FN):
            logging.debug("Ignore a message from myself")
            return False

        message = ""
        for character in DieHard.CHARACTERS:
            if mess.getBody().find("(%s)" % character) != -1:
                message += "(%s) %s  " % (character, self.diehard.get_random(character))
        if message:
            self.send(mess.getFrom(), message, message_type=mess.getType())
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Wed Nov 26 20:07:18 2014

@author: rlabbe
"""

import math
from numpy import array, dot, asarray
from numpy.random import randn
import matplotlib.pyplot as plt

from filterpy.kalman import UnscentedKalmanFilter as UKF
from filterpy.kalman import ScaledUnscentedKalmanFilter as SUKF
from filterpy.common import Q_discrete_white_noise

""" Note that the commented out code contains code from the old formulation
of the scaled unscented transform which no longer exists"""

""" This is an example of the bearing only problem. You have a platform,
usually a ship, that can only get the bearing to a moving target. Assuming
platform is stationary, this is a very difficult problem because there are
an infinite number of solutions. The literature is filled with this example,
along with proposed solutions (usually, platform makes manuevers).

"""
dt = 0.1
y = 20
platform_pos=(0,20)

sf = SUKF(2, 1, dt, alpha=1.e-4, beta=2., kappa=1.)
sf.Q = Q_discrete_white_noise(2, dt, .1)

f = UKF(2, 1, dt, kappa=0.)
f.Q = Q_discrete_white_noise(2, dt, .1)

def fx(x,dt):
    """ state transition function"""

    # pos = pos + vel
    # vel = vel
    return array([x[0]+x[1], x[1]])

def hx(x):
    """ measurement function - convert position to bearing"""

    return math.atan2(platform_pos[1],x[0]-platform_pos[0])

xs_scaled = []
xs = []
for i in range(300):
    angle = hx([i+randn()*.1, 0]) + randn()
    sf.update(angle, hx, fx)
    xs_scaled.append(sf.x)

    f.update(angle, hx, fx)
    xs.append(f.x)

xs_scaled = asarray(xs_scaled)
xs = asarray(xs)

plt.subplot(211)
plt.plot(xs_scaled[:,0],label='scaled')
plt.plot(xs[:,0], label='Julier')
plt.legend(loc=4)

plt.subplot(212)
plt.plot(xs_scaled[:,1],label='scaled')
plt.plot(xs[:,1], label='Julier')
plt.legend(loc=4)
plt.show()

<commit_msg>Removed obsolete comment.<commit_after># -*- coding: utf-8 -*-
"""
Created on Wed Nov 26 20:07:18 2014

@author: rlabbe
"""

import math
from numpy import array, asarray
from numpy.random import randn
import matplotlib.pyplot as plt

from filterpy.kalman import UnscentedKalmanFilter as UKF
from filterpy.kalman import ScaledUnscentedKalmanFilter as SUKF
from filterpy.common import Q_discrete_white_noise

""" This is an example of the bearing only problem. You have a platform,
usually a ship, that can only get the bearing to a moving target. Assuming
platform is stationary, this is a very difficult problem because there are
an infinite number of solutions. The literature is filled with this example,
along with proposed solutions (usually, platform makes manuevers).

"""
dt = 0.1
y = 20
platform_pos=(0,20)

sf = SUKF(2, 1, dt, alpha=1.e-4, beta=2., kappa=1.)
sf.Q = Q_discrete_white_noise(2, dt, .1)

f = UKF(2, 1, dt, kappa=0.)
f.Q = Q_discrete_white_noise(2, dt, .1)

def fx(x,dt):
    """ state transition function"""

    # pos = pos + vel
    # vel = vel
    return array([x[0]+x[1], x[1]])

def hx(x):
    """ measurement function - convert position to bearing"""

    return math.atan2(platform_pos[1],x[0]-platform_pos[0])

xs_scaled = []
xs = []
for i in range(300):
    angle = hx([i+randn()*.1, 0]) + randn()
    sf.update(angle, hx, fx)
    xs_scaled.append(sf.x)

    f.update(angle, hx, fx)
    xs.append(f.x)

xs_scaled = asarray(xs_scaled)
xs = asarray(xs)

plt.subplot(211)
plt.plot(xs_scaled[:,0],label='scaled')
plt.plot(xs[:,0], label='Julier')
plt.legend(loc=4)

plt.subplot(212)
plt.plot(xs_scaled[:,1],label='scaled')
plt.plot(xs[:,1], label='Julier')
plt.legend(loc=4)
plt.show()

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from flask import Flask
import time
import twitter_rss

app = Flask(__name__)

@app.route("/")
def hello():
    return "Goto domain.com/twitterusername.xml"

@app.route('/<username>.xml')
def twitter_to_xml(username):
    try:
        with open(username + '.xml') as tweets:
            tweets = tweets.read()
            return tweets
    except IOError:
        print 'File does not exist: Creating feed...'

        tweets = twitter_rss.UserTweetGetter(username)
        with open(username + '.xml', 'w') as cache:
            cache.write(tweets.to_rss().encode('utf-8'))
        cache.close()

        return tweets.to_rss()

@app.route('/tag-<username>.xml')
def hashtag_to_xml(username):
    tweets = twitter_rss.HashtagTweetGetter(username)
    return tweets.to_rss()

if __name__ == "__main__":
    app.run(debug=True)<commit_msg>works now with hashtag AND user<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from flask import Flask, render_template, request, redirect, url_for
import time
import twitter_rss
# from jinja2 import Template
host = 'localhost:5000'

app = Flask(__name__)

@app.route('/')
def index():
    return render_template('index.tpl')

@app.route('/user', methods=['POST'])
def handle_user():
    feed = request.form['user_data']
    return redirect(url_for('user_to_xml', feed=feed))

@app.route('/htag', methods=['POST'])
def handle_htag():
    feed = request.form['user_data']
    return redirect(url_for('hashtag_to_xml', feed=feed))

@app.route('/user/<feed>.xml')
def user_to_xml(feed):
    try:
        with open('user/' + feed + '.xml') as tweets:
            tweets = tweets.read()
            return tweets
    except IOError:
        print 'File does not exist: Creating feed...'

        tweets = twitter_rss.UserTweetGetter(feed)
        with open('user/' + feed + '.xml', 'w') as cache:
            cache.write(tweets.to_rss().encode('utf-8'))
        cache.close()

        return tweets.to_rss()

@app.route('/htag/<feed>.xml')
def hashtag_to_xml(feed):
    try:
        with open('htag/' + feed + '.xml') as tweets:
            tweets = tweets.read()
            return tweets
    except IOError:
        print 'File does not exist: Creating feed...'

        tweets = twitter_rss.HashtagTweetGetter(feed)
        with open('htag/' + feed + '.xml', 'w') as cache:
            cache.write(tweets.to_rss().encode('utf-8'))
        cache.close()

        return tweets.to_rss()

# @app.route('/feed', methods=['POST'])
# def handle_username():
#     try:
#         return redirect(url_for('twitter_to_xml', username=request.form['rss']))
#     except:
#         return redirect(url_for('hashtag_to_xml', hashtag=request.form['rss']))

# @app.route('/<username>.xml')
# def twitter_to_xml(username):
#     try:
#         with open(username + '.xml') as tweets:
#             tweets = tweets.read()
#             return tweets
#     except IOError:
#         print 'File does not exist: Creating feed...'

#         tweets = twitter_rss.UserTweetGetter(username)
#         with open(username + '.xml', 'w') as cache:
#             cache.write(tweets.to_rss().encode('utf-8'))
#         cache.close()

#         return tweets.to_rss()

# @app.route('/tag-<hashtag>.xml')
# def hashtag_to_xml(hashtag):
#     try:
#         with open('tag-' + hashtag + '.xml') as tweets:
#             tweets = tweets.read()
#             return tweets
#     except IOError:
#         print 'File does not exist: Creating feed...'

#         tweets = twitter_rss.HashtagTweetGetter(hashtag)
#         with open('tag-' + hashtag + '.xml', 'w') as cache:
#             cache.write(tweets.to_rss().encode('utf-8'))
#         cache.close()

#     return tweets.to_rss()

@app.errorhandler(404)
def page_not_found(error):
    return redirect(url_for('index'))
    # error = 'Account or hashtag not found!'
    # return render_template('index.tpl', error=error)

@app.errorhandler(500)
def page_not_found(error):
    return redirect(url_for('index'))

if __name__ == "__main__":
    app.run(debug=True)
    # app.run(host='0.0.0.0')
    # app.run()<|endoftext|>


<commit_before># vim:ts=4:sw=4:expandtab
'''An event hub that supports sockets and timers, based
on Python 2.6's select & epoll support.
'''
try:
    import select26 as select
except ImportError:
    import select

from collections import deque
from time import time
import thread
from Queue import Queue, Empty
import fcntl
import os

class Timer(object):
    '''A timer is a promise to call some function at a future date.
    '''
    ALLOWANCE = 0.03 # If we're within 30ms, the timer is due
    def __init__(self, hub, interval, f, *args, **kw):
        self.hub = hub
        self.trigger_time = time() + interval
        self.f = f
        self.args = args
        self.kw = kw
        self.pending = True
        self.inq = False

    def cancel(self):
        self.pending = False
        if self.inq:
            self.inq = False
            self.hub.timers.remove(self)
            self.hub = None

    def callback(self):
        '''When the external entity checks this timer and determines
        it's due, this function is called, which calls the original
        callback.
        '''
        self.pending = False
        self.inq = False
        self.hub = None
        return self.f(*self.args, **self.kw)

    @property
    def due(self):
        '''Is it time to run this timer yet?

        The allowance provides some give-and-take so that if a
        sleep() delay comes back a little early, we still go.
        '''
        return (self.trigger_time - time()) < self.ALLOWANCE

    def __cmp__(self, o):
        return cmp(self.trigger_time, o.trigger_time)

class _PipeWrap(object):
    def __init__(self, p):
        self.p = p

    def fileno(self):
        return self.p

class AbstractEventHub(object):
    def __init__(self):
        self.timers = deque()
        self.new_timers = []
        self.run = True
        self.events = {}
        self.run_now = deque()
        self.fdmap = {}
        self._setup_threading()

    def _setup_threading(self):
        self._t_recv, self._t_wakeup = os.pipe()
        fcntl.fcntl(self._t_recv, fcntl.F_SETFL, os.O_NONBLOCK)
        fcntl.fcntl(self._t_wakeup, fcntl.F_SETFL, os.O_NONBLOCK)
        self.thread_comp_in = Queue()

        def handle_thread_done():
            try:
                os.read(self._t_recv, 65536)
            except IOError:
                pass
            while True:
                try:
                    c, v = self.thread_comp_in.get(False)
                except Empty:
                    break
                else:
                    c(v)
        self.register(_PipeWrap(self._t_recv), handle_thread_done, None, None)

    def run_in_thread(self, reschedule, f, *args, **kw):
        def wrap():
            try:
                res = f(*args, **kw)
            except Exception, e:
                self.thread_comp_in.put((reschedule, e))
            else:
                self.thread_comp_in.put((reschedule, res))
            self.wake_from_other_thread()
        thread.start_new_thread(wrap, ())

    def wake_from_other_thread(self):
        try:
            os.write(self._t_wakeup, '\0')
        except IOError:
            pass

    def handle_events(self):
        '''Run one pass of event handling.
        '''
        raise NotImplementedError

    def call_later(self, interval, f, *args, **kw):
        '''Schedule a timer on the hub.
        '''
        t = Timer(self, interval, f, *args, **kw)
        self.new_timers.append(t)
        return t

    def schedule(self, c):
        self.run_now.append(c)

    def register(self, fd, read_callback, write_callback, error_callback):
        '''Register a socket fd with the hub, providing callbacks
        for read (data is ready to be recv'd) and write (buffers are
        ready for send()).

        By default, only the read behavior will be polled and the
        read callback used until enable_write is invoked.
        '''
        fn = fd.fileno()
        self.fdmap[fd] = fn
        assert fn not in self.events
        self.events[fn] = (read_callback, write_callback, error_callback)
        self._add_fd(fd)

    def _add_fd(self, fd):
        '''Add this socket to the list of sockets used in the
        poll call.
        '''
        raise NotImplementedError

    def enable_write(self, fd):
        '''Enable write polling and the write callback.
        '''
        raise NotImplementedError

    def disable_write(self, fd):
        '''Disable write polling and the write callback.
        '''
        raise NotImplementedError

    def unregister(self, fd):
        '''Remove this socket from the list of sockets the
        hub is polling on.
        '''
        if fd in self.fdmap:
            fn = self.fdmap.pop(fd)
            del self.events[fn]

            self._remove_fd(fd)

    def _remove_fd(self, fd):
        '''Remove this socket from the list of sockets the
        hub is polling on.
        '''
        raise NotImplementedError

class SelectEventHub(AbstractEventHub):
    '''A select-based hub.
    '''
    def __init__(self):
        self.in_set, self.out_set = set(), set()
        super(SelectEventHub, self).__init__()

    def handle_events(self):
        '''Run one pass of event handling.

        select() is called, with a timeout equal to the next-scheduled
        timer.  When select returns, all fd-related events (if any) are
        handled, and timers are handled as well.
        '''
        while self.run_now and self.run:
            self.run_now.popleft()()

        if self.new_timers:
            for tr in self.new_timers:
                if tr.pending:
                    tr.inq = True
                    self.timers.append(tr)
            self.timers = deque(sorted(self.timers))
            self.new_timers = []

        tm = time()
        timeout = (self.timers[0].trigger_time - tm) if self.timers else 1e6
        # epoll, etc, limit to 2^31/1000 or OverflowError
        timeout = min(timeout, 1e6)
        if timeout < 0:
            timeout = 0

        # Run timers first, to try to nail their timings
        while self.timers:
            if self.timers[0].due:
                t = self.timers.popleft()
                if t.pending:
                    t.callback()
                    while self.run_now and self.run:
                        self.run_now.popleft()()
                    if not self.run:
                        return
            else:
                break

        select_lists = select.select(self.in_set, self.out_set, [], timeout)
        for i, fds in enumerate(select_lists):
            for fd in fds:
                self.events[fd.fileno()][i]()
                while self.run_now and self.run:
                    self.run_now.popleft()()

                if not self.run:
                    return

    def _add_fd(self, fd):
        '''Add this socket to the list of sockets used in the
        poll call.
        '''
        self.in_set.add(fd)

    def enable_write(self, fd):
        '''Enable write polling and the write callback.
        '''
        self.out_set.add(fd)

    def disable_write(self, fd):
        '''Disable write polling and the write callback.
        '''
        if fd in self.out_set:
            self.out_set.remove(fd)

    def _remove_fd(self, fd):
        '''Remove this socket from the list of sockets the
        hub is polling on.
        '''
        self.in_set.remove(fd)
        if fd in self.out_set:
            self.out_set.remove(fd)

class EPollEventHub(AbstractEventHub):
    '''A epoll-based hub.
    '''
    SIZE_HINT = 50000
    def __init__(self):
        self.epoll = select.epoll(self.SIZE_HINT)
        super(EPollEventHub, self).__init__()

    def handle_events(self):
        '''Run one pass of event handling.

        epoll() is called, with a timeout equal to the next-scheduled
        timer.  When epoll returns, all fd-related events (if any) are
        handled, and timers are handled as well.
        '''
        while self.run_now and self.run:
            self.run_now.popleft()()

        if self.new_timers:
            for tr in self.new_timers:
                if tr.pending:
                    tr.inq = True
                    self.timers.append(tr)
            self.timers = deque(sorted(self.timers))
            self.new_timers = []

        tm = time()
        timeout = (self.timers[0].trigger_time - tm) if self.timers else 1e6
        # epoll, etc, limit to 2^^31/1000 or OverflowError
        timeout = min(timeout, 1e6)
        if timeout < 0:
            timeout = 0

        # Run timers first, to try to nail their timings
        while self.timers:
            if self.timers[0].due:
                t = self.timers.popleft()
                if t.pending:
                    t.callback()
                    while self.run_now and self.run:
                        self.run_now.popleft()()
                    if not self.run:
                        return
            else:
                break

        # Handle all socket I/O
        for (fd, evtype) in self.epoll.poll(timeout):
            if evtype & select.EPOLLIN or evtype & select.EPOLLPRI:
                self.events[fd][0]()
            elif evtype & select.EPOLLERR or evtype & select.EPOLLHUP:
                self.events[fd][2]()
            # fd could be removed by above read
            if evtype & select.EPOLLOUT and fd in self.events:
                self.events[fd][1]()

            while self.run_now and self.run:
                self.run_now.popleft()()

            if not self.run:
                return

    def _add_fd(self, fd):
        '''Add this socket to the list of sockets used in the
        poll call.
        '''
        self.epoll.register(fd, select.EPOLLIN | select.EPOLLPRI)

    def enable_write(self, fd):
        '''Enable write polling and the write callback.
        '''
        self.epoll.modify(
                fd, select.EPOLLIN | select.EPOLLPRI | select.EPOLLOUT)

    def disable_write(self, fd):
        '''Disable write polling and the write callback.
        '''
        self.epoll.modify(fd, select.EPOLLIN | select.EPOLLPRI)

    def _remove_fd(self, fd):
        '''Remove this socket from the list of sockets the
        hub is polling on.
        '''
        self.epoll.unregister(fd)

# Expose a usable EventHub implementation
if os.environ.get('DIESEL_NO_EPOLL') or not hasattr(select, 'epoll'):
    EventHub = SelectEventHub
else:
    EventHub = EPollEventHub

if __name__ == '__main__':
    hub = EventHub()
    def whatever(message, other=None):
        print 'got', message, other
    hub.call_later(3.0, whatever, 'yes!', other='rock!')
    import socket, sys
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.bind(('', 11911))
    s.listen(5)
    hub.register(s, lambda: sys.stdout.write('new socket!'), lambda: sys.stdout.write('arg!'))
    while True:
        hub.handle_events()
<commit_msg>epoll size hint has been ignored since Linux 2.6.8<commit_after># vim:ts=4:sw=4:expandtab
'''An event hub that supports sockets and timers, based
on Python 2.6's select & epoll support.
'''
try:
    import select26 as select
except ImportError:
    import select

from collections import deque
from time import time
import thread
from Queue import Queue, Empty
import fcntl
import os

class Timer(object):
    '''A timer is a promise to call some function at a future date.
    '''
    ALLOWANCE = 0.03 # If we're within 30ms, the timer is due
    def __init__(self, hub, interval, f, *args, **kw):
        self.hub = hub
        self.trigger_time = time() + interval
        self.f = f
        self.args = args
        self.kw = kw
        self.pending = True
        self.inq = False

    def cancel(self):
        self.pending = False
        if self.inq:
            self.inq = False
            self.hub.timers.remove(self)
            self.hub = None

    def callback(self):
        '''When the external entity checks this timer and determines
        it's due, this function is called, which calls the original
        callback.
        '''
        self.pending = False
        self.inq = False
        self.hub = None
        return self.f(*self.args, **self.kw)

    @property
    def due(self):
        '''Is it time to run this timer yet?

        The allowance provides some give-and-take so that if a
        sleep() delay comes back a little early, we still go.
        '''
        return (self.trigger_time - time()) < self.ALLOWANCE

    def __cmp__(self, o):
        return cmp(self.trigger_time, o.trigger_time)

class _PipeWrap(object):
    def __init__(self, p):
        self.p = p

    def fileno(self):
        return self.p

class AbstractEventHub(object):
    def __init__(self):
        self.timers = deque()
        self.new_timers = []
        self.run = True
        self.events = {}
        self.run_now = deque()
        self.fdmap = {}
        self._setup_threading()

    def _setup_threading(self):
        self._t_recv, self._t_wakeup = os.pipe()
        fcntl.fcntl(self._t_recv, fcntl.F_SETFL, os.O_NONBLOCK)
        fcntl.fcntl(self._t_wakeup, fcntl.F_SETFL, os.O_NONBLOCK)
        self.thread_comp_in = Queue()

        def handle_thread_done():
            try:
                os.read(self._t_recv, 65536)
            except IOError:
                pass
            while True:
                try:
                    c, v = self.thread_comp_in.get(False)
                except Empty:
                    break
                else:
                    c(v)
        self.register(_PipeWrap(self._t_recv), handle_thread_done, None, None)

    def run_in_thread(self, reschedule, f, *args, **kw):
        def wrap():
            try:
                res = f(*args, **kw)
            except Exception, e:
                self.thread_comp_in.put((reschedule, e))
            else:
                self.thread_comp_in.put((reschedule, res))
            self.wake_from_other_thread()
        thread.start_new_thread(wrap, ())

    def wake_from_other_thread(self):
        try:
            os.write(self._t_wakeup, '\0')
        except IOError:
            pass

    def handle_events(self):
        '''Run one pass of event handling.
        '''
        raise NotImplementedError

    def call_later(self, interval, f, *args, **kw):
        '''Schedule a timer on the hub.
        '''
        t = Timer(self, interval, f, *args, **kw)
        self.new_timers.append(t)
        return t

    def schedule(self, c):
        self.run_now.append(c)

    def register(self, fd, read_callback, write_callback, error_callback):
        '''Register a socket fd with the hub, providing callbacks
        for read (data is ready to be recv'd) and write (buffers are
        ready for send()).

        By default, only the read behavior will be polled and the
        read callback used until enable_write is invoked.
        '''
        fn = fd.fileno()
        self.fdmap[fd] = fn
        assert fn not in self.events
        self.events[fn] = (read_callback, write_callback, error_callback)
        self._add_fd(fd)

    def _add_fd(self, fd):
        '''Add this socket to the list of sockets used in the
        poll call.
        '''
        raise NotImplementedError

    def enable_write(self, fd):
        '''Enable write polling and the write callback.
        '''
        raise NotImplementedError

    def disable_write(self, fd):
        '''Disable write polling and the write callback.
        '''
        raise NotImplementedError

    def unregister(self, fd):
        '''Remove this socket from the list of sockets the
        hub is polling on.
        '''
        if fd in self.fdmap:
            fn = self.fdmap.pop(fd)
            del self.events[fn]

            self._remove_fd(fd)

    def _remove_fd(self, fd):
        '''Remove this socket from the list of sockets the
        hub is polling on.
        '''
        raise NotImplementedError

class SelectEventHub(AbstractEventHub):
    '''A select-based hub.
    '''
    def __init__(self):
        self.in_set, self.out_set = set(), set()
        super(SelectEventHub, self).__init__()

    def handle_events(self):
        '''Run one pass of event handling.

        select() is called, with a timeout equal to the next-scheduled
        timer.  When select returns, all fd-related events (if any) are
        handled, and timers are handled as well.
        '''
        while self.run_now and self.run:
            self.run_now.popleft()()

        if self.new_timers:
            for tr in self.new_timers:
                if tr.pending:
                    tr.inq = True
                    self.timers.append(tr)
            self.timers = deque(sorted(self.timers))
            self.new_timers = []

        tm = time()
        timeout = (self.timers[0].trigger_time - tm) if self.timers else 1e6
        # epoll, etc, limit to 2^31/1000 or OverflowError
        timeout = min(timeout, 1e6)
        if timeout < 0:
            timeout = 0

        # Run timers first, to try to nail their timings
        while self.timers:
            if self.timers[0].due:
                t = self.timers.popleft()
                if t.pending:
                    t.callback()
                    while self.run_now and self.run:
                        self.run_now.popleft()()
                    if not self.run:
                        return
            else:
                break

        select_lists = select.select(self.in_set, self.out_set, [], timeout)
        for i, fds in enumerate(select_lists):
            for fd in fds:
                self.events[fd.fileno()][i]()
                while self.run_now and self.run:
                    self.run_now.popleft()()

                if not self.run:
                    return

    def _add_fd(self, fd):
        '''Add this socket to the list of sockets used in the
        poll call.
        '''
        self.in_set.add(fd)

    def enable_write(self, fd):
        '''Enable write polling and the write callback.
        '''
        self.out_set.add(fd)

    def disable_write(self, fd):
        '''Disable write polling and the write callback.
        '''
        if fd in self.out_set:
            self.out_set.remove(fd)

    def _remove_fd(self, fd):
        '''Remove this socket from the list of sockets the
        hub is polling on.
        '''
        self.in_set.remove(fd)
        if fd in self.out_set:
            self.out_set.remove(fd)

class EPollEventHub(AbstractEventHub):
    '''A epoll-based hub.
    '''
    def __init__(self):
        self.epoll = select.epoll()
        super(EPollEventHub, self).__init__()

    def handle_events(self):
        '''Run one pass of event handling.

        epoll() is called, with a timeout equal to the next-scheduled
        timer.  When epoll returns, all fd-related events (if any) are
        handled, and timers are handled as well.
        '''
        while self.run_now and self.run:
            self.run_now.popleft()()

        if self.new_timers:
            for tr in self.new_timers:
                if tr.pending:
                    tr.inq = True
                    self.timers.append(tr)
            self.timers = deque(sorted(self.timers))
            self.new_timers = []

        tm = time()
        timeout = (self.timers[0].trigger_time - tm) if self.timers else 1e6
        # epoll, etc, limit to 2^^31/1000 or OverflowError
        timeout = min(timeout, 1e6)
        if timeout < 0:
            timeout = 0

        # Run timers first, to try to nail their timings
        while self.timers:
            if self.timers[0].due:
                t = self.timers.popleft()
                if t.pending:
                    t.callback()
                    while self.run_now and self.run:
                        self.run_now.popleft()()
                    if not self.run:
                        return
            else:
                break

        # Handle all socket I/O
        for (fd, evtype) in self.epoll.poll(timeout):
            if evtype & select.EPOLLIN or evtype & select.EPOLLPRI:
                self.events[fd][0]()
            elif evtype & select.EPOLLERR or evtype & select.EPOLLHUP:
                self.events[fd][2]()
            # fd could be removed by above read
            if evtype & select.EPOLLOUT and fd in self.events:
                self.events[fd][1]()

            while self.run_now and self.run:
                self.run_now.popleft()()

            if not self.run:
                return

    def _add_fd(self, fd):
        '''Add this socket to the list of sockets used in the
        poll call.
        '''
        self.epoll.register(fd, select.EPOLLIN | select.EPOLLPRI)

    def enable_write(self, fd):
        '''Enable write polling and the write callback.
        '''
        self.epoll.modify(
                fd, select.EPOLLIN | select.EPOLLPRI | select.EPOLLOUT)

    def disable_write(self, fd):
        '''Disable write polling and the write callback.
        '''
        self.epoll.modify(fd, select.EPOLLIN | select.EPOLLPRI)

    def _remove_fd(self, fd):
        '''Remove this socket from the list of sockets the
        hub is polling on.
        '''
        self.epoll.unregister(fd)

# Expose a usable EventHub implementation
if os.environ.get('DIESEL_NO_EPOLL') or not hasattr(select, 'epoll'):
    EventHub = SelectEventHub
else:
    EventHub = EPollEventHub

if __name__ == '__main__':
    hub = EventHub()
    def whatever(message, other=None):
        print 'got', message, other
    hub.call_later(3.0, whatever, 'yes!', other='rock!')
    import socket, sys
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.bind(('', 11911))
    s.listen(5)
    hub.register(s, lambda: sys.stdout.write('new socket!'), lambda: sys.stdout.write('arg!'))
    while True:
        hub.handle_events()
<|endoftext|>


<commit_before>import tornado.web
from tornado.ioloop import IOLoop
from terminado import TermSocket, SingleTermManager

if __name__ == '__main__':
    term_manager = SingleTermManager(shell_command=['bash'])
    handlers = [
        (r"/ws", TermSocket, {'term_manager': term_manager}),
    ]
    app = tornado.web.Application(handlers)
    app.listen(80)
    try:
        IOLoop.current().start()
    except KeyboardInterrupt:
        pass
    finally:
        term_manager.shutdown()
        print("\n")
<commit_msg>Changed port.<commit_after>import tornado.web
from tornado.ioloop import IOLoop
from terminado import TermSocket, SingleTermManager

if __name__ == '__main__':
    term_manager = SingleTermManager(shell_command=['bash'])
    handlers = [
        (r"/ws", TermSocket, {'term_manager': term_manager}),
    ]
    app = tornado.web.Application(handlers)
    app.listen(8000)
    try:
        IOLoop.current().start()
    except KeyboardInterrupt:
        pass
    finally:
        term_manager.shutdown()
        print("\n")
<|endoftext|>