<commit_before>#!/bin/env python
#
# module to wrap bosco_cluster and related functionality
# .bosco/clusterlist
# .bosco/.pass
# entry=griddev03.racf.bnl.gov max_queued=-1 cluster_type=slurm
#

import logging
import os
import subprocess
import shutil
import threading
import time

# Added to support running module as script from arbitrary location.
from os.path import dirname, realpath, sep, pardir
fullpathlist = realpath(__file__).split(sep)
prepath = sep.join(fullpathlist[:-2])
import sys
sys.path.insert(0, prepath)

from autopyfactory.interfaces import Singleton

# module level threadlock
boscolock = threading.Lock()

class BoscoCluster(object):

    def __init__(self, entry, cluster_type='pbs', port=22, max_queued=-1,  ):
        self.log = logging.getLogger('boscocluster')
        self.entry = entry
        (self.user, self.host) = entry.split('@')
        self.port = port
        self.max_queued = max_queued
        self.cluster_type = cluster_type
        self.log.debug("Init complete. ")

    def getentry(self):
        s += "entry=%s max_queueud=%d cluster_type=%s " % (self.entry, self.max_queued, self.cluster_type)
        return s

    def __str__(self):
        return self.getentry()

class BoscoCLI(object):
    '''
    Encapsulates all bosco_cluster command functionality.

    '''
    __metaclass__ = Singleton

    def __init__(self):
        self.log = logging.getLogger("bosco")
        self.log.debug("Initializing bosco module...")
        self.boscopubkeyfile = os.path.expanduser("~/.ssh/bosco_key.rsa.pub")
        self.boscoprivkeyfile = os.path.expanduser("~/.ssh/bosco_key.rsa")
        self.boscopassfile = os.path.expanduser("~/.bosco/.pass")
        self.boscokeydir = os.path.expanduser("~/.ssh")
        self.boscodir = os.path.expanduser('~/.bosco')
        if os.path.exists(self.boscokeydir) and os.path.isdir(self.boscokeydir):
            self.log.debug("boscokeydir exists.")
        else:
            self.log.debug("Making boscokeydir.")
            os.makedirs(self.boscokeydir)
        if os.path.exists(self.boscodir) and os.path.isdir(self.boscodir):
            self.log.debug('bosco dir exists')
        else:
            self.log.debug("Making boscodir.")
            os.makedirs(self.boscodir)

    def _checkbosco(self):
        '''
        Confirm BOSCO is installed.
        '''
        self.log.debug("Checking to see if local bosco_cluster is on path...")
        isinstalled = False
        exetouse = None
        for path in os.environ['PATH'].split(os.pathsep):
            path = path.strip('"')
            exefile = os.path.join(path,'bosco_cluster')
            if os.path.isfile(exefile) and os.access(exefile, os.X_OK):
                isinstalled = True
                exetouse = exefile
        if not isinstalled:
            self.log.error('Missing dep. bosco_cluster not on PATH: %s' % os.environ['PATH'])
            #raise MissingDependencyFailure('Missing dep. bosco_cluster not on PATH: %s' % os.environ['PATH'] )
        else:
            self.log.debug("Using bosco_cluster: %s" % exetouse )
        return isinstalled

    def _getBoscoClusters(self):
        '''
        [jhover@grid05 ~]$ bosco_cluster -l
        griddev03.racf.bnl.gov/slurm

        '''
        cmd = 'bosco_cluster -l'
        self.log.trace("cmd is %s" % cmd)
        before = time.time()
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
        out = None
        (out, err) = p.communicate()
        delta = time.time() - before
        self.log.trace('It took %s seconds to issue the command' %delta)
        self.log.trace('%s seconds to issue command' %delta)
        if p.returncode == 0 or p.returncode == 2:
            self.log.trace('Leaving with OK return code.')
        else:
            self.log.warning('Leaving with bad return code. rc=%s err=%s' %(p.returncode, err ))
        self.clusters = []
        lines = out.split("\n")
        self.log.trace("got %d lines" % len(lines))
        for line in lines:
            self.log.trace("line is %s" % line)
            if line.strip() == 'No clusters configured':
                self.log.debug("No clusters configured.")
            elif len(line) < 2:
                self.log.trace('empty line discarded')
            else:
                host, batch = line.split('/')
                #  entry, cluster_type='pbs', port=22, max_queued=-1,
                self.log.trace('got entry from bosco: %s %s Making object... ' % (host, batch))
                bentry = BoscoCluster(entry=host, cluster_type=batch)
                self.log.trace('made bentry object, appending.')
                self.clusters.append(bentry)

        return self.clusters

    def _clusteradd(self, user,  host, port, batch, pubkeyfile, privkeyfile, passfile=None):
        self.log.info("Setting up cluster %s@%s/%s " % (user, host, batch))

        self.log.trace("ensuring pubkeyfile")
        shutil.copy(pubkeyfile, self.boscopubkeyfile)
        self.log.trace("ensuring privkeyfile")
        shutil.copy(privkeyfile, self.boscoprivkeyfile)
        if passfile:
            self.log.trace("ensuring passfile")
            shutil.copy(passfile, self.boscopassfile )

        self._start_agent(pubkeyfile, privkeyfile, passfile)

        cmd = 'bosco_cluster -a %s@%s %s ' % (user, host, batch)
        self.log.trace("cmd is %s" % cmd)
        before = time.time()
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
        out = None
        (out, err) = p.communicate()
        self.log.debug('bosco_cluster -a output was %s' % out)
        delta = time.time() - before
        self.log.trace('It took %s seconds to issue the command' %delta)
        self.log.trace('%s seconds to issue command' %delta)

        # remove bosco files to ensure account separation...
        for fn in [self.boscopubkeyfile, self.boscoprivkeyfile, self.boscopassfile ]:
            try:
                self.log.trace("removing %s" % fn)
                os.remove(fn)
            except OSError:
                # file might not exist
                pass
            except TypeError:
                # passfile might be None
                pass

    def _start_agent(self, pubkeyfile, privkeyfile, passfile=None):
        self.log.trace('cmd is ssh-agent')
        cmd = 'ssh-agent'
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
        out = None
        (out,err) = p.communicate()
        lines = out.split('\n')
        for line in lines:
            self.log.trace('line is %s' % line)
            exp = line.split(';')
            if '=' in line:
                (k,v) = exp[0].split('=')
                if k == 'SSH_AUTH_SOCK':
                    os.environ['SSH_AUTH_SOCK'] = v
                    self.log.trace('setting SSH_AUTH_SOCK= %s' % v )
                elif k == 'SSH_AGENT_PID':
                    os.environ['SSH_AGENT_PID'] = v
                    self.log.trace('setting SSH_AGENT_PID= %s' % v )
                else:
                    pass
        self.log.debug('SSH agent started, environment set....')

        #cmd = '/usr/bin/bosco_ssh_start --key %s --pass %s ' % (privkeyfile, passfile)
        cmd = '/usr/bin/bosco_ssh_start --key %s --nopass' % privkeyfile
        self.log.trace('cmd is %s' % cmd)
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
        out = None
        (out,err) = p.communicate()
        self.log.debug('ssh_start completed.')

        cmd = 'ssh-add -l '
        self.log.trace('cmd is %s' % cmd)
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
        out = None
        (out,err) = p.communicate()
        self.log.debug('ssh-add -l : %s' % out)

    def _clusteraddnative(self,host,batch):
        self.log.info("Setting up cluster %s/%s " % (host,batch))

    def _checktarget(self, user, host, port, batch, pubkeyfile, privkeyfile, passfile=None ):
        '''
        Ensure bosco_cluster has been run.
        '''
        #Ensure paths
        pubkeyfile = os.path.expanduser(pubkeyfile)
        privkeyfile = os.path.expanduser(privkeyfile)
        try:
            passfile = os.path.expanduser(passfile)
        except AttributeError:
            pass

        self.log.debug("Checking to see if remote bosco is installed and up to date...")
        host = host.lower()
        batch = batch.lower()
        try:
            self.log.trace("getting lock")
            boscolock.acquire()
            clist = self._getBoscoClusters()
            self.log.trace("got list of %d clusters" % len(clist))
            found = False
            for c in clist:
                if c.user == user and c.host == host and c.cluster_type == batch:
                    found = True
            if not found:
                self.log.info("Setting up cluster %s/%s " % (host,batch))
                self._clusteradd(user, host, port, batch, pubkeyfile, privkeyfile, passfile)
            else:
                self.log.trace("Cluster %s@%s/%s already set up." % (user,host,batch))

        except Exception, e:
            self.log.exception("Exception during bosco remote installation. ")

        finally:
            self.log.trace("releasing lock")
            boscolock.release()

if __name__ == '__main__':
    # Set up logging.
    debug = 0
    info = 0
    trace = 1

    # Add TRACE level
    logging.TRACE = 5
    logging.addLevelName(logging.TRACE, 'TRACE')

    def trace(self, msg, *args, **kwargs):
        self.log(logging.TRACE, msg, *args, **kwargs)

    logging.Logger.trace = trace
    # Check python version
    major, minor, release, st, num = sys.version_info

    # Set up logging, handle differences between Python versions...
    # In Python 2.3, logging.basicConfig takes no args
    #
    FORMAT23="[ %(levelname)s ] %(asctime)s %(filename)s (Line %(lineno)d): %(message)s"
    FORMAT24=FORMAT23
    FORMAT25="[%(levelname)s] %(asctime)s %(module)s.%(funcName)s(): %(message)s"
    FORMAT26=FORMAT25

    if major == 2:
        if minor ==3:
            formatstr = FORMAT23
        elif minor == 4:
            formatstr = FORMAT24
        elif minor == 5:
            formatstr = FORMAT25
        elif minor == 6:
            formatstr = FORMAT26
        elif minor == 7:
            formatstr = FORMAT26

    log = logging.getLogger()
    hdlr = logging.StreamHandler(sys.stdout)
    formatter = logging.Formatter(FORMAT23)
    hdlr.setFormatter(formatter)
    log.addHandler(hdlr)

    if debug:
        log.setLevel(logging.DEBUG) # Override with command line switches
    if info:
        log.setLevel(logging.INFO) # Override with command line switches
    if trace:
        log.setLevel(logging.TRACE)
    log.debug("Logging initialized.")

    bcli = BoscoCLI()
    bcli._checktarget('gridev03.racf.bnl.gov', 'slurm', '~/etc/apf/jhover/pass')

    <commit_msg>making BoscoCLI a new singleton style<commit_after>#!/bin/env python
#
# module to wrap bosco_cluster and related functionality
# .bosco/clusterlist
# .bosco/.pass
# entry=griddev03.racf.bnl.gov max_queued=-1 cluster_type=slurm
#

import logging
import os
import subprocess
import shutil
import threading
import time

# Added to support running module as script from arbitrary location.
from os.path import dirname, realpath, sep, pardir
fullpathlist = realpath(__file__).split(sep)
prepath = sep.join(fullpathlist[:-2])
import sys
sys.path.insert(0, prepath)

# module level threadlock
boscolock = threading.Lock()

class BoscoCluster(object):

    def __init__(self, entry, cluster_type='pbs', port=22, max_queued=-1,  ):
        self.log = logging.getLogger('boscocluster')
        self.entry = entry
        (self.user, self.host) = entry.split('@')
        self.port = port
        self.max_queued = max_queued
        self.cluster_type = cluster_type
        self.log.debug("Init complete. ")

    def getentry(self):
        s += "entry=%s max_queueud=%d cluster_type=%s " % (self.entry, self.max_queued, self.cluster_type)
        return s

    def __str__(self):
        return self.getentry()

class _boscocli(object):
    '''
    Encapsulates all bosco_cluster command functionality.

    '''

    def __init__(self):
        self.log = logging.getLogger("bosco")
        self.log.debug("Initializing bosco module...")
        self.boscopubkeyfile = os.path.expanduser("~/.ssh/bosco_key.rsa.pub")
        self.boscoprivkeyfile = os.path.expanduser("~/.ssh/bosco_key.rsa")
        self.boscopassfile = os.path.expanduser("~/.bosco/.pass")
        self.boscokeydir = os.path.expanduser("~/.ssh")
        self.boscodir = os.path.expanduser('~/.bosco')
        if os.path.exists(self.boscokeydir) and os.path.isdir(self.boscokeydir):
            self.log.debug("boscokeydir exists.")
        else:
            self.log.debug("Making boscokeydir.")
            os.makedirs(self.boscokeydir)
        if os.path.exists(self.boscodir) and os.path.isdir(self.boscodir):
            self.log.debug('bosco dir exists')
        else:
            self.log.debug("Making boscodir.")
            os.makedirs(self.boscodir)

    def _checkbosco(self):
        '''
        Confirm BOSCO is installed.
        '''
        self.log.debug("Checking to see if local bosco_cluster is on path...")
        isinstalled = False
        exetouse = None
        for path in os.environ['PATH'].split(os.pathsep):
            path = path.strip('"')
            exefile = os.path.join(path,'bosco_cluster')
            if os.path.isfile(exefile) and os.access(exefile, os.X_OK):
                isinstalled = True
                exetouse = exefile
        if not isinstalled:
            self.log.error('Missing dep. bosco_cluster not on PATH: %s' % os.environ['PATH'])
            #raise MissingDependencyFailure('Missing dep. bosco_cluster not on PATH: %s' % os.environ['PATH'] )
        else:
            self.log.debug("Using bosco_cluster: %s" % exetouse )
        return isinstalled

    def _getBoscoClusters(self):
        '''
        [jhover@grid05 ~]$ bosco_cluster -l
        griddev03.racf.bnl.gov/slurm

        '''
        cmd = 'bosco_cluster -l'
        self.log.trace("cmd is %s" % cmd)
        before = time.time()
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
        out = None
        (out, err) = p.communicate()
        delta = time.time() - before
        self.log.trace('It took %s seconds to issue the command' %delta)
        self.log.trace('%s seconds to issue command' %delta)
        if p.returncode == 0 or p.returncode == 2:
            self.log.trace('Leaving with OK return code.')
        else:
            self.log.warning('Leaving with bad return code. rc=%s err=%s' %(p.returncode, err ))
        self.clusters = []
        lines = out.split("\n")
        self.log.trace("got %d lines" % len(lines))
        for line in lines:
            self.log.trace("line is %s" % line)
            if line.strip() == 'No clusters configured':
                self.log.debug("No clusters configured.")
            elif len(line) < 2:
                self.log.trace('empty line discarded')
            else:
                host, batch = line.split('/')
                #  entry, cluster_type='pbs', port=22, max_queued=-1,
                self.log.trace('got entry from bosco: %s %s Making object... ' % (host, batch))
                bentry = BoscoCluster(entry=host, cluster_type=batch)
                self.log.trace('made bentry object, appending.')
                self.clusters.append(bentry)

        return self.clusters

    def _clusteradd(self, user,  host, port, batch, pubkeyfile, privkeyfile, passfile=None):
        self.log.info("Setting up cluster %s@%s/%s " % (user, host, batch))

        self.log.trace("ensuring pubkeyfile")
        shutil.copy(pubkeyfile, self.boscopubkeyfile)
        self.log.trace("ensuring privkeyfile")
        shutil.copy(privkeyfile, self.boscoprivkeyfile)
        if passfile:
            self.log.trace("ensuring passfile")
            shutil.copy(passfile, self.boscopassfile )

        self._start_agent(pubkeyfile, privkeyfile, passfile)

        cmd = 'bosco_cluster -a %s@%s %s ' % (user, host, batch)
        self.log.trace("cmd is %s" % cmd)
        before = time.time()
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
        out = None
        (out, err) = p.communicate()
        self.log.debug('bosco_cluster -a output was %s' % out)
        delta = time.time() - before
        self.log.trace('It took %s seconds to issue the command' %delta)
        self.log.trace('%s seconds to issue command' %delta)

        # remove bosco files to ensure account separation...
        for fn in [self.boscopubkeyfile, self.boscoprivkeyfile, self.boscopassfile ]:
            try:
                self.log.trace("removing %s" % fn)
                os.remove(fn)
            except OSError:
                # file might not exist
                pass
            except TypeError:
                # passfile might be None
                pass

    def _start_agent(self, pubkeyfile, privkeyfile, passfile=None):
        self.log.trace('cmd is ssh-agent')
        cmd = 'ssh-agent'
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
        out = None
        (out,err) = p.communicate()
        lines = out.split('\n')
        for line in lines:
            self.log.trace('line is %s' % line)
            exp = line.split(';')
            if '=' in line:
                (k,v) = exp[0].split('=')
                if k == 'SSH_AUTH_SOCK':
                    os.environ['SSH_AUTH_SOCK'] = v
                    self.log.trace('setting SSH_AUTH_SOCK= %s' % v )
                elif k == 'SSH_AGENT_PID':
                    os.environ['SSH_AGENT_PID'] = v
                    self.log.trace('setting SSH_AGENT_PID= %s' % v )
                else:
                    pass
        self.log.debug('SSH agent started, environment set....')

        #cmd = '/usr/bin/bosco_ssh_start --key %s --pass %s ' % (privkeyfile, passfile)
        cmd = '/usr/bin/bosco_ssh_start --key %s --nopass' % privkeyfile
        self.log.trace('cmd is %s' % cmd)
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
        out = None
        (out,err) = p.communicate()
        self.log.debug('ssh_start completed.')

        cmd = 'ssh-add -l '
        self.log.trace('cmd is %s' % cmd)
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
        out = None
        (out,err) = p.communicate()
        self.log.debug('ssh-add -l : %s' % out)

    def _clusteraddnative(self,host,batch):
        self.log.info("Setting up cluster %s/%s " % (host,batch))

    def _checktarget(self, user, host, port, batch, pubkeyfile, privkeyfile, passfile=None ):
        '''
        Ensure bosco_cluster has been run.
        '''
        #Ensure paths
        pubkeyfile = os.path.expanduser(pubkeyfile)
        privkeyfile = os.path.expanduser(privkeyfile)
        try:
            passfile = os.path.expanduser(passfile)
        except AttributeError:
            pass

        self.log.debug("Checking to see if remote bosco is installed and up to date...")
        host = host.lower()
        batch = batch.lower()
        try:
            self.log.trace("getting lock")
            boscolock.acquire()
            clist = self._getBoscoClusters()
            self.log.trace("got list of %d clusters" % len(clist))
            found = False
            for c in clist:
                if c.user == user and c.host == host and c.cluster_type == batch:
                    found = True
            if not found:
                self.log.info("Setting up cluster %s/%s " % (host,batch))
                self._clusteradd(user, host, port, batch, pubkeyfile, privkeyfile, passfile)
            else:
                self.log.trace("Cluster %s@%s/%s already set up." % (user,host,batch))

        except Exception, e:
            self.log.exception("Exception during bosco remote installation. ")

        finally:
            self.log.trace("releasing lock")
            boscolock.release()

# Singleton implementation
class BoscoCLI(object):

    instance = None

    def __new__(cls, *k, **kw):
        if not BoscoCLI.instance:
            BoscoCLI.instance = _boscocli(*k, **kw)
        return BoscoCLI.instance

# =============================================================================

if __name__ == '__main__':
    # Set up logging.
    debug = 0
    info = 0
    trace = 1

    # Add TRACE level
    logging.TRACE = 5
    logging.addLevelName(logging.TRACE, 'TRACE')

    def trace(self, msg, *args, **kwargs):
        self.log(logging.TRACE, msg, *args, **kwargs)

    logging.Logger.trace = trace
    # Check python version
    major, minor, release, st, num = sys.version_info

    # Set up logging, handle differences between Python versions...
    # In Python 2.3, logging.basicConfig takes no args
    #
    FORMAT23="[ %(levelname)s ] %(asctime)s %(filename)s (Line %(lineno)d): %(message)s"
    FORMAT24=FORMAT23
    FORMAT25="[%(levelname)s] %(asctime)s %(module)s.%(funcName)s(): %(message)s"
    FORMAT26=FORMAT25

    if major == 2:
        if minor ==3:
            formatstr = FORMAT23
        elif minor == 4:
            formatstr = FORMAT24
        elif minor == 5:
            formatstr = FORMAT25
        elif minor == 6:
            formatstr = FORMAT26
        elif minor == 7:
            formatstr = FORMAT26

    log = logging.getLogger()
    hdlr = logging.StreamHandler(sys.stdout)
    formatter = logging.Formatter(FORMAT23)
    hdlr.setFormatter(formatter)
    log.addHandler(hdlr)

    if debug:
        log.setLevel(logging.DEBUG) # Override with command line switches
    if info:
        log.setLevel(logging.INFO) # Override with command line switches
    if trace:
        log.setLevel(logging.TRACE)
    log.debug("Logging initialized.")

    bcli = BoscoCLI()
    bcli._checktarget('gridev03.racf.bnl.gov', 'slurm', '~/etc/apf/jhover/pass')

<|endoftext|>


<commit_before>#!/usr/bin/env python

"""API blueprint.

Inspired by Flask-restless.

"""

from flask import abort, Blueprint, jsonify, request
from os.path import abspath, dirname, join
from time import time
from werkzeug.exceptions import HTTPException

class APIError(HTTPException):

  """Thrown when an API call is invalid.

  The error code will sent as error code for the response.

  """

  def __init__(self, code, message):
    self.code = code
    super(APIError, self).__init__(message)

  def __repr__(self):
    return '<APIError %r: %r>' % (self.code, self.message)

class APIManager(object):

  config = {
    'URL_PREFIX': '/api',
  }

  def __init__(self, **kwargs):
    for k, v in kwargs.items():
      self.config[k.upper()] = v
    self.Models = {}

  def add_model(self, Model, default_collection_depth=0, default_model_depth=1,
                default_limit=20, max_limit=None, **kwargs):
    self.Models[Model.__name__] = {
      'class': Model,
      'collection_view_options': {
        'default_depth': default_collection_depth,
        'default_limit': default_limit,
        'max_limit': max_limit
      },
      'model_view_options': {
        'default_depth': default_model_depth,
      },
      'route_options': kwargs
    }

  def _create_blueprint(self, project):
    return Blueprint(
      'api',
      project.config['PROJECT']['APP_FOLDER'] + '.api',
      template_folder=abspath(join(dirname(__file__), 'templates', 'api')),
      url_prefix=self.config['URL_PREFIX']
    )

  def _create_model_views(self, data):
    Model = data['class']
    views = [
      ModelView(Model=Model, **data['model_view_options']),
      CollectionView(Model=Model, **data['collection_view_options'])
    ]
    for rel in Model.get_relationships().values():
      if rel.lazy == 'dynamic':
        views.extend([
          ModelView(
            Model=Model, relationship=rel, **data['model_view_options']
          ),
          CollectionView(
            Model=Model, relationship=rel, **data['collection_view_options']
          )
        ])
    for view in views:
      self.blueprint.add_url_rule(
        view.url, view.endpoint, view, **data['route_options']
      )

  def _before_register(self, project):
    self.blueprint = self._create_blueprint(project)
    for data in self.Models.values():
      self._create_model_views(data)
    index_view = IndexView()
    self.blueprint.add_url_rule(
      index_view.url, index_view.endpoint, index_view
    )

  def _after_register(self, project):
    pass

class APIView(object):

  """Base API view."""

  __all__ = []

  Model = None
  relationship = None

  def __init__(self, **kwargs):
    for k, v in kwargs.items():
      setattr(self, k, v)
    self.__all__.append(self)

class IndexView(APIView):

  def __call__(self):
    return jsonify({
      'status': 'Welcome',
      'available_endpoints': [v.url for v in self.__all__]
    })

  @property
  def endpoint(self):
    return 'index'

  @property
  def url(self):
    return '/'

class CollectionView(APIView):

  params = set(['depth', 'limit', 'offset', 'loaded', 'sort'])
  default_depth = 0
  default_limit = 20
  max_limit = None

  def __call__(self, **kwargs):
    if id is None:
      query = self.Model.query
    else:
      model = self.Model.query.get(kwargs.values())
      if not model:
        return abort(404), 404
      query = getattr(model, self.relationship.key)
    try:
      results = self.process_query(query)
    except APIError as e:
      return jsonify({
        'status': 'Error',
        'request': {
          'base_url': request.base_url,
          'method': request.method,
          'values': request.values
        },
        'content': e.message
      }), e.code
    else:
      return jsonify({
        'status': 'Success',
        'processing_time': results['processing_times'],
        'matches': {
          'total': results['total_matches'],
          'returned': len(results['content'])
        },
        'request': {
          'base_url': request.base_url,
          'method': request.method,
          'values': request.values
        },
        'content': results['content']
      }), 200

  def process_query(self, query):
    Model = query.column_descriptions[0]['type']
    column_names = [c.key for c in Model.get_columns()]
    timer = time()
    processing_times = []
    filters = {}
    for k, v in request.args.items():
      if k in column_names:
        filters[k] = v
      elif not k in self.params:
        raise APIError(400, 'Bad Request')
    try:
      offset = max(0, int(request.args.get('offset', 0)))
      limit = max(0, int(request.args.get('limit', self.default_limit)))
      if self.max_limit:
        limit = min(limit, self.max_limit)
      depth = max(0, int(request.args.get('depth', self.default_depth)))
      loaded = [int(e) for e in request.args.get('loaded', '').split(',') if e]
      sort = request.args.get('sort', '')
    except ValueError as e:
      raise APIError(400, 'Bad Request')
    processing_times.append(('request', time() - timer))
    timer = time()
    for k, v in filters.items():
      query = query.filter(getattr(Model, k) == v)
    total_matches = 10 # query.count()
    processing_times.append(('query', time() - timer))
    timer = time()
    if loaded:
      query = query.filter(~Model.id.in_(loaded))
    if sort:
      attr = sort.strip('-')
      if not attr in column_names:
        raise APIError(400, 'Bad Request')
      if sort[0] == '-':
        query = query.order_by(-getattr(Model, attr))
      else:
        query = query.order_by(getattr(Model, attr))
    if limit:
      query = query.limit(limit)
    return {
      'processing_times': processing_times,
      'total_matches': total_matches,
      'content': [
        e.jsonify(depth=depth)
        for e in query.offset(offset)
      ]
    }

  @property
  def endpoint(self):
    if not self.relationship:
      return '%s_collection_view' % self.Model.__tablename__
    else:
      return '%s_%s_collection_view' % (
        self.Model.__tablename__, self.relationship.key
      )

  @property
  def url(self):
    if not self.relationship:
      return '/%s' % self.Model.__tablename__
    else:
      return '/%s/<id>/%s' % (
        self.Model.__tablename__, self.relationship.key
      )

class ModelView(APIView):

  params = set(['depth'])
  default_depth = 1

  @property
  def endpoint(self):
    if not self.relationship:
      return '%s_model_view' % self.Model.__tablename__
    else:
      return '%s_%s_model_view' % (
        self.Model.__tablename__, self.relationship.key
      )

  @property
  def url(self):
    url = '/%s' % self.Model.__tablename__
    url += ''.join('/<%s>' % n for n in self.Model.get_primary_key_names())
    if self.relationship:
      url += '/%s/<position>' % self.relationship.key
    return url

  def __call__(self, **kwargs):
    position = kwargs.pop('position', None)
    try:
      depth = int(request.args.get('depth', self.default_depth))
      model = self.Model.query.get(kwargs.values())
      if position is not None:
        try:
          pos = int(position) - 1
        except ValueError:
          raise APIError(400, 'Bad Request')
        else:
          if pos >= 0:
            model = getattr(model, self.relationship.key).offset(pos).first()
          else:
            raise APIError(400, 'Bad Request')
      if not model:
        raise APIError(404, 'Not Found')
    except APIError as e:
      return jsonify({
        'status': 'Error',
        'request': {
          'base_url': request.base_url,
          'method': request.method,
          'values': request.values
        },
        'content': e.message
      }), e.code
    else:
      return jsonify(model.jsonify(depth=depth))

<commit_msg>API now supports GET, PUT, POST, DELETE.<commit_after>#!/usr/bin/env python

"""API blueprint.

Inspired by Flask-restless.

"""

from flask import abort, Blueprint, jsonify, request
from os.path import abspath, dirname, join
from sqlalchemy.orm.collections import InstrumentedList
from time import time
from werkzeug.exceptions import HTTPException

from ..project import current_project

db = current_project.db

class APIError(HTTPException):

  """Thrown when an API call is invalid.

  The error code will sent as error code for the response.

  """

  def __init__(self, code, message):
    self.code = code
    super(APIError, self).__init__(message)

  def __repr__(self):
    return '<APIError %r: %r>' % (self.code, self.message)

class APIManager(object):

  _authorize = None
  _validate = None

  config = {
    'URL_PREFIX': '/api',
    'DEFAULT_COLLECTION_DEPTH': 0,
    'DEFAULT_MODEL_DEPTH': 1,
    'DEFAULT_LIMIT': 20,
    'MAX_LIMIT': None,
  }

  def __init__(self, **kwargs):
    for k, v in kwargs.items():
      self.config[k.upper()] = v
    self.Models = {}

  def add_model(self, Model, relationships=True,
                methods=frozenset(['GET', 'POST', 'PUT', 'DELETE'])):
    """Flag a Model to be added.

    relationships can either be ``True`` or a list of relationship keys. In the
    first case, all one to many relationships will have a hook created,
    otherwise only those mentioned in the list.

    """
    self.Models[Model.__name__] = {
      'Model': Model,
      'methods': methods,
      'relationships': relationships,
    }

  def authorize(self, func):
    """Decorator to set the authorizer function.

    The authorizer is passed three arguments:

    * the model class
    * the relationship (or ``None``)
    * the request method

    The third argument is for convenience (the full request object can be
    accessed as usual).

    If the function returns a truthful value, the request will proceed.
    Otherwise a 403 exception is raised.

    """
    self._authorize = func

  def validate(self, func):
    """Decorator to set the validate function, called on POST and PUT.

    The validator will be passed two arguments:

    * the model class
    * the request json
    * the request method

    If the function returns a truthful value, the request will proceed.
    Otherwise a 400 exception is raised.

    """
    self._validate = func

  def _create_model_views(self, Model, relationships, methods):
    """Creates the views associated with the model.

    Sane defaults are chosen.

    """
    views = []
    methods = set(methods)
    model_methods = methods & set(['GET', 'PUT', 'DELETE'])
    if model_methods:
      views.append(ModelView(Model=Model, methods=model_methods))
    collection_methods = methods & set(['GET', 'POST'])
    if collection_methods:
      views.append(CollectionView(Model=Model, methods=collection_methods))
    rels = relationships or []
    rel_methods = set(['GET']) & methods
    if rel_methods:
      for rel in Model.get_relationships().values():
        if (rels is True or rel.key in rels) and rel.uselist:
          views.extend([
            ModelView(Model=Model, relationship=rel, methods=rel_methods),
            CollectionView(Model=Model, relationship=rel, methods=rel_methods)
          ])
    for view in views:
      self.blueprint.add_url_rule(view.url, view.endpoint, view)

  def _before_register(self, project):
    self.blueprint = Blueprint(
      'api',
      project.config['PROJECT']['APP_FOLDER'] + '.api',
      template_folder=abspath(join(dirname(__file__), 'templates', 'api')),
      url_prefix=self.config['URL_PREFIX']
    )
    for data in self.Models.values():
      self._create_model_views(**data)
    index_view = IndexView()
    self.blueprint.add_url_rule(
      index_view.url, index_view.endpoint, index_view
    )

  def _after_register(self, project):
    """Give access to defaults and authorizer to views."""
    APIView._manager = self

class APIView(object):

  """Base API view.

  Note that the methods class attribute seems to be passed to the add_url_rule
  function somehow (!).

  """

  __all__ = []

  _manager = None

  Model = None
  relationship = None
  methods = set()
  params = set()

  def __init__(self, **kwargs):
    for k, v in kwargs.items():
      setattr(self, k, v)
    self.__all__.append(self)

  def __call__(self, **kwargs):
    """Redirects to corresponding request handler."""
    try:
      if not self.is_authorized():
        raise APIError(403, 'Forbidden')
      if request.method in self.methods:
        params = self._parse_params()
        return getattr(self, request.method.lower())(params, **kwargs)
      else:
        raise APIError(405, 'Method Not Allowed')
    except APIError as e:
      return jsonify({
        'status': 'Error',
        'request': {
          'base_url': request.base_url,
          'method': request.method,
          'values': request.values
        },
        'content': e.message
      }), e.code

  @property
  def authorized_methods(self):
    return set(
      method for method in self.methods
      if self._manager._authorize(self.Model, self.relationship, method)
    )

  @property
  def defaults(self):
    return self._manager.config

  def is_validated(self, json):
    """Validate changes."""
    if not self._manager._validate:
      return True
    return self._mananager._validate(self.Model, json, request.method)

  def is_authorized(self):
    if not self._manager._authorize:
      return True
    return self._manager._authorize(
      self.Model, self.relationship, request.method
    )

  def _parse_params(self):
    return request.args

class IndexView(APIView):

  methods = set(['GET'])

  @property
  def endpoint(self):
    return 'index'

  @property
  def url(self):
    return '/'

  def get(self, params, **kwargs):
    return jsonify({
      'status': 'Welcome',
      'available_endpoints': [
        '%s (%s)' % (v.url, ', '.join(v.authorized_methods))
        for v in self.__all__
        if v.authorized_methods
      ]
    })

class CollectionView(APIView):

  params = set(['depth', 'limit', 'offset', 'loaded', 'sort'])

  @property
  def endpoint(self):
    if not self.relationship:
      return '%s_collection_view' % self.Model.__tablename__
    else:
      return '%s_%s_collection_view' % (
        self.Model.__tablename__, self.relationship.key
      )

  @property
  def url(self):
    url = '/%s' % self.Model.__tablename__
    if self.relationship:
      url += ''.join('/<%s>' % n for n in self.Model.get_primary_key_names())
      url += '/%s' % self.relationship.key
    url += '/'
    return url

  def get(self, params, **kwargs):
    if not kwargs:
      query = self.Model.query
    else:
      model = self.Model.query.get(kwargs.values())
      if not model:
        raise APIError(404, 'Not Found')
      query = getattr(model, self.relationship.key)
    if isinstance(query, InstrumentedList):
      results = self._process_list(query, params)
    else:
      results = self._process_query(query, params)
    return jsonify({
      'status': 'Success',
      'processing_time': results['processing_times'],
      'matches': {
        'total': results['total_matches'],
        'returned': len(results['content'])
      },
      'request': {
        'base_url': request.base_url,
        'method': request.method,
        'values': request.values
      },
      'content': results['content']
    }), 200

  def post(self, params, **kwargs):
    if self.is_validated(request.json):
      model = Model(**request.json)
      db.session.add(model)
    else:
      raise APIError(400, 'Bad Request')

  def _process_list(self, lst, params):
    return {
      'processing_times': [],
      'total_matches': len(lst),
      'content': [e.jsonify(self.defaults['DEFAULT_COLLECTION_DEPTH']) for e in lst]
    }

  def _process_query(self, query, params):
    Model = query.column_descriptions[0]['type']
    column_names = [c.key for c in Model.get_columns()]
    timer = time()
    processing_times = []
    filters = {}
    for k, v in request.args.items():
      if k in column_names:
        filters[k] = v
      elif not k in self.params:
        raise APIError(400, 'Bad Request')
    try:
      offset = max(0, int(request.args.get('offset', 0)))
      limit = max(0, int(
        request.args.get('limit', self.defaults['DEFAULT_LIMIT'])
      ))
      if self.defaults['MAX_LIMIT']:
        limit = min(limit, self.defaults['MAX_LIMIT'])
      depth = max(0, int(
        request.args.get('depth', self.defaults['DEFAULT_COLLECTION_DEPTH'])
      ))
      loaded = [int(e) for e in request.args.get('loaded', '').split(',') if e]
      sort = request.args.get('sort', '')
    except ValueError as e:
      raise APIError(400, 'Bad Request')
    processing_times.append(('request', time() - timer))
    timer = time()
    for k, v in filters.items():
      query = query.filter(getattr(Model, k) == v)
    total_matches = 10 # query.count()
    processing_times.append(('query', time() - timer))
    timer = time()
    if loaded:
      query = query.filter(~Model.id.in_(loaded))
    if sort:
      attr = sort.strip('-')
      if not attr in column_names:
        raise APIError(400, 'Bad Request')
      if sort[0] == '-':
        query = query.order_by(-getattr(Model, attr))
      else:
        query = query.order_by(getattr(Model, attr))
    if limit:
      query = query.limit(limit)
    return {
      'processing_times': processing_times,
      'total_matches': total_matches,
      'content': [
        e.jsonify(depth=depth)
        for e in query.offset(offset)
      ]
    }

class ModelView(APIView):

  params = set(['depth'])

  @property
  def endpoint(self):
    if not self.relationship:
      return '%s_model_view' % self.Model.__tablename__
    else:
      return '%s_%s_model_view' % (
        self.Model.__tablename__, self.relationship.key
      )

  @property
  def url(self):
    url = '/%s' % self.Model.__tablename__
    url += ''.join('/<%s>' % n for n in self.Model.get_primary_key_names())
    if self.relationship:
      url += '/%s/<position>' % self.relationship.key
    return url

  def get(self, params, **kwargs):
    position = kwargs.pop('position', None)
    depth = int(request.args.get('depth', self.defaults['DEFAULT_MODEL_DEPTH']))
    model = self.Model.query.get(kwargs.values())
    if position is not None:
      try:
        pos = int(position) - 1
      except ValueError:
        raise APIError(400, 'Bad Request')
      else:
        if pos >= 0:
          query_or_list = getattr(model, self.relationship.key)
          if isinstance(query_or_list, InstrumentedList):
            model = query_or_list[pos]
          else:
            model = query_or_list.offset(pos).first()
        else:
          raise APIError(400, 'Bad Request')
    if not model:
      raise APIError(404, 'Not Found')
    return jsonify(model.jsonify(depth=depth))

  def put(self, params, **kwargs):
    model = self.Model.query.get(kwargs.values())
    if self.is_validated(request.json):
      for k, v in request.json.items():
        setattr(model, k, v)
    else:
      raise APIError(400, 'Bad Request')

  def delete(self, params, **kwargs):
    model = self.Model.query.get(kwargs.values())
    db.session.delete(model)

<|endoftext|>


<commit_before>from __future__ import absolute_import
from flask import Response
from functools import wraps
from attest import Tests

class ComparableResponse(Response):

    def __eq__(self, other):
        self.freeze()
        other.freeze()
        return all(getattr(self, name) == getattr(other, name)
                   for name in ('status_code', 'headers', 'data'))

    def __ne__(self, other):
        return not self == other

class AppTests(Tests):

    def __init__(self, appfactory, *args, **kwargs):
        Tests.__init__(self, *args, **kwargs)

        @self.context
        def request_context():
            app = appfactory()
            app.response_class = ComparableResponse
            client = app.test_client()
            yield client

def get(*args, **kwargs):
    def decorator(func):
        @wraps(func)
        def wrapper(client):
            response = client.get(*args, **kwargs)
            return func(response)
        return wrapper
    return decorator
<commit_msg>Restore app.response_class<commit_after>from __future__ import absolute_import
from flask import Response
from functools import wraps
from attest import Tests

class ComparableResponse(Response):

    def __eq__(self, other):
        self.freeze()
        other.freeze()
        return all(getattr(self, name) == getattr(other, name)
                   for name in ('status_code', 'headers', 'data'))

    def __ne__(self, other):
        return not self == other

class AppTests(Tests):

    def __init__(self, appfactory, *args, **kwargs):
        Tests.__init__(self, *args, **kwargs)

        @self.context
        def request_context():
            app = appfactory()
            app.response_class, orig = ComparableResponse, app.response_class
            client = app.test_client()
            app.response_class = orig
            yield client

def get(*args, **kwargs):
    def decorator(func):
        @wraps(func)
        def wrapper(client):
            response = client.get(*args, **kwargs)
            return func(response)
        return wrapper
    return decorator
<|endoftext|>


<commit_before>from __future__ import absolute_import, unicode_literals
from .smartsql import RelationQSMixIn, smartsql
from ..models import registry

try:
    str = unicode  # Python 2.* compatible
    str_types = ()
    string_types = (basestring,)
    integer_types = (int, long)
except NameError:
    string_types = (str,)
    integer_types = (int,)

class Relation(RelationQSMixIn):

    def __init__(self, model, field=None, qs=None):
        self.model = model
        self.field = field
        self.qs = qs

    def _set_up(self, instance, owner):
        if isinstance(self.model, string_types):
            self.model = registry.get(self.model)

class ForeignKey(Relation):

    def __get__(self, instance, owner):
        super(ForeignKey, self)._set_up(instance, owner)
        if not instance:
            return self.model
        if not self.field:
            self.field = '{0}_id'.format(self.model._meta.db_table.split("_").pop())
        return self.filter(**{self.model._meta.pk: getattr(instance, self.field)})[0]

class OneToMany(Relation):

    def __get__(self, instance, owner):
        super(OneToMany, self)._set_up(instance, owner)
        if not instance:
            return self.model
        if not self.field:
            self.field = '{0}_id'.format(instance._meta.db_table.split("_").pop())
        return self.filter(**{self.field: getattr(instance, instance._meta.pk)})
<commit_msg>Relation refactory<commit_after>from __future__ import absolute_import, unicode_literals
from .smartsql import RelationQSMixIn, smartsql
from ..models import registry

try:
    str = unicode  # Python 2.* compatible
    str_types = ()
    string_types = (basestring,)
    integer_types = (int, long)
except NameError:
    string_types = (str,)
    integer_types = (int,)

class Relation(RelationQSMixIn):

    def __init__(self, model, field=None, qs=None):
        self.model = model
        self.field = field
        self.qs = qs

    def set_up(self, instance, owner):
        if isinstance(self.model, string_types):
            self.model = registry.get(self.model)

    def set_field(self, model):
        if self.field is None and model:
            self.field = '{0}_id'.format(model._meta.db_table.split("_").pop())

class ForeignKey(Relation):

    def set_up(self, instance, owner):
        super(ForeignKey, self).set_up(instance, owner)
        self.set_field(self.model)

    def __get__(self, instance, owner):
        self.set_up(instance, owner)
        if not instance:
            return self.model
        return self.filter(**{self.model._meta.pk: getattr(instance, self.field)})[0]

class OneToMany(Relation):

    def set_up(self, instance, owner):
        super(OneToMany, self).set_up(instance, owner)
        self.set_field(instance)

    def __get__(self, instance, owner):
        self.set_up(instance, owner)
        if not instance:
            return self.model
        return self.filter(**{self.field: getattr(instance, instance._meta.pk)})
<|endoftext|>


<commit_before>"""A simple flow network module.

This module contains the classes and functions written to create, examine,
and solve simple problems on flow networks.

Copyright (C) 2017 Chris Digirolamo

"""
from collections import deque, OrderedDict
from decimal import Decimal
import weakref

INF = Decimal('Infinity')

class FlowNode(object):
    """
    A node with flow network functionality. Automatically
    creates, adjusts, and deletes edges and connects by
    simply using the `add_flow` and `remove_flow` functions.

    Args:
        key (int): A unique identifier for the node.

    Attributes:
        key (int): A unique identifier for the node.

        edge_flow (dict[FlowNode, int]): Maps the flow for this node.
            Stores the flow of children nodes. Basically stores the
            edge.
        parents (set): The parents are the nodes that flow into
            this node.

    """
    def __init__(self, key):
        self._parents = set([])
        self._key = key
        self.edge_flow = OrderedDict()

    @property
    def flow(self):
        """int: The total flow from this FlowNode."""
        return sum(self.edge_flow.itervalues())

    @property
    def key(self):
        """object: The unique node identifier."""
        return self._key

    def add_flow(self, child, amount):
        """
        Adds a a flow from this node to a child node.
        If the edge already exists, adds to it, else
        we create a new edge.

        Args:
            child (FlowNode): The child who receives
                flow from this node.
            amount (int): The amount of flow to add.

        """
        if child not in self.edge_flow:
            self.edge_flow[child] = amount
            child._parents.add(weakref.ref(self))
        else:
            self.edge_flow[child] += amount

    def reduce_flow(self, child, amount=INF):
        """
        Removes flow from an edge. If the flow becomes
        <= 0, the edge is removed.

        Args:
            child (FlowNode): The child node to remove
                flow from.
            amount (int): The amount of flow to remove.
                Defaults to infinity (to guarantees removal
                of the edge)

        """
        if amount < self.edge_flow[child]:
            self.edge_flow[child] -= amount
        else:
            del self.edge_flow[child]
            child._parents.remove(weakref.ref(self))

    def iter_children(self):
        """Iterate through the nodes that this node flows into.

        Yields:
            FlowNode: Each child of this node.

            """
        for child in self.edge_flow.iterkeys():
            yield child

    def iter_parents(self):
        """Iterate through the nodes that flow into this node.

        Yields:
            FlowNode: Each parents of this node.

            """
        #Stored as weakrefs
        for ref in self._parents:
            parent = ref()
            if parent is not None:
                yield parent

    def iter_edges(self):
        """Iterate through the edges of this node.

        Yields:
            tuple: (parent, child, capacity) of the flow.

        """
        for child, capacity in self.edge_flow.iteritems():
            yield (self, child, capacity)

    def iter_dfs_edges(self):
        """ A depth first search following the flow from this node.

        Note:
            The first node yielded is the calling instance itself.

        Yields:
            tuple[FlowNode]: The next (parent, child) nodes in the search.

        """
        node_queue = deque([(self, c) for c in self.edge_flow.iterkeys()])
        visited = set([c for p, c in node_queue])
        visited.add(self)
        while node_queue:
            parent, child = node_queue.pop()
            yield parent, child
            for granchild in child.edge_flow.iterkeys():
                if granchild not in visited:
                    visited.add(granchild)
                    node_queue.append((child, granchild))

    def __repr__(self):
        cls_name = self.__class__.__name__
        text = "{}({})"
        return text.format(cls_name, self.key)

class SuperSink(FlowNode):
    """The only sink node.

    If there are multiple sinks in a graph, one instance of this node
    can handle them.

    Attributes:
        consumed (int): As a sink, flows are consumed, this
            counts the consumed flow.

    """
    def __init__(self, *args, **kwargs):
        super(SuperSink, self).__init__(*args, **kwargs)
        self.consumed = 0

    @property
    def flow(self):
        """int: The total flow from this node."""
        return INF

    def add_flow(self, child, amount):
        """
        Rather than adding flow from this node to a child node,
        the sink consumes flow.

        Args:
            child (FlowNode): The child who receives
                flow from this node.
            amount (int): The amount of flow to add.

        """
        self.consumed += amount

class SuperSource(FlowNode):
    """The only source node.

    If there are multiple sources in a graph, one instance of this node
    can handle them.

    """

class FlowNetwork(object):
    """
    A class for flow network functionality. Flow networks created with this
    class automatically reduce the network size by using one sink and one
    source.

    Attributes:
        node_key_dict (OrderedDict[key, FlowNode]): The dict of all created
            nodes.

    """
    def __init__(self):
        self._source = SuperSource("+")
        self._sink = SuperSink("-")
        start_d = [(self.source.key, self.source), (self.sink.key, self.sink)]
        self.node_key_dict = OrderedDict(start_d)

    @property
    def source(self):
        """SuperSource: The only source node."""
        return self._source

    @property
    def sink(self):
        """SuperSink: The only sink node."""
        return self._sink

    def add_flow_edge(self, parent, child, capacity):
        """
        Adds a a flow from a parent node to a child node.
        If the node and edge already exists, adds to it, else
        we create a new node and/or edge.

        Args:
            parent (str): The key of the parent who is sending the
                flow.
            child (str): The key of the child who receives
                flow from this node.
            capacity (int): The max amount of flow to add.

        """
        if isinstance(parent, FlowNode) or isinstance(child, FlowNode):
            raise TypeError("add_flow_edge args are node keys, not nodes themselves.")
        if parent not in self.node_key_dict:
            self.node_key_dict[parent] = FlowNode(parent)
        parent = self.node_key_dict[parent]
        if child not in self.node_key_dict:
            self.node_key_dict[child] = FlowNode(child)
        child = self.node_key_dict[child]
        parent.add_flow(child, capacity)

    def get_maximum_flow(self):
        """Gets the maximum flow from source to sink.

        Returns:
            int: The maximum flow.

        """
        new_copy = self.from_flow_network(self)
        start_consumed = new_copy.sink.consumed
        new_copy.send_max_flow_to_sink()
        return new_copy.sink.consumed - start_consumed

    def send_max_flow_to_sink(self):
        """
        Sends the maximum amount of flow from the source to the sink.

        Partially implements the Ford-Fulkerson maximum flow algorithm
        to solve maximum flow by sending flow using function in this
        FlowNetwork and the FlowNode class.

        You end up finding the maximum flow if you check
        how much the sink consumed.

        """
        parent_map = self._get_dfs_sink_flow_path()
        while parent_map:
            minimum_path_flow = INF
            node = self.sink
            while node is not self.source:
                parent = parent_map[node]
                flow = parent.edge_flow[node]
                minimum_path_flow = min(minimum_path_flow, flow)
                node = parent
            node = self.sink
            while node is not self.source:
                parent = parent_map[node]
                flow = parent.edge_flow[node]
                parent.reduce_flow(node, minimum_path_flow)
                node.add_flow(parent, minimum_path_flow)
                node = parent
            parent_map = self._get_dfs_sink_flow_path()

    def _get_dfs_sink_flow_path(self):
        """A single path from this node to the sink.

        A breadth first search following the flow from
        this node.

        Returns:
            dict[FlowNode, FlowNode]: Each child to parent in
            the path.

        """
        node_path = {}
        for parent, child in self.source.iter_dfs_edges():
            node_path[child] = parent
            if child is self.sink:
                break
        if self.sink not in node_path:
            return dict()
        child = self.sink
        parent = node_path[child]
        exact_path = {child: parent}
        while parent is not self.source:
            child = parent
            parent = node_path[child]
            exact_path[child] = parent
        return exact_path

    def iter_edge_values(self):
        """Iterator of all edges.

        Yields:
            tuple[]: (parent, child, value) for all edges in the
            flow network.

        """
        visited = set()
        for node in self.node_key_dict.values():
            if node in visited:
                continue
            for parent, child in node.iter_dfs_edges():
                edge = (parent, child)
                if edge in visited:
                    continue
                visited.add(edge)
                yield parent, child, parent.edge_flow[child]

    @classmethod
    def from_adjacency_matrix(cls, adjacency_matrix, sources, sinks):
        """Creates a Flow Network from an adjacency matrix.

        Args:
            adjacency_matrix (list[list[int]): The adjacency matrix representation
                of a flow graph.
            sources (list[int]): list of integers denoting indexes of sources.
            sinks (list[int]): list of integers denoting indexes of sinks.

        """
        flow_network = cls()
        key_to_node = {}
        sources = set(sources)
        sinks = set(sinks)
        size = len(adjacency_matrix)
        for node_idx in xrange(size):
            if node_idx in sinks:
                node = flow_network.sink
            elif node_idx in sources:
                node = flow_network.source
            else:
                node = FlowNode(node_idx)
            key_to_node[node_idx] = node
        # Add Flow
        for node_idx in xrange(size):
            row = adjacency_matrix[node_idx]
            node = key_to_node[node_idx]
            for next_node_idx in xrange(size):
                next_node = key_to_node[next_node_idx]
                capacity = row[next_node_idx]
                flow_network.add_flow_edge(node.key, next_node.key, capacity)
        return flow_network

    @classmethod
    def from_flow_network(cls, flow_network):
        """Creates a copy of a flow network.

        Args:
            flow_network (FlowNetwork): The instance to copy.

        """
        new_flow_network = cls()
        for parent, child, capacity in flow_network.iter_edge_values():
            new_flow_network.add_flow_edge(parent.key, child.key, capacity)
        return new_flow_network

    def __str__(self):
        max_num = max(e[2] for e in self.iter_edge_values())
        largest_str = len(str(max_num))
        f_temp = " {:" + str(largest_str) + "}"
        all_strings = []
        for node in self.node_key_dict.itervalues():
            node_strings = []
            for node_2 in self.node_key_dict.itervalues():
                if node_2 not in node.edge_flow:
                    txt = f_temp.format(0)
                    if node is self.sink:
                        txt = f_temp.format(node.key)
                else:
                    txt = f_temp.format(node.edge_flow[node_2])
                node_strings.append(txt)
            node_string = ", ".join(node_strings)
            node_string = "[{}]".format(node_string)
            if node is self.source:
                node_string += " Source"
            elif node is self.sink:
                node_string += " Sink ({})".format(self.sink.consumed)
            else:
                node_string += " {}".format(node.key)
            all_strings.append(node_string)
        entire_string = ",\n ".join(all_strings)
        entire_string = "[{}]".format(entire_string)
        return entire_string
<commit_msg>__str__ method is even more accurate now.<commit_after>"""A simple flow network module.

This module contains the classes and functions written to create, examine,
and solve simple problems on flow networks.

Copyright (C) 2017 Chris Digirolamo

"""
from collections import deque, OrderedDict
from decimal import Decimal
import weakref

INF = Decimal('Infinity')

class FlowNode(object):
    """
    A node with flow network functionality. Automatically
    creates, adjusts, and deletes edges and connects by
    simply using the `add_flow` and `remove_flow` functions.

    Args:
        key (int): A unique identifier for the node.

    Attributes:
        key (int): A unique identifier for the node.

        edge_flow (dict[FlowNode, int]): Maps the flow for this node.
            Stores the flow of children nodes. Basically stores the
            edge.
        parents (set): The parents are the nodes that flow into
            this node.

    """
    def __init__(self, key):
        self._parents = set([])
        self._key = key
        self.edge_flow = OrderedDict()

    @property
    def flow(self):
        """int: The total flow from this FlowNode."""
        return sum(self.edge_flow.itervalues())

    @property
    def key(self):
        """object: The unique node identifier."""
        return self._key

    def add_flow(self, child, amount):
        """
        Adds a a flow from this node to a child node.
        If the edge already exists, adds to it, else
        we create a new edge.

        Args:
            child (FlowNode): The child who receives
                flow from this node.
            amount (int): The amount of flow to add.

        """
        if child is self:
            raise ValueError("Cannot add flow to self.")
        if child not in self.edge_flow:
            self.edge_flow[child] = amount
            child._parents.add(weakref.ref(self))
        else:
            self.edge_flow[child] += amount

    def reduce_flow(self, child, amount=INF):
        """
        Removes flow from an edge. If the flow becomes
        <= 0, the edge is removed.

        Args:
            child (FlowNode): The child node to remove
                flow from.
            amount (int): The amount of flow to remove.
                Defaults to infinity (to guarantees removal
                of the edge)

        """
        if child is self:
            raise ValueError("Cannot remove flow from self.")
        if amount < self.edge_flow[child]:
            self.edge_flow[child] -= amount
        else:
            del self.edge_flow[child]
            child._parents.remove(weakref.ref(self))

    def iter_children(self):
        """Iterate through the nodes that this node flows into.

        Yields:
            FlowNode: Each child of this node.

            """
        for child in self.edge_flow.iterkeys():
            yield child

    def iter_parents(self):
        """Iterate through the nodes that flow into this node.

        Yields:
            FlowNode: Each parents of this node.

            """
        #Stored as weakrefs
        for ref in self._parents:
            parent = ref()
            if parent is not None:
                yield parent

    def iter_edges(self):
        """Iterate through the edges of this node.

        Yields:
            tuple: (parent, child, capacity) of the flow.

        """
        for child, capacity in self.edge_flow.iteritems():
            yield (self, child, capacity)

    def iter_dfs_edges(self):
        """ A depth first search following the flow from this node.

        Note:
            The first node yielded is the calling instance itself.

        Yields:
            tuple[FlowNode]: The next (parent, child) nodes in the search.

        """
        node_queue = deque([(self, c) for c in self.edge_flow.iterkeys()])
        visited = set([c for p, c in node_queue])
        visited.add(self)
        while node_queue:
            parent, child = node_queue.pop()
            yield parent, child
            for granchild in child.edge_flow.iterkeys():
                if granchild not in visited:
                    visited.add(granchild)
                    node_queue.append((child, granchild))

    def __repr__(self):
        cls_name = self.__class__.__name__
        text = "{}({})"
        return text.format(cls_name, self.key)

class SuperSink(FlowNode):
    """The only sink node.

    If there are multiple sinks in a graph, one instance of this node
    can handle them.

    Attributes:
        consumed (int): As a sink, flows are consumed, this
            counts the consumed flow.

    """
    def __init__(self, *args, **kwargs):
        super(SuperSink, self).__init__(*args, **kwargs)
        self.consumed = 0

    @property
    def flow(self):
        """int: The total flow from this node."""
        return INF

    def add_flow(self, child, amount):
        """
        Rather than adding flow from this node to a child node,
        the sink consumes flow.

        Args:
            child (FlowNode): The child who receives
                flow from this node.
            amount (int): The amount of flow to add.

        """
        self.consumed += amount

class SuperSource(FlowNode):
    """The only source node.

    If there are multiple sources in a graph, one instance of this node
    can handle them.

    """

class FlowNetwork(object):
    """
    A class for flow network functionality. Flow networks created with this
    class automatically reduce the network size by using one sink and one
    source.

    Attributes:
        node_key_dict (OrderedDict[key, FlowNode]): The dict of all created
            nodes.

    """
    def __init__(self):
        self._source = SuperSource("+")
        self._sink = SuperSink("-")
        start_d = [(self.source.key, self.source), (self.sink.key, self.sink)]
        self.node_key_dict = OrderedDict(start_d)

    @property
    def source(self):
        """SuperSource: The only source node."""
        return self._source

    @property
    def sink(self):
        """SuperSink: The only sink node."""
        return self._sink

    def add_flow_edge(self, parent, child, capacity):
        """
        Adds a a flow from a parent node to a child node.
        If the node and edge already exists, adds to it, else
        we create a new node and/or edge.

        Args:
            parent (str): The key of the parent who is sending the
                flow.
            child (str): The key of the child who receives
                flow from this node.
            capacity (int): The max amount of flow to add.

        """
        if isinstance(parent, FlowNode) or isinstance(child, FlowNode):
            raise TypeError("add_flow_edge args are node keys, not nodes themselves.")
        if parent not in self.node_key_dict:
            self.node_key_dict[parent] = FlowNode(parent)
        parent = self.node_key_dict[parent]
        if child not in self.node_key_dict:
            self.node_key_dict[child] = FlowNode(child)
        child = self.node_key_dict[child]
        parent.add_flow(child, capacity)

    def get_maximum_flow(self):
        """Gets the maximum flow from source to sink.

        Returns:
            int: The maximum flow.

        """
        new_copy = self.from_flow_network(self)
        start_consumed = new_copy.sink.consumed
        new_copy.send_max_flow_to_sink()
        return new_copy.sink.consumed - start_consumed

    def send_max_flow_to_sink(self):
        """
        Sends the maximum amount of flow from the source to the sink.

        Partially implements the Ford-Fulkerson maximum flow algorithm
        to solve maximum flow by sending flow using function in this
        FlowNetwork and the FlowNode class.

        You end up finding the maximum flow if you check
        how much the sink consumed.

        """
        parent_map = self._get_dfs_sink_flow_path()
        while parent_map:
            minimum_path_flow = INF
            node = self.sink
            while node is not self.source:
                parent = parent_map[node]
                flow = parent.edge_flow[node]
                minimum_path_flow = min(minimum_path_flow, flow)
                node = parent
            node = self.sink
            while node is not self.source:
                parent = parent_map[node]
                flow = parent.edge_flow[node]
                parent.reduce_flow(node, minimum_path_flow)
                node.add_flow(parent, minimum_path_flow)
                node = parent
            parent_map = self._get_dfs_sink_flow_path()

    def _get_dfs_sink_flow_path(self):
        """A single path from this node to the sink.

        A breadth first search following the flow from
        this node.

        Returns:
            dict[FlowNode, FlowNode]: Each child to parent in
            the path.

        """
        node_path = {}
        for parent, child in self.source.iter_dfs_edges():
            node_path[child] = parent
            if child is self.sink:
                break
        if self.sink not in node_path:
            return dict()
        child = self.sink
        parent = node_path[child]
        exact_path = {child: parent}
        while parent is not self.source:
            child = parent
            parent = node_path[child]
            exact_path[child] = parent
        return exact_path

    def iter_edge_values(self):
        """Iterator of all edges.

        Yields:
            tuple[]: (parent, child, value) for all edges in the
            flow network.

        """
        visited = set()
        for node in self.node_key_dict.values():
            if node in visited:
                continue
            for parent, child in node.iter_dfs_edges():
                edge = (parent, child)
                if edge in visited:
                    continue
                visited.add(edge)
                yield parent, child, parent.edge_flow[child]

    @classmethod
    def from_adjacency_matrix(cls, adjacency_matrix, sources, sinks):
        """Creates a Flow Network from an adjacency matrix.

        Args:
            adjacency_matrix (list[list[int]): The adjacency matrix representation
                of a flow graph.
            sources (list[int]): list of integers denoting indexes of sources.
            sinks (list[int]): list of integers denoting indexes of sinks.

        """
        flow_network = cls()
        key_to_node = {}
        sources = set(sources)
        sinks = set(sinks)
        size = len(adjacency_matrix)
        for node_idx in xrange(size):
            if node_idx in sinks:
                node = flow_network.sink
            elif node_idx in sources:
                node = flow_network.source
            else:
                node = FlowNode(node_idx)
            key_to_node[node_idx] = node
        # Add Flow
        for node_idx in xrange(size):
            row = adjacency_matrix[node_idx]
            node = key_to_node[node_idx]
            for next_node_idx in xrange(size):
                next_node = key_to_node[next_node_idx]
                if node is next_node:
                    continue
                capacity = row[next_node_idx]
                flow_network.add_flow_edge(node.key, next_node.key, capacity)
        return flow_network

    @classmethod
    def from_flow_network(cls, flow_network):
        """Creates a copy of a flow network.

        Args:
            flow_network (FlowNetwork): The instance to copy.

        """
        new_flow_network = cls()
        for parent, child, capacity in flow_network.iter_edge_values():
            new_flow_network.add_flow_edge(parent.key, child.key, capacity)
        return new_flow_network

    def __str__(self):
        max_num = max(e[2] for e in self.iter_edge_values())
        largest_str = len(str(max_num))
        f_temp = " {:" + str(largest_str) + "}"
        all_strings = []
        for node in self.node_key_dict.itervalues():
            node_strings = []
            for node_2 in self.node_key_dict.itervalues():
                if  node_2 is node or node is self.sink:
                    txt = f_temp.format('-')
                elif node_2 not in node.edge_flow:
                    txt = f_temp.format(0)
                else:
                    txt = f_temp.format(node.edge_flow[node_2])
                node_strings.append(txt)
            node_string = ", ".join(node_strings)
            node_string = "[{}], #".format(node_string)
            if node is self.source:
                node_string += " Source"
            elif node is self.sink:
                node_string += " Sink ({})".format(self.sink.consumed)
            else:
                node_string += " {}".format(node.key)
            all_strings.append(node_string)
        if all_strings:
            all_strings[-1] = all_strings[-1].replace("], #", "]] #")
        entire_string = "\n ".join(all_strings)
        entire_string = "[{}".format(entire_string)
        return entire_string
<|endoftext|>


<commit_before>import os

from django.http import HttpResponseRedirect
from django.shortcuts import render_to_response, get_object_or_404
from django.template import RequestContext
from django.core.urlresolvers import reverse
from django.contrib.auth.decorators import login_required
from django.contrib.auth import models as auth

from etcetera.settings import SITE_ROOT
from etcetera.extras.mailer import error_mail
from etcetera.extras import models as extras
from etcetera.extras import forms as exforms
from etcetera.service import models as service
from etcetera.checkout import models as checkout

from docutils.core import publish_parts
from git import *

def error_mail(request):
	error_mail(request)
	return HttpResponseRedirect('etcetera-index')

def index(request):
	#if not request.META['REQUEST_URI'].endswith('/'):
	#	return HttpResponseRedirect("etcetera%s" % request.path)
	#repo = Repo(SITE_ROOT)
	#commit_history = repo.commits('master', max_count=3)
	posts = extras.Post.objects.all()[:3]
	context = {
		'object_list': posts,
		#'commit_history': commit_history,
	}
	return render_to_response(
		"index.html",
		context,
		context_instance=RequestContext(request)
	)

@login_required
def profile(request, the_user):
	if not the_user == request.user:
		the_user = get_object_or_404(auth.User, username=the_user)
	try:
		the_user.profile = the_user.get_profile()
	except extras.UserProfile.DoesNotExist:
		if the_user == request.user:
			return HttpResponseRedirect(reverse('edit-profile'))
		else:
			the_user.profile = None
	# Get work order counts for user.
	workorders = service.WorkOrder.objects.filter(
		technician=the_user)
	the_user.workorders_closed = workorders.exclude(
		completion_date=None).count()
	the_user.workorders_open = workorders.filter(
		completion_date=None).count()
	# Get ticket counts.
	the_user.checkouts_handled = checkout.Checkout.objects.filter(
		handling_user=the_user).count()
	the_user.checkouts_delivered = checkout.Checkout.objects.filter(
		delivering_user=the_user).count()
	context = {
		'the_user': the_user,
	}
	return render_to_response(
		"extras/profile.html",
		context,
		context_instance=RequestContext(request)
	)

def edit_profile(request):
	if request.method == 'POST':
		try:
			form = exforms.UserProfileForm(
				request.POST, request.FILES,
				instance=request.user.get_profile()
			)
		except extras.UserProfile.DoesNotExist:
			form = exforms.UserProfileForm(request.POST, request.FILES)
		if form.is_valid():
			if not form.cleaned_data['user']:
				form.cleaned_data['user'] = request.user
			form.save()
			return HttpResponseRedirect(reverse(
				'etcetera-user',
				kwargs={'the_user': request.user},
			))
	else:
		try:
			form = exforms.UserProfileForm(instance=request.user.get_profile())
		except extras.UserProfile.DoesNotExist:
			form = exforms.UserProfileForm()
		except AttributeError:
			return HttpResponseRedirect(reverse('etcetera-login'))
	context = {
		'the_user': request.user,
		'form': form,
	}
	return render_to_response(
		"extras/edit-profile.html",
		context,
		context_instance=RequestContext(request)
	)

@login_required
def change_password(request):
	user = request.user
	if request.method == 'POST':
		form = exforms.PasswordForm(request.POST)
		if form.is_valid():
			cd = form.cleaned_data
			user.set_password(cd['password'])
			user.save()
			return HttpResponseRedirect(reverse(
				'etcetera-user',
				kwargs={'the_user': request.user},
			))
	else:
		form = exforms.PasswordForm()
	context = {
		'user': user,
		'form': form,
	}
	return render_to_response(
		"extras/change-password.html",
		context,
		context_instance=RequestContext(request)
	)
<commit_msg>Removing fix.<commit_after>import os

from django.http import HttpResponseRedirect
from django.shortcuts import render_to_response, get_object_or_404
from django.template import RequestContext
from django.core.urlresolvers import reverse
from django.contrib.auth.decorators import login_required
from django.contrib.auth import models as auth

from etcetera.settings import SITE_ROOT
from etcetera.extras.mailer import error_mail
from etcetera.extras import models as extras
from etcetera.extras import forms as exforms
from etcetera.service import models as service
from etcetera.checkout import models as checkout

from docutils.core import publish_parts
from git import *

def error_mail(request):
	error_mail(request)
	return HttpResponseRedirect('etcetera-index')

def index(request):
	if not request.META['REQUEST_URI'].endswith('/'):
		return HttpResponseRedirect("etcetera%s" % request.path)
	#repo = Repo(SITE_ROOT)
	#commit_history = repo.commits('master', max_count=3)
	posts = extras.Post.objects.all()[:3]
	context = {
		'object_list': posts,
		#'commit_history': commit_history,
	}
	return render_to_response(
		"index.html",
		context,
		context_instance=RequestContext(request)
	)

@login_required
def profile(request, the_user):
	if not the_user == request.user:
		the_user = get_object_or_404(auth.User, username=the_user)
	try:
		the_user.profile = the_user.get_profile()
	except extras.UserProfile.DoesNotExist:
		if the_user == request.user:
			return HttpResponseRedirect(reverse('edit-profile'))
		else:
			the_user.profile = None
	# Get work order counts for user.
	workorders = service.WorkOrder.objects.filter(
		technician=the_user)
	the_user.workorders_closed = workorders.exclude(
		completion_date=None).count()
	the_user.workorders_open = workorders.filter(
		completion_date=None).count()
	# Get ticket counts.
	the_user.checkouts_handled = checkout.Checkout.objects.filter(
		handling_user=the_user).count()
	the_user.checkouts_delivered = checkout.Checkout.objects.filter(
		delivering_user=the_user).count()
	context = {
		'the_user': the_user,
	}
	return render_to_response(
		"extras/profile.html",
		context,
		context_instance=RequestContext(request)
	)

def edit_profile(request):
	if request.method == 'POST':
		try:
			form = exforms.UserProfileForm(
				request.POST, request.FILES,
				instance=request.user.get_profile()
			)
		except extras.UserProfile.DoesNotExist:
			form = exforms.UserProfileForm(request.POST, request.FILES)
		if form.is_valid():
			if not form.cleaned_data['user']:
				form.cleaned_data['user'] = request.user
			form.save()
			return HttpResponseRedirect(reverse(
				'etcetera-user',
				kwargs={'the_user': request.user},
			))
	else:
		try:
			form = exforms.UserProfileForm(instance=request.user.get_profile())
		except extras.UserProfile.DoesNotExist:
			form = exforms.UserProfileForm()
		except AttributeError:
			return HttpResponseRedirect(reverse('etcetera-login'))
	context = {
		'the_user': request.user,
		'form': form,
	}
	return render_to_response(
		"extras/edit-profile.html",
		context,
		context_instance=RequestContext(request)
	)

@login_required
def change_password(request):
	user = request.user
	if request.method == 'POST':
		form = exforms.PasswordForm(request.POST)
		if form.is_valid():
			cd = form.cleaned_data
			user.set_password(cd['password'])
			user.save()
			return HttpResponseRedirect(reverse(
				'etcetera-user',
				kwargs={'the_user': request.user},
			))
	else:
		form = exforms.PasswordForm()
	context = {
		'user': user,
		'form': form,
	}
	return render_to_response(
		"extras/change-password.html",
		context,
		context_instance=RequestContext(request)
	)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: UTF-8 -*-
#
# ezIBpy: Pythonic Wrapper for IbPy
# https://github.com/ranaroussi/ezibpy
#
# Copyright 2015 Ran Aroussi
#
# Licensed under the GNU Lesser General Public License, v3.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.gnu.org/licenses/lgpl-3.0.en.html
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# IB Referrence:
# --------------
# https://www.interactivebrokers.com/en/software/api/apiguide/java/reqhistoricaldata.htm
# https://www.interactivebrokers.com/en/software/api/apiguide/tables/tick_types.htm
import logging

from ib.ext.Contract import Contract
from ib.ext.Order import Order

dataTypes = {
    "MONTH_CODES" : ['','F','G','H','J','K','M','N','Q','U','V','X','Z'],

    "PRICE_TICKS" : { 1:"bid", 2:"ask", 4:"last", 6:"high", 7:"low", 9:"close", 14:"open"},
    "SIZE_TICKS"  : { 0:"bid", 3:"ask", 5:"last", 8:"volume"},
    "RTVOL_TICKS" : { "instrument":"", "ticketId":0, "price":0, "size":0, "time":0, "volume":0, "vwap":0, "single":0 },

    # API warning codes that are not actually problems and should not be logged
    "BENIGN_ERROR_CODES"             : (200, 2104, 2106),
    # API error codes indicating IB/TWS disconnection
    "DISCONNECT_ERROR_CODES"         : (504, 502, 1100, 1300, 2110),

    "FIELD_BID_SIZE"                  : 0,
    "FIELD_BID_PRICE"                 : 1,
    "FIELD_ASK_PRICE"                 : 2,
    "FIELD_ASK_SIZE"                  : 3,
    "FIELD_LAST_PRICE"                : 4,
    "FIELD_LAST_SIZE"                 : 5,
    "FIELD_HIGH"                      : 6,
    "FIELD_LOW"                       : 7,
    "FIELD_VOLUME"                    : 8,
    "FIELD_CLOSE_PRICE"               : 9,

    "FIELD_BID_OPTION_COMPUTATION"    : 10,
    "FIELD_ASK_OPTION_COMPUTATION"    : 11,
    "FIELD_LAST_OPTION_COMPUTATION"   : 12,
    "FIELD_MODEL_OPTION_COMPUTATION"  : 13,

    "FIELD_OPEN_INTEREST"             : 22, #  tickSize()
    "FIELD_OPTION_HISTORICAL_VOL"     : 23, #  tickGeneric()
    "FIELD_OPTION_IMPLIED_VOL"        : 24, #  tickGeneric()
    "FIELD_OPTION_CALL_OPEN_INTEREST" : 27, #  tickSize()
    "FIELD_OPTION_PUT_OPEN_INTEREST"  : 28, #  tickSize()
    "FIELD_OPTION_CALL_VOLUME"        : 29, #  tickSize()
    "FIELD_OPTION_PUT_VOLUME"         : 30, #  tickSize()

    "FIELD_AVG_VOLUME"                : 21,
    "FIELD_BID_EXCH"                  : 32,
    "FIELD_ASK_EXCH"                  : 33,
    "FIELD_AUCTION_VOLUME"            : 34,
    "FIELD_AUCTION_PRICE"             : 35,
    "FIELD_LAST_TIMESTAMP"            : 45,
    "FIELD_RTVOLUME"                  : 48,
    "FIELD_HALTED"                    : 49,
    "FIELD_TRADE_COUNT"               : 54,
    "FIELD_TRADE_RATE"                : 55,
    "FIELD_VOLUME_RATE"               : 56,

    "FIELD_HALTED_NOT_HALTED"         : 0,
    "FIELD_HALTED_IS_HALTED"          : 1,
    "FIELD_HALTED_BY_VOLATILITY"      : 2,

    "DURATION_1_HR"                   : "3600 S",
    "DURATION_1_MIN"                  : "60 S",
    "DURATION_1_DAY"                  : "1 D",

    "BAR_SIZE_1_SEC"                  : "1 secs",
    "BAR_SIZE_1_MIN"                  : "1 min",

    "RTH_ALL"                         : 0,
    "RTH_ONLY_TRADING_HRS"            : 1,

    "WHAT_TO_SHOW_TRADES"             : "TRADES",
    "WHAT_TO_SHOW_MID_PT"             : "MIDPOINT",
    "WHAT_TO_SHOW_BID"                : "BID",
    "WHAT_TO_SHOW_ASK"                : "ASK",
    "WHAT_TO_SHOW_BID_ASK"            : "BID_ASK",
    "WHAT_TO_SHOW_HVOL"               : "HISTORICAL_VOLATILITY",
    "WHAT_TO_SHOW_OPT_IMPV"           : "OPTION_IMPLIED_VOLATILITY",

    "DATEFORMAT_STRING"               : 1,
    "DATEFORMAT_UNIX_TS"              : 2,

    "MSG_CURRENT_TIME"                : "currentTime",
    "MSG_COMMISSION_REPORT"           : "commissionReport",
    "MSG_CONNECTION_CLOSED"           : "connectionClosed",

    "MSG_CONTRACT_DETAILS"            : "contractDetails",
    "MSG_CONTRACT_DETAILS_END"        : "contractDetailsEnd",
    "MSG_TICK_SNAPSHOT_END"           : "tickSnapshotEnd",

    "MSG_TYPE_HISTORICAL_DATA"        : "historicalData",
    "MSG_TYPE_ACCOUNT_UPDATES"        : "updateAccountValue",
    "MSG_TYPE_ACCOUNT_TIME_UPDATES"   : "updateAccountTime",
    "MSG_TYPE_PORTFOLIO_UPDATES"      : "updatePortfolio",
    "MSG_TYPE_POSITION"               : "position",
    "MSG_TYPE_MANAGED_ACCOUNTS"       : "managedAccounts",

    "MSG_TYPE_NEXT_ORDER_ID"          : "nextValidId",
    "MSG_TYPE_OPEN_ORDER"             : "openOrder",
    "MSG_TYPE_ORDER_STATUS"           : "orderStatus",

    "MSG_TYPE_MKT_DEPTH"              : "updateMktDepth",
    "MSG_TYPE_MKT_DEPTH_L2"           : "updateMktDepthL2",

    "MSG_TYPE_TICK_PRICE"             : "tickPrice",
    "MSG_TYPE_TICK_STRING"            : "tickString",
    "MSG_TYPE_TICK_GENERIC"           : "tickGeneric",
    "MSG_TYPE_TICK_SIZE"              : "tickSize",
    "MSG_TYPE_TICK_OPTION"            : "tickOptionComputation",

    "DATE_FORMAT"                     : "%Y%m%d",
    "DATE_TIME_FORMAT"                : "%Y%m%d %H:%M:%S",
    "DATE_TIME_FORMAT_LONG"           : "%Y-%m-%d %H:%M:%S",
    "DATE_TIME_FORMAT_LONG_MILLISECS" : "%Y-%m-%d %H:%M:%S.%f",
    "DATE_TIME_FORMAT_HISTORY"        : "%Y%m%d %H:%M:%S",
    "DATE_FORMAT_HISTORY"             : "%Y-%m-%d",

    "GENERIC_TICKS_NONE"              : "",
    "GENERIC_TICKS_RTVOLUME"          : "233",

    "SNAPSHOT_NONE"                   : False,
    "SNAPSHOT_TRUE"                   : True,

    # MARKET ORDERS
    "ORDER_TYPE_MARKET"               : "MKT",
    "ORDER_TYPE_MOC"                  : "MOC", # MARKET ON CLOSE
    "ORDER_TYPE_MOO"                  : "MOO", # MARKET ON OPEN
    "ORDER_TYPE_MIT"                  : "MIT", # MARKET IF TOUCHED

    # LIMIT ORDERS
    "ORDER_TYPE_LIMIT"                : "LMT",
    "ORDER_TYPE_LOC"                  : "LOC", # LIMIT ON CLOSE
    "ORDER_TYPE_LOO"                  : "LOO", # LIMIT ON OPEN
    "ORDER_TYPE_LIT"                  : "LIT", # LIMIT IF TOUCHED

    # STOP ORDERS
    "ORDER_TYPE_STOP"                 : "STP",
    "ORDER_TYPE_STOP_LIMIT"           : "STP LMT",
    "ORDER_TYPE_TRAIL_STOP"           : "TRAIL",
    "ORDER_TYPE_TRAIL_STOP_LIMIT"     : "TRAIL LIMIT",
    "ORDER_TYPE_TRAIL_STOP_LIT"       : "TRAIL LIT", # LIMIT IF TOUCHED
    "ORDER_TYPE_TRAIL_STOP_MIT"       : "TRAIL MIT", # MARKET IF TOUCHED

    # MINC ORDERS
    "ORDER_TYPE_ONE_CANCELS_ALL"      : "OCA",
    "ORDER_TYPE_RELATIVE"             : "REL",
    "ORDER_TYPE_PEGGED_TO_MIDPOINT"   : "PEG MID",

    # ORDER ACTIONS
    "ORDER_ACTION_SELL"               : "SELL",
    "ORDER_ACTION_BUY"                : "BUY",
    "ORDER_ACTION_SHORT"              : "SSHORT"

}

def createLogger(name, level=logging.WARNING):
    """:Return: a logger with the given `name` and optional `level`."""
    logger = logging.getLogger(name)
    logger.setLevel(level)
    handler = logging.StreamHandler()
    handler.setFormatter(logging.Formatter('%(asctime)s [%(levelname)s] %(name)s: %(message)s'))
    logger.addHandler(handler)
    logger.propagate = False
    return logger

def order_to_dict(order):
    """Convert an IBPy Order object to a dict containing any non-default values."""
    default = Order()
    return {field: val for field, val in vars(order).items() if val != getattr(default, field, None)}

def contract_to_dict(contract):
    """Convert an IBPy Contract object to a dict containing any non-default values."""
    default = Contract()
    return {field: val for field, val in vars(contract).items() if val != getattr(default, field, None)}
<commit_msg>added dividers<commit_after>#!/usr/bin/env python
# -*- coding: UTF-8 -*-
#
# ezIBpy: Pythonic Wrapper for IbPy
# https://github.com/ranaroussi/ezibpy
#
# Copyright 2015 Ran Aroussi
#
# Licensed under the GNU Lesser General Public License, v3.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.gnu.org/licenses/lgpl-3.0.en.html
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# IB Referrence:
# --------------
# https://www.interactivebrokers.com/en/software/api/apiguide/java/reqhistoricaldata.htm
# https://www.interactivebrokers.com/en/software/api/apiguide/tables/tick_types.htm
import logging

from ib.ext.Contract import Contract
from ib.ext.Order import Order

# ---------------------------------------------------------
dataTypes = {
    "MONTH_CODES" : ['','F','G','H','J','K','M','N','Q','U','V','X','Z'],

    "PRICE_TICKS" : { 1:"bid", 2:"ask", 4:"last", 6:"high", 7:"low", 9:"close", 14:"open"},
    "SIZE_TICKS"  : { 0:"bid", 3:"ask", 5:"last", 8:"volume"},
    "RTVOL_TICKS" : { "instrument":"", "ticketId":0, "price":0, "size":0, "time":0, "volume":0, "vwap":0, "single":0 },

    # API warning codes that are not actually problems and should not be logged
    "BENIGN_ERROR_CODES"             : (200, 2104, 2106),
    # API error codes indicating IB/TWS disconnection
    "DISCONNECT_ERROR_CODES"         : (504, 502, 1100, 1300, 2110),

    "FIELD_BID_SIZE"                  : 0,
    "FIELD_BID_PRICE"                 : 1,
    "FIELD_ASK_PRICE"                 : 2,
    "FIELD_ASK_SIZE"                  : 3,
    "FIELD_LAST_PRICE"                : 4,
    "FIELD_LAST_SIZE"                 : 5,
    "FIELD_HIGH"                      : 6,
    "FIELD_LOW"                       : 7,
    "FIELD_VOLUME"                    : 8,
    "FIELD_CLOSE_PRICE"               : 9,

    "FIELD_BID_OPTION_COMPUTATION"    : 10,
    "FIELD_ASK_OPTION_COMPUTATION"    : 11,
    "FIELD_LAST_OPTION_COMPUTATION"   : 12,
    "FIELD_MODEL_OPTION_COMPUTATION"  : 13,

    "FIELD_OPEN_INTEREST"             : 22, #  tickSize()
    "FIELD_OPTION_HISTORICAL_VOL"     : 23, #  tickGeneric()
    "FIELD_OPTION_IMPLIED_VOL"        : 24, #  tickGeneric()
    "FIELD_OPTION_CALL_OPEN_INTEREST" : 27, #  tickSize()
    "FIELD_OPTION_PUT_OPEN_INTEREST"  : 28, #  tickSize()
    "FIELD_OPTION_CALL_VOLUME"        : 29, #  tickSize()
    "FIELD_OPTION_PUT_VOLUME"         : 30, #  tickSize()

    "FIELD_AVG_VOLUME"                : 21,
    "FIELD_BID_EXCH"                  : 32,
    "FIELD_ASK_EXCH"                  : 33,
    "FIELD_AUCTION_VOLUME"            : 34,
    "FIELD_AUCTION_PRICE"             : 35,
    "FIELD_LAST_TIMESTAMP"            : 45,
    "FIELD_RTVOLUME"                  : 48,
    "FIELD_HALTED"                    : 49,
    "FIELD_TRADE_COUNT"               : 54,
    "FIELD_TRADE_RATE"                : 55,
    "FIELD_VOLUME_RATE"               : 56,

    "FIELD_HALTED_NOT_HALTED"         : 0,
    "FIELD_HALTED_IS_HALTED"          : 1,
    "FIELD_HALTED_BY_VOLATILITY"      : 2,

    "DURATION_1_HR"                   : "3600 S",
    "DURATION_1_MIN"                  : "60 S",
    "DURATION_1_DAY"                  : "1 D",

    "BAR_SIZE_1_SEC"                  : "1 secs",
    "BAR_SIZE_1_MIN"                  : "1 min",

    "RTH_ALL"                         : 0,
    "RTH_ONLY_TRADING_HRS"            : 1,

    "WHAT_TO_SHOW_TRADES"             : "TRADES",
    "WHAT_TO_SHOW_MID_PT"             : "MIDPOINT",
    "WHAT_TO_SHOW_BID"                : "BID",
    "WHAT_TO_SHOW_ASK"                : "ASK",
    "WHAT_TO_SHOW_BID_ASK"            : "BID_ASK",
    "WHAT_TO_SHOW_HVOL"               : "HISTORICAL_VOLATILITY",
    "WHAT_TO_SHOW_OPT_IMPV"           : "OPTION_IMPLIED_VOLATILITY",

    "DATEFORMAT_STRING"               : 1,
    "DATEFORMAT_UNIX_TS"              : 2,

    "MSG_CURRENT_TIME"                : "currentTime",
    "MSG_COMMISSION_REPORT"           : "commissionReport",
    "MSG_CONNECTION_CLOSED"           : "connectionClosed",

    "MSG_CONTRACT_DETAILS"            : "contractDetails",
    "MSG_CONTRACT_DETAILS_END"        : "contractDetailsEnd",
    "MSG_TICK_SNAPSHOT_END"           : "tickSnapshotEnd",

    "MSG_TYPE_HISTORICAL_DATA"        : "historicalData",
    "MSG_TYPE_ACCOUNT_UPDATES"        : "updateAccountValue",
    "MSG_TYPE_ACCOUNT_TIME_UPDATES"   : "updateAccountTime",
    "MSG_TYPE_PORTFOLIO_UPDATES"      : "updatePortfolio",
    "MSG_TYPE_POSITION"               : "position",
    "MSG_TYPE_MANAGED_ACCOUNTS"       : "managedAccounts",

    "MSG_TYPE_NEXT_ORDER_ID"          : "nextValidId",
    "MSG_TYPE_OPEN_ORDER"             : "openOrder",
    "MSG_TYPE_ORDER_STATUS"           : "orderStatus",

    "MSG_TYPE_MKT_DEPTH"              : "updateMktDepth",
    "MSG_TYPE_MKT_DEPTH_L2"           : "updateMktDepthL2",

    "MSG_TYPE_TICK_PRICE"             : "tickPrice",
    "MSG_TYPE_TICK_STRING"            : "tickString",
    "MSG_TYPE_TICK_GENERIC"           : "tickGeneric",
    "MSG_TYPE_TICK_SIZE"              : "tickSize",
    "MSG_TYPE_TICK_OPTION"            : "tickOptionComputation",

    "DATE_FORMAT"                     : "%Y%m%d",
    "DATE_TIME_FORMAT"                : "%Y%m%d %H:%M:%S",
    "DATE_TIME_FORMAT_LONG"           : "%Y-%m-%d %H:%M:%S",
    "DATE_TIME_FORMAT_LONG_MILLISECS" : "%Y-%m-%d %H:%M:%S.%f",
    "DATE_TIME_FORMAT_HISTORY"        : "%Y%m%d %H:%M:%S",
    "DATE_FORMAT_HISTORY"             : "%Y-%m-%d",

    "GENERIC_TICKS_NONE"              : "",
    "GENERIC_TICKS_RTVOLUME"          : "233",

    "SNAPSHOT_NONE"                   : False,
    "SNAPSHOT_TRUE"                   : True,

    # MARKET ORDERS
    "ORDER_TYPE_MARKET"               : "MKT",
    "ORDER_TYPE_MOC"                  : "MOC", # MARKET ON CLOSE
    "ORDER_TYPE_MOO"                  : "MOO", # MARKET ON OPEN
    "ORDER_TYPE_MIT"                  : "MIT", # MARKET IF TOUCHED

    # LIMIT ORDERS
    "ORDER_TYPE_LIMIT"                : "LMT",
    "ORDER_TYPE_LOC"                  : "LOC", # LIMIT ON CLOSE
    "ORDER_TYPE_LOO"                  : "LOO", # LIMIT ON OPEN
    "ORDER_TYPE_LIT"                  : "LIT", # LIMIT IF TOUCHED

    # STOP ORDERS
    "ORDER_TYPE_STOP"                 : "STP",
    "ORDER_TYPE_STOP_LIMIT"           : "STP LMT",
    "ORDER_TYPE_TRAIL_STOP"           : "TRAIL",
    "ORDER_TYPE_TRAIL_STOP_LIMIT"     : "TRAIL LIMIT",
    "ORDER_TYPE_TRAIL_STOP_LIT"       : "TRAIL LIT", # LIMIT IF TOUCHED
    "ORDER_TYPE_TRAIL_STOP_MIT"       : "TRAIL MIT", # MARKET IF TOUCHED

    # MINC ORDERS
    "ORDER_TYPE_ONE_CANCELS_ALL"      : "OCA",
    "ORDER_TYPE_RELATIVE"             : "REL",
    "ORDER_TYPE_PEGGED_TO_MIDPOINT"   : "PEG MID",

    # ORDER ACTIONS
    "ORDER_ACTION_SELL"               : "SELL",
    "ORDER_ACTION_BUY"                : "BUY",
    "ORDER_ACTION_SHORT"              : "SSHORT"

}

# ---------------------------------------------------------
def createLogger(name, level=logging.WARNING):
    """:Return: a logger with the given `name` and optional `level`."""
    logger = logging.getLogger(name)
    logger.setLevel(level)
    handler = logging.StreamHandler()
    handler.setFormatter(logging.Formatter('%(asctime)s [%(levelname)s] %(name)s: %(message)s'))
    logger.addHandler(handler)
    logger.propagate = False
    return logger

# ---------------------------------------------------------
def order_to_dict(order):
    """Convert an IBPy Order object to a dict containing any non-default values."""
    default = Order()
    return {field: val for field, val in vars(order).items() if val != getattr(default, field, None)}

# ---------------------------------------------------------
def contract_to_dict(contract):
    """Convert an IBPy Contract object to a dict containing any non-default values."""
    default = Contract()
    return {field: val for field, val in vars(contract).items() if val != getattr(default, field, None)}
<|endoftext|>