<commit_before>"""Provides convenient access to SQLObject-managed and/or SQLAlchemy-managed
databases.

This module enables easy use of an SQLObject database by providing an
auto-connect hub that will utilize the db uri string given in the Paste conf
file called ``sqlobject.dburi``.

A SQLAlchemy ``SessionContext`` is also available: it provides both thread and
process safe ``Session`` objects via the ``session_context.current`` property.
"""
import logging
import thread

from paste.deploy.converters import asbool

import pylons

__all__ = ["AutoConnectHub", "PackageHub"]

log = logging.getLogger(__name__)

try:
    import sqlalchemy
    from sqlalchemy.ext import sessioncontext

    def app_scope():
        """Return the id keying the current database session's scope.

        The session is particular to the current Pylons application -- this
        returns an id generated from the current thread and the current Pylons
        application's Globals object at pylons.g (if one is registered).
        """
        try:
            app_scope_id = str(id(pylons.config._current_obj()))
        except TypeError:
            app_scope_id = ''
        log.debug("Returning %s as the database session scope id",
                  app_scope_id)
        return '%s|%i' % (app_scope_id, thread.get_ident())

    def create_engine(uri=None, echo=None, **kwargs):
        """Return a SQLAlchemy db engine. Uses the configuration values from
        ``get_engine_conf`` for uri and echo if none are specified.

        Engines are cached in the ``get_engines`` dict.
        """
        uri, echo = get_engine_conf(uri, echo)
        kwargs['echo'] = echo
        engine_key = '%s|%s' % (uri, str(kwargs))
        db_engines = pylons.config['pylons.db_engines']
        if engine_key in db_engines:
            engine = db_engines[engine_key]
        else:
            engine = db_engines[engine_key] = \
                sqlalchemy.create_engine(uri, **kwargs)
        log.debug("Created engine using uri: %s with echo %s", uri, echo)
        return engine

    def get_engine_conf(uri=None, echo=None):
        """Returns a tuple of SQLAlchemy engine configuration values (uri,
        echo) from the Pylons config file values ``sqlalchemy.dburi`` and
        ``sqlalchemy.echo`` when none are specified."""
        if uri is None:
            uri = pylons.config.get('sqlalchemy.dburi')
        if not uri:
            raise KeyError('No SQLAlchemy database config found!')
        if echo is None:
            echo = pylons.config.get('sqlalchemy.echo', False)
            if echo != 'debug':
                echo = asbool(echo)
        return uri, echo

    def make_session(uri=None, echo=None, session_kwargs=None, **kwargs):
        """Returns a SQLAlchemy session for the specified database uri from
        the the engine cache (returned from ``get_engines``)``. Uses the
        configuration values from ``get_engine_conf`` for uri and echo when
        None are specified.

        ``session_kwargs`` are passed to SQLAlchemy's ``create_session``
        function as keyword arguments.

        If the uri's engine does not exist, it will be created and added to
        the engine cache.
        """
        if session_kwargs is None:
            session_kwargs = {}
        engine = create_engine(uri, echo=echo, **kwargs)
        log.debug("Created engine for session context")
        return sqlalchemy.create_session(bind_to=engine, **session_kwargs)

    session_context = sessioncontext.SessionContext(make_session,
                                                    scopefunc=app_scope)

    __all__.extend(['app_scope', 'create_engine', 'get_engine_conf',
                    'make_session', 'session_context'])

except ImportError:
    pass

# Provide support for sqlobject
try:
    import sqlobject
    from sqlobject.dbconnection import ConnectionHub, Transaction, TheURIOpener
except:
    ConnectionHub = object

class AutoConnectHub(ConnectionHub):
    """Connects to the database once per thread.

    The AutoConnectHub also provides convenient methods for managing
    transactions.
    """
    uri = None
    params = {}

    def __init__(self, uri=None, pool_connections=True):
        if not uri:
            uri = pylons.config.get('sqlobject.dburi')
        self.uri = uri
        self.pool_connections = pool_connections
        ConnectionHub.__init__(self)

    def getConnection(self):
        try:
            conn = self.threadingLocal.connection
            return conn
        except AttributeError:
            if self.uri:
                conn = sqlobject.connectionForURI(self.uri)
                # the following line effectively turns off the DBAPI connection
                # cache. We're already holding on to a connection per thread,
                # and the cache causes problems with sqlite.
                if self.uri.startswith("sqlite"):
                    TheURIOpener.cachedURIs = {}
                self.threadingLocal.connection = conn
                if not self.pool_connections:
                    # This disables pooling
                    conn._pool = None
                return conn
            try:
                return self.processConnection
            except AttributeError:
                raise AttributeError(
                    "No connection has been defined for this thread "
                    "or process")

    def begin(self):
        """Starts a transaction."""
        conn = self.getConnection()
        if isinstance(conn, Transaction):
            if conn._obsolete:
                conn.begin()
            return
        self.threadingLocal.old_conn = conn
        self.threadingLocal.connection = conn.transaction()

    def commit(self):
        """Commits the current transaction."""
        conn = self.threadingLocal.connection
        if isinstance(conn, Transaction):
            self.threadingLocal.connection.commit()

    def rollback(self):
        """Rolls back the current transaction."""
        conn = self.threadingLocal.connection
        if isinstance(conn, Transaction) and not conn._obsolete:
            self.threadingLocal.connection.rollback()

    def end(self):
        """Ends the transaction, returning to a standard connection."""
        conn = self.threadingLocal.connection
        if not isinstance(conn, Transaction):
            return
        if not conn._obsolete:
            conn.rollback()
        self.threadingLocal.connection = self.threadingLocal.old_conn
        del self.threadingLocal.old_conn
        self.threadingLocal.connection.cache.clear()

# This dictionary stores the AutoConnectHubs used for each
# connection URI
_hubs = dict()

class UnconfiguredConnectionError(KeyError):
    """
    Raised when no configuration is available to set up a connection.
    """

class PackageHub(object):
    """Transparently proxies to an AutoConnectHub for the URI
    that is appropriate for this package. A package URI is
    configured via "packagename.dburi" in the Paste ini file
    settings. If there is no package DB URI configured, the
    default (provided by "sqlobject.dburi") is used.

    The hub is not instantiated until an attempt is made to
    use the database.

    If pool_connections=False, then a new database connection
    will be opened for every request.  This will avoid
    problems with database connections that periodically die.
    """
    def __init__(self, packagename, dburi=None, pool_connections=True):
        self.packagename = packagename
        self.hub = None
        self.dburi = dburi
        self.pool_connections = pool_connections

    def __get__(self, obj, type):
        if not self.hub:
            try:
                self.set_hub()
            except UnconfiguredConnectionError, e:
                raise AttributeError(str(e))
        return self.hub.__get__(obj, type)

    def __set__(self, obj, type):
        if not self.hub:
            self.set_hub()
        return self.hub.__set__(obj, type)

    def __getattr__(self, name):
        if not self.hub:
            self.set_hub()
        return getattr(self.hub, name)

    def set_hub(self):
        dburi = self.dburi
        if not dburi:
            try:
                dburi = pylons.config.get("%s.dburi" % \
                                                   self.packagename)
            except TypeError, e:
                # No configuration is registered
                raise UnconfiguredConnectionError(str(e))
        if not dburi:
            dburi = pylons.config.get("sqlobject.dburi")
        if not dburi:
            raise UnconfiguredConnectionError(
                "No database configuration found!")
        hub = _hubs.get(dburi)
        if not hub:
            hub = AutoConnectHub(
                dburi, pool_connections=self.pool_connections)
            _hubs[dburi] = hub
        self.hub = hub
<commit_msg>[svn] Improve pylons.database get_engine_conf and create_engine to support general configuration options such as convert_unicode and encoding. Backward compatibility is maintained.<commit_after>"""Provides convenient access to SQLObject-managed and/or SQLAlchemy-managed
databases.

This module enables easy use of an SQLObject database by providing an
auto-connect hub that will utilize the db uri string given in the Paste conf
file called ``sqlobject.dburi``.

A SQLAlchemy ``SessionContext`` is also available: it provides both thread and
process safe ``Session`` objects via the ``session_context.current`` property.
"""
import logging
import thread

from paste.deploy.converters import asbool

import pylons

__all__ = ["AutoConnectHub", "PackageHub"]

log = logging.getLogger(__name__)

try:
    import sqlalchemy
    from sqlalchemy.ext import sessioncontext

    BOOL_OPTIONS = set([
            "convert_unicode",
            "echo",
            "echo_pool",
            "threaded",
            "use_ansi",
            "use_oids",
            ])

    INT_OPTIONS = set([
            "max_overflow",
            "pool_size",
            "pool_recycle",
            "pool_timeout",
            ])

    def app_scope():
        """Return the id keying the current database session's scope.

        The session is particular to the current Pylons application -- this
        returns an id generated from the current thread and the current Pylons
        application's Globals object at pylons.g (if one is registered).
        """
        try:
            app_scope_id = str(id(pylons.config._current_obj()))
        except TypeError:
            app_scope_id = ''
        log.debug("Returning %s as the database session scope id",
                  app_scope_id)
        return '%s|%i' % (app_scope_id, thread.get_ident())

    def create_engine(uri=None, echo=None, **kwargs):
        """Return a SQLAlchemy db engine. Uses the configuration values from
        ``get_engine_conf``.

        Engines are cached in the ``get_engines`` dict.
        """
        conf = get_engine_conf()
        conf.update(kwargs)

        # replace 'dburi' with 'uri' for consistency
        if 'dburi' in conf:
            if not 'uri' in conf:
                conf['uri'] = conf['dburi']
            del conf['dburi']

        # override config with passed-in values
        conf['uri'] = uri or conf.get('uri')
        conf['echo'] = asbool(echo) or conf.get('echo')

        uri = conf.pop('uri')
        assert uri

        # call create_engine or fetch engine from cache

        ## use a sorted list of tuples since order isn't guaranteed
        ## in the dict
        conf_key = str(sorted(conf.items(), key=lambda x: x[0]))

        engine_key = '%s|%s' % (uri, conf_key)
        db_engines = pylons.config['pylons.db_engines']
        if engine_key in db_engines:
            engine = db_engines[engine_key]
        else:
            engine = db_engines[engine_key] = \
                sqlalchemy.create_engine(uri, **conf)

        log.debug("Created engine using uri: %s with engine arguments %s", uri, conf)
        return engine

    def get_engine_conf():
        """Returns a dict of SQLAlchemy engine configuration values
        from the Pylons config file values ``sqlalchemy.*``"""
        result = {}
        for k,v in pylons.config.iteritems():
            if not k.startswith('sqlalchemy.'):
                continue
            k = k[11:]
            if k in BOOL_OPTIONS:
                result[k] = asbool(v)
            elif k in INT_OPTIONS:
                try:
                    result[k] = int(v)
                except ValueError:
                    reason = 'config sqlalchemy.%s is not an integer: %s'
                    raise ValueError(reason % (k,v))
            else:
                result[k] = v
        return result

    def make_session(uri=None, echo=None, session_kwargs=None, **kwargs):
        """Returns a SQLAlchemy session for the specified database uri from
        the the engine cache (returned from ``get_engines``)``. Uses the
        configuration values from ``get_engine_conf`` for uri and echo when
        None are specified.

        ``session_kwargs`` are passed to SQLAlchemy's ``create_session``
        function as keyword arguments.

        If the uri's engine does not exist, it will be created and added to
        the engine cache.
        """
        if session_kwargs is None:
            session_kwargs = {}
        engine = create_engine(uri, echo=echo, **kwargs)
        log.debug("Created engine for session context")
        return sqlalchemy.create_session(bind_to=engine, **session_kwargs)

    session_context = sessioncontext.SessionContext(make_session,
                                                    scopefunc=app_scope)

    __all__.extend(['app_scope', 'create_engine', 'get_engine_conf',
                    'make_session', 'session_context'])

except ImportError:
    pass

# Provide support for sqlobject
try:
    import sqlobject
    from sqlobject.dbconnection import ConnectionHub, Transaction, TheURIOpener
except:
    ConnectionHub = object

class AutoConnectHub(ConnectionHub):
    """Connects to the database once per thread.

    The AutoConnectHub also provides convenient methods for managing
    transactions.
    """
    uri = None
    params = {}

    def __init__(self, uri=None, pool_connections=True):
        if not uri:
            uri = pylons.config.get('sqlobject.dburi')
        self.uri = uri
        self.pool_connections = pool_connections
        ConnectionHub.__init__(self)

    def getConnection(self):
        try:
            conn = self.threadingLocal.connection
            return conn
        except AttributeError:
            if self.uri:
                conn = sqlobject.connectionForURI(self.uri)
                # the following line effectively turns off the DBAPI connection
                # cache. We're already holding on to a connection per thread,
                # and the cache causes problems with sqlite.
                if self.uri.startswith("sqlite"):
                    TheURIOpener.cachedURIs = {}
                self.threadingLocal.connection = conn
                if not self.pool_connections:
                    # This disables pooling
                    conn._pool = None
                return conn
            try:
                return self.processConnection
            except AttributeError:
                raise AttributeError(
                    "No connection has been defined for this thread "
                    "or process")

    def begin(self):
        """Starts a transaction."""
        conn = self.getConnection()
        if isinstance(conn, Transaction):
            if conn._obsolete:
                conn.begin()
            return
        self.threadingLocal.old_conn = conn
        self.threadingLocal.connection = conn.transaction()

    def commit(self):
        """Commits the current transaction."""
        conn = self.threadingLocal.connection
        if isinstance(conn, Transaction):
            self.threadingLocal.connection.commit()

    def rollback(self):
        """Rolls back the current transaction."""
        conn = self.threadingLocal.connection
        if isinstance(conn, Transaction) and not conn._obsolete:
            self.threadingLocal.connection.rollback()

    def end(self):
        """Ends the transaction, returning to a standard connection."""
        conn = self.threadingLocal.connection
        if not isinstance(conn, Transaction):
            return
        if not conn._obsolete:
            conn.rollback()
        self.threadingLocal.connection = self.threadingLocal.old_conn
        del self.threadingLocal.old_conn
        self.threadingLocal.connection.cache.clear()

# This dictionary stores the AutoConnectHubs used for each
# connection URI
_hubs = dict()

class UnconfiguredConnectionError(KeyError):
    """
    Raised when no configuration is available to set up a connection.
    """

class PackageHub(object):
    """Transparently proxies to an AutoConnectHub for the URI
    that is appropriate for this package. A package URI is
    configured via "packagename.dburi" in the Paste ini file
    settings. If there is no package DB URI configured, the
    default (provided by "sqlobject.dburi") is used.

    The hub is not instantiated until an attempt is made to
    use the database.

    If pool_connections=False, then a new database connection
    will be opened for every request.  This will avoid
    problems with database connections that periodically die.
    """
    def __init__(self, packagename, dburi=None, pool_connections=True):
        self.packagename = packagename
        self.hub = None
        self.dburi = dburi
        self.pool_connections = pool_connections

    def __get__(self, obj, type):
        if not self.hub:
            try:
                self.set_hub()
            except UnconfiguredConnectionError, e:
                raise AttributeError(str(e))
        return self.hub.__get__(obj, type)

    def __set__(self, obj, type):
        if not self.hub:
            self.set_hub()
        return self.hub.__set__(obj, type)

    def __getattr__(self, name):
        if not self.hub:
            self.set_hub()
        return getattr(self.hub, name)

    def set_hub(self):
        dburi = self.dburi
        if not dburi:
            try:
                dburi = pylons.config.get("%s.dburi" % \
                                                   self.packagename)
            except TypeError, e:
                # No configuration is registered
                raise UnconfiguredConnectionError(str(e))
        if not dburi:
            dburi = pylons.config.get("sqlobject.dburi")
        if not dburi:
            raise UnconfiguredConnectionError(
                "No database configuration found!")
        hub = _hubs.get(dburi)
        if not hub:
            hub = AutoConnectHub(
                dburi, pool_connections=self.pool_connections)
            _hubs[dburi] = hub
        self.hub = hub
<|endoftext|>


<commit_before>#!/usr/bin/env python
import rospy
import actionlib

# NEED TO DO: I want to make sure that this random, which is used to make the wave a little 'sloppy' and unpredictable
# so it seems less robotic, is permissable
import random

from ros_pololu_servo.msg import *
from std_msgs.msg import String
from trajectory_msgs.msg import JointTrajectoryPoint

class ZenoArms:

    # NOTE ON IMPLEMENTATION
    # -------------------------------------
    # I use 'position' values are mapped from [-1.0, 1.0] values inclusive. These assume that there exists a function
    # that will use more specific measurements to actually call position.append
    #
    # Velocities will be calculated after the pts are computed, based on the delay from point to point
    #
    # which will translate these [-1.0, 1.0] values to proper radian values for use by the program. Some of these
    # positions may be inverted. arm_map
    # will take this into account and use a -1.0 directional value in order to swap directions, if necessary.
    #
    # This is being done because we do not yet have real config files.

    # NOTE ON JOINT ORIENTATION
    # I am using a human sense of joint orientation to make animating easier (ie: -1 means the same thing for each elbow
    # In reality is more than likely that joints on opposite sides of the body are flipped/mirrored, so -1 on one side
    # will be 1 on the other.
    #
    # Assumptions:
    #       Hinge (Pitch) Joints are max at fully clenched and min at fully extended
    #       Wheel (Rotation) Joints are max when:
    #           - For the Shoulders: When the arms are lifted above the head
    #           - For the Elbows: Assume the arms are slack. max will rotate the hard tip of the elbow inward.
    #           - For the Wrists: Same orientation as the elbows. Palms will turn out/up.
    #           - For the Waist: Right shoulder is forward, left shoulder is back
    #
    # Corrections:
    #       Corrections should be easy to make using the adjustDir array, featured below. This will quickly flip the
    #       radians values.
    #       We are assuming from config.xlsx that 'outward' rotates elbows in, and palms up/out
    #       and 'up' for the shoulder pitch means extended, and 'in' for the elbow means clenched
    #--------------------------------------

    # Will be filled with trajectories by the function's like 'define_wave'
    trajAnims = {}

    #NEED TO BE HAND-SET
    #---------------------
    #the names I will be using for the motors
    names = ['arm_RSR', 'arm_RSP', 'arm_RER', 'arm_REP', 'arm_RHR', 'arm_RHP',
             'arm_LSR', 'arm_LSP', 'arm_LER', 'arm_LEP', 'arm_LHR', 'arm_LHP',
             'waist_R']

    # Entered correct radian bounds for each of the above-listed motors
    minRads = [-0.8727, -0.6545, -0.7854, -0.523, -0.7854, -0.2618,
               -0.8727, -0.6545, -0.7854, -0.523, -0.7854, -0.2618,
               -0.8727]

    maxRads = [0.8727, 0.6545, 0.7854, 0.523, 0.7854, 0.2618,
               0.8727, 0.6545, 0.7854, 0.523, 0.7854, 0.2618,
               0.8727]

    # NEED TO DO: make sure this is calibrated
    # for quick directional adjustments outside of the config file, if necessary
    adjustDir = [-1.0, 1.0, -1.0, -1.0, -1.0, -1.0,
                 1.0, -1.0, 1.0, 1.0, 1.0, 1.0,
                 1.0]

    motorIds = [6, 0, 1, 2, 3, 4,
                13, 12, 14, 15, 16, 17,
                8]

    minPulses = [992, 992, 996.75, 1398, 992, 992,
                 992, 992, 992, 1136, 996, 992,
                 1016.25]

    maxPulses = [1990, 1745.5, 2000, 1931.25, 2000, 1706.25,
                 1600, 2000, 2000, 1792, 2000, 1701.5,
                 1907]

    # COMPUTED WHILE RUNNING
    # ------------------------------
    # this is overwritten in init
    # it will be multiplied against the [-1.0, 1.0 range to get the value in radians that positions needs to be passed
    rangeRads = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                 0.0]

    rangePulses = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                   0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                   0.0]

    # this is overwritten in init
    # it will be added to the radians computed with rangeRads if any offset is needed (if abs(minRads) != abs(maxRads)
    offsetRads = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                  0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                  0.0]

    # for quick programming purposes, I may often want most of the motors NOT to use. This variable stores motor
    # positions so I can retrieve them later
    storePos = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                0.0]

    #a constant: assess, do I want to use the storedPosition for this motor? If so, I will pass in storeConst instead of a
    #[-1.0, 1.0] value.
    storeConst = 12.0

    # initialize many of the values we may require in order to correctly transform [-1.0, 1.0] range values
    # into the proper radian ranges, and to compute velocities as well

    def __init__(self):

        #initialize ros node
        rospy.init_node('ZenoArms')

       #iterate through all of the motors
        for i in range(0, len(self.names)):

            #NEED TO DO: Any additional initializations using proper config files, motor numbers, etc?

            print("minRad: " + self.minRads[i])
            print("maxRad: " + self.maxRads[i])

            #calculate the offset (if for some strange reason abs(maxrads) and abs(minRads) are not equal,
            ## I want to account for it
            self.offsetRads[i] = self.maxRads[i] - self.minRads[i]

            #calculate the 'range' which I will later use
            self.rangeRads[i] = (self.maxRads[i] + self.minRads[i]) / 2.0

            #calculate the range for the pulses as well, in case we need them.
            #this should allow us to cast [-1.0, 1.0] values to pulses more easily
            self.rangePulses[i] = self.maxPulses[i] - self.minPulses[i]

        #call definition functions
        self.define_wave()

       #more ros setup
        self.client = actionlib.SimpleActionClient('pololu_trajectory_action_server', pololu_trajectoryAction)
        self.client.wait_for_server()

        #subscribe to listen on the topic of when zeno needs animations for his arms
        rospy.Subscriber("zeno_popolu_animation", String, self.callback)

    def callback(self, msg):
        print ("ZenoArms.callback ran")
        if msg.data in self.trajAnims:
            print(" and found trajAnims correctly.")
            goal = pololu_trajectoryGoal()
            goal.joint_trajectory = self.trajAnims[msg.data]
            self.client.send_goal(goal)
            self.client.wait_for_result()

    # takes in itself, the point list, and the positions for each of len(self.names) amount of motors
    def map_motors(self, pts, positions):

        # iterate through all of the motors
        for i in range(0, len(positions)):

            # see if the position is carrying storeConst, which means to use whatever position was stored
            # for the motor last. (the "-" & "<" are used instead of == because these are floats and shouldn't
            # be compared directly)
            if positions[i] - self.storeConst < 0.01:
                pts.positions.append(self.storePos[i])

            # otherwise flip the direction of the [-1.0, 1.0] value if necessary, transform it into the radians
            # range, and offset it by the correct amount
            else:
                pts.positions.append(self.rangeRads * positions[i] * self.adjustDir[i] + self.offsetRads[i])

                # do not update the storePos array UNLESS we actually have a pos (heavens forbid we throw in a value
                # like "12.0"...
                self.storePos[i] = positions[i]

            #NEED TO DO: This is a placeholder; actual velocities should be computed later
            pts.velocities.append(1.0)

    def new_points(self, traj, positions, begin_point):
        # initialize the point
        pts = JointTrajectoryPoint()

        #Adjust as necessary. A little delay should keep strange abrupt jerks from occurring
        pts.time_from_start = rospy.Duration(begin_point)

        # map_motors will do all our transformations from the [-1.0, 1.0] system to what the motors actually need,
        # and append position values to pts
        self.map_motors(pts, positions)

        # we are appending the points to the overall trajectory variable right now. Later, an array will go back
        # and give the points proper velocities.
        traj.points.append(pts)

    def define_wave(self):

        #initialize the joint trajectory
        traj = JointTrajectory()

        #really quickly add all of the motor names
        for i in range(0, len(self.names)):
            traj.joint_names.append(self.names[i])

        #another lovely helping variable that allows us to pass in the storeConst with the word 'same' when we want
        #positions to stay the same
        same = self.storeConst

        #this variable will help us keep track of how much time has elapsed, so we don't accidentlaly put one trajectory
        #behind another
        time_when = 0.0

        # NEUTRAL POSE
        # ----------------------
        # Zeno should 'gently' assume a neutral position to wave from. This posture should have the arms dropped
        # at the side with the hands pointed towards the thigh.

        # Problems are likely to be found in the Shoulder Roll Joints (arm_RSR and arm_LSR) which ought to be at
        # something like 0.5 or -0.5, and the hands, which do not appear to be attached so that 0.0 really lines
        # up with anything...

        # This helper variable should be easy to edit. It lists the motor positions, [-1.0, 1.0] and exists to
        # conveniently pass them into map_motors for each new position. arm_poses will pass in storeConst from now on
        # if it wants to use the last position array.
        arm_poses = [0.5, 1.0, 0.0, -1.0, 0.5, 0.0,
                     0.5, 1.0, 0.0, -1.0, 0.5, 0.0,
                     0.0]

        time_when += 0.5

        # and now of course call the function
        self.new_points(traj, arm_poses, time_when)

        #here we decide whether to wave 2 to 4 times
        sloppy_wave_count = random.randrange(0, 2) + 2
        for j in range(0, sloppy_wave_count):

            #this should make the wave a little uneven and humanish
            sloppy_values = []
            for k in range(0, 4):
                sloppy_values.append(random.uniform(0.0, 0.1) - 0.05)

            # RAISE LEFT ARM
            # The only motors moving should be the shoulder wheel, the elbow, the wrist, the hand, and the waist
            # Zeno raises the shoulder to max, clenches the elbow to max, rotates the wrist to face the hand out (min),
            # extends the fingers to the best of his ability (min), and attempts to rotate the waist so the left shoulder face forward
            #-------------------------------
            arm_poses = [same, same, same, same, same,
                         1.0, 1.0, same, 1.0, -0.75, -1.0,
                         -0.25]

            time_when += 0.5 + sloppy_values[0]

            self.new_points(traj, arm_poses, time_when)

            # EXTEND ARM THROUGH WAVE
            # Zeno waves by unclenching his shoulder hinge (min) and rotating his wrist to try and keep the hand
            # facing forward

            #helper variable for a sloppily extended shoulder hinge (s_piv) and wrist rotation (s_wri) and elbow
            #hinge (s_elb)
            s_piv = sloppy_values[2] - 0.8
            s_wri = sloppy_values[2] - 0.25
            s_elb = sloppy_values[3] * 2 - 0.5

            arm_poses = [same, same, same, same, same,
                         same, s_piv , same, s_elb, s_wri, same,
                         -s_wri]

            time_when += 0.5 + sloppy_values[1]

            self.new_points(traj, arm_poses, time_when)

        # ASSUME NEUTRAL POSE (This will draw the arm back from the extended pose, not the retracted pose)
        arm_poses = [0.5, 1.0, 0.0, -1.0, 0.5, 0.0,
                     0.5, 1.0, 0.0, -1.0, 0.5, 0.0,
                     0.0]

        time_when += 0.75

        self.new_points(traj, arm_poses, time_when)

        # And now that we are done, add it to our animations
        self.trajAnims['zeno_wave'] = traj

if __name__ == '__main__':
    arms=ZenoArms()
    rospy.spin()
<commit_msg>Changed some of the motor values to reflect the config file we just got, and also commented out some materials we no longer need<commit_after>#!/usr/bin/env python
import rospy
import actionlib

# NEED TO DO: I want to make sure that this random, which is used to make the wave a little 'sloppy' and unpredictable
# so it seems less robotic, is permissable
import random

from ros_pololu_servo.msg import *
from std_msgs.msg import String
from trajectory_msgs.msg import JointTrajectoryPoint

class ZenoArms:

    # NOTE ON IMPLEMENTATION
    # -------------------------------------
    # I use 'position' values are mapped from [-1.0, 1.0] values inclusive. These assume that there exists a function
    # that will use more specific measurements to actually call position.append
    #
    # Velocities will be calculated after the pts are computed, based on the delay from point to point
    #
    # which will translate these [-1.0, 1.0] values to proper radian values for use by the program. Some of these
    # positions may be inverted. arm_map
    # will take this into account and use a -1.0 directional value in order to swap directions, if necessary.
    #
    # This is being done because we do not yet have real config files.

    # NOTE ON JOINT ORIENTATION
    # I am using a human sense of joint orientation to make animating easier (ie: -1 means the same thing for each elbow
    # In reality is more than likely that joints on opposite sides of the body are flipped/mirrored, so -1 on one side
    # will be 1 on the other.
    #
    # Assumptions:
    #       Hinge (Pitch) Joints are max at fully clenched and min at fully extended
    #       Wheel (Rotation) Joints are max when:
    #           - For the Shoulders: When the arms are lifted above the head
    #           - For the Elbows: Assume the arms are slack. max will rotate the hard tip of the elbow inward.
    #           - For the Wrists: Same orientation as the elbows. Palms will turn out/up.
    #           - For the Waist: Right shoulder is forward, left shoulder is back
    #
    # Corrections:
    #       Corrections should be easy to make using the adjustDir array, featured below. This will quickly flip the
    #       radians values.
    #       We are assuming from config.xlsx that 'outward' rotates elbows in, and palms up/out
    #       and 'up' for the shoulder pitch means extended, and 'in' for the elbow means clenched
    #--------------------------------------

    # Will be filled with trajectories by the function's like 'define_wave'
    trajAnims = {}

    #NEED TO BE HAND-SET
    #---------------------
    #the names I will be using for the motors, pre-config file delivery
    #names = ['arm_RSR', 'arm_RSP', 'arm_RER', 'arm_REP', 'arm_RHR', 'arm_RHP',
    #         'arm_LSR', 'arm_LSP', 'arm_LER', 'arm_LEP', 'arm_LHR', 'arm_LHP',
    #         'waist_R']

    #the names I am now using, relative to the config file I was given
    names = ['r_shoulder_roll_joint', 'r_shoulder_pitch_joint', 'r_elbow_roll_joint', 'r_elbow_pitch_joint', 'r_wrist_roll_joint', 'r_hand_grasp',
             'l_shoulder_roll_joint', 'l_shoulder_pitch_joint', 'l_elbow_roll_joint', 'l_elbow_pitch_joint', 'l_wrist_roll_joint', 'l_hand_grasp',
             'torso_joint']

    # Entered correct radian bounds for each of the above-listed motors
    minRads = [-0.872, -0.654, -0.785, -0.523, -0.785, -0.261,
               -0.872, -0.654, -0.785, -0.523, -0.785, -0.261,
               -0.872]

    maxRads = [0.872, 0.654, 0.785, 0.523, 0.785, 0.261,
               0.872, 0.654, 0.785, 0.523, 0.785, 0.261,
               0.872]

    # NEED TO DO: make sure this is calibrated
    # for quick directional adjustments outside of the config file, if necessary
    adjustDir = [-1.0, 1.0, -1.0, -1.0, -1.0, -1.0,
                 1.0, -1.0, 1.0, 1.0, 1.0, 1.0,
                 1.0]

    #These are in the config file and we currently do not need them
    # motorIds = [6, 0, 1, 2, 3, 4,
    #             13, 12, 14, 15, 16, 17,
    #             8]
    #
    # minPulses = [992, 992, 996.75, 1398, 992, 992,
    #              992, 992, 992, 1136, 996, 992,
    #              1016.25]
    #
    # maxPulses = [1990, 1745.5, 2000, 1931.25, 2000, 1706.25,
    #              1600, 2000, 2000, 1792, 2000, 1701.5,
    #              1907]

    # COMPUTED WHILE RUNNING
    # ------------------------------
    # this is overwritten in init
    # it will be multiplied against the [-1.0, 1.0 range to get the value in radians that positions needs to be passed
    rangeRads = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                 0.0]

    rangePulses = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                   0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                   0.0]

    # this is overwritten in init
    # it will be added to the radians computed with rangeRads if any offset is needed (if abs(minRads) != abs(maxRads)
    offsetRads = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                  0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                  0.0]

    # for quick programming purposes, I may often want most of the motors NOT to use. This variable stores motor
    # positions so I can retrieve them later
    storePos = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                0.0]

    #a constant: assess, do I want to use the storedPosition for this motor? If so, I will pass in storeConst instead of a
    #[-1.0, 1.0] value.
    storeConst = 12.0

    # initialize many of the values we may require in order to correctly transform [-1.0, 1.0] range values
    # into the proper radian ranges, and to compute velocities as well

    def __init__(self):

        #initialize ros node
        rospy.init_node('ZenoArms')

       #iterate through all of the motors
        for i in range(0, len(self.names)):

            #NEED TO DO: Any additional initializations using proper config files, motor numbers, etc?

            print("minRad: " + self.minRads[i])
            print("maxRad: " + self.maxRads[i])

            #calculate the offset (if for some strange reason abs(maxrads) and abs(minRads) are not equal,
            ## I want to account for it
            self.offsetRads[i] = self.maxRads[i] - self.minRads[i]

            #calculate the 'range' which I will later use
            self.rangeRads[i] = (self.maxRads[i] + self.minRads[i]) / 2.0

            #calculate the range for the pulses as well, in case we need them.
            #this should allow us to cast [-1.0, 1.0] values to pulses more easily
            self.rangePulses[i] = self.maxPulses[i] - self.minPulses[i]

        #call definition functions
        self.define_wave()

       #more ros setup
        self.client = actionlib.SimpleActionClient('pololu_trajectory_action_server', pololu_trajectoryAction)
        self.client.wait_for_server()

        #subscribe to listen on the topic of when zeno needs animations for his arms
        rospy.Subscriber("zeno_popolu_animation", String, self.callback)

    def callback(self, msg):
        print ("ZenoArms.callback ran")
        if msg.data in self.trajAnims:
            print(" and found trajAnims correctly.")
            goal = pololu_trajectoryGoal()
            goal.joint_trajectory = self.trajAnims[msg.data]
            self.client.send_goal(goal)
            self.client.wait_for_result()

    # takes in itself, the point list, and the positions for each of len(self.names) amount of motors
    def map_motors(self, pts, positions):

        # iterate through all of the motors
        for i in range(0, len(positions)):

            # see if the position is carrying storeConst, which means to use whatever position was stored
            # for the motor last. (the "-" & "<" are used instead of == because these are floats and shouldn't
            # be compared directly)
            if (positions[i] - self.storeConst < 0.01) and (self.storeConst - positions[i] < 0.01):
                pts.positions.append(self.storePos[i])

            # otherwise flip the direction of the [-1.0, 1.0] value if necessary, transform it into the radians
            # range, and offset it by the correct amount
            else:
                pts.positions.append(self.rangeRads * positions[i] * self.adjustDir[i] + self.offsetRads[i])

                # do not update the storePos array UNLESS we actually have a pos (heavens forbid we throw in a value
                # like "12.0"...
                self.storePos[i] = positions[i]

            #NEED TO DO: This is a placeholder; actual velocities should be computed later
            pts.velocities.append(1.0)

    def new_points(self, traj, positions, begin_point):
        # initialize the point
        pts = JointTrajectoryPoint()

        #Adjust as necessary. A little delay should keep strange abrupt jerks from occurring
        pts.time_from_start = rospy.Duration(begin_point)

        # map_motors will do all our transformations from the [-1.0, 1.0] system to what the motors actually need,
        # and append position values to pts
        self.map_motors(pts, positions)

        # we are appending the points to the overall trajectory variable right now. Later, an array will go back
        # and give the points proper velocities.
        traj.points.append(pts)

    def define_wave(self):

        #initialize the joint trajectory
        traj = JointTrajectory()

        #really quickly add all of the motor names
        for i in range(0, len(self.names)):
            traj.joint_names.append(self.names[i])

        #another lovely helping variable that allows us to pass in the storeConst with the word 'same' when we want
        #positions to stay the same
        same = self.storeConst

        #this variable will help us keep track of how much time has elapsed, so we don't accidentlaly put one trajectory
        #behind another
        time_when = 0.0

        # NEUTRAL POSE
        # ----------------------
        # Zeno should 'gently' assume a neutral position to wave from. This posture should have the arms dropped
        # at the side with the hands pointed towards the thigh.

        # Problems are likely to be found in the Shoulder Roll Joints (arm_RSR and arm_LSR) which ought to be at
        # something like 0.5 or -0.5, and the hands, which do not appear to be attached so that 0.0 really lines
        # up with anything...

        # This helper variable should be easy to edit. It lists the motor positions, [-1.0, 1.0] and exists to
        # conveniently pass them into map_motors for each new position. arm_poses will pass in storeConst from now on
        # if it wants to use the last position array.
        arm_poses = [0.5, 1.0, 0.0, -1.0, 0.5, 0.0,
                     0.5, 1.0, 0.0, -1.0, 0.5, 0.0,
                     0.0]

        time_when += 0.5

        # and now of course call the function
        self.new_points(traj, arm_poses, time_when)

        #here we decide whether to wave 2 to 4 times
        sloppy_wave_count = random.randrange(0, 2) + 2
        for j in range(0, sloppy_wave_count):

            #this should make the wave a little uneven and humanish
            sloppy_values = []
            for k in range(0, 4):
                sloppy_values.append(random.uniform(0.0, 0.1) - 0.05)

            # RAISE LEFT ARM
            # The only motors moving should be the shoulder wheel, the elbow, the wrist, the hand, and the waist
            # Zeno raises the shoulder to max, clenches the elbow to max, rotates the wrist to face the hand out (min),
            # extends the fingers to the best of his ability (min), and attempts to rotate the waist so the left shoulder face forward
            #-------------------------------
            arm_poses = [same, same, same, same, same,
                         1.0, 1.0, same, 1.0, -0.75, -1.0,
                         -0.25]

            time_when += 0.5 + sloppy_values[0]

            self.new_points(traj, arm_poses, time_when)

            # EXTEND ARM THROUGH WAVE
            # Zeno waves by unclenching his shoulder hinge (min) and rotating his wrist to try and keep the hand
            # facing forward

            #helper variable for a sloppily extended shoulder hinge (s_piv) and wrist rotation (s_wri) and elbow
            #hinge (s_elb)
            s_piv = sloppy_values[2] - 0.8
            s_wri = sloppy_values[2] - 0.25
            s_elb = sloppy_values[3] * 2 - 0.5

            arm_poses = [same, same, same, same, same,
                         same, s_piv , same, s_elb, s_wri, same,
                         -s_wri]

            time_when += 0.5 + sloppy_values[1]

            self.new_points(traj, arm_poses, time_when)

        # ASSUME NEUTRAL POSE (This will draw the arm back from the extended pose, not the retracted pose)
        arm_poses = [0.5, 1.0, 0.0, -1.0, 0.5, 0.0,
                     0.5, 1.0, 0.0, -1.0, 0.5, 0.0,
                     0.0]

        time_when += 0.75

        self.new_points(traj, arm_poses, time_when)

        # And now that we are done, add it to our animations
        self.trajAnims['zeno_wave'] = traj

if __name__ == '__main__':
    arms=ZenoArms()
    rospy.spin()
<|endoftext|>


<commit_before>"""

Utilities for sorting query entries.

"""

import numpy as np
import sklearn.utils

def get_sorted_y_positions(y, y_pred, check=True):
    if check:
        y = sklearn.utils.validation.column_or_1d(y)
        y_pred = sklearn.utils.validation.column_or_1d(y_pred)
        sklearn.utils.validation.check_consistent_length(y, y_pred)
    return np.lexsort((y, -y_pred))

def get_sorted_y(y, y_pred, check=True):
    """Returns a copy of `y` sorted by position in `y_pred`.

    Parameters
    ----------
    y : array_like of shape = [n_samples_in_query]
        List of sample scores for a query.
    y_pred : array_like of shape = [n_samples_in_query]
        List of predicted scores for a query.

    Returns
    -------
    y_sorted : array_like of shape = [n_samples_in_query]
        Copy of `y` sorted by descending order of `y_pred`.
        Ties are broken in ascending order of `y`.

    """
    if check:
        y = sklearn.utils.validation.column_or_1d(y)
        y_pred = sklearn.utils.validation.column_or_1d(y_pred)
        sklearn.utils.validation.check_consistent_length(y, y_pred)
    return y[get_sorted_y_positions(y, y_pred, check=False)]
<commit_msg>Remove code duplication<commit_after>"""

Utilities for sorting query entries.

"""

import numpy as np
import sklearn.utils

def get_sorted_y_positions(y, y_pred, check=True):
    if check:
        y = sklearn.utils.validation.column_or_1d(y)
        y_pred = sklearn.utils.validation.column_or_1d(y_pred)
        sklearn.utils.validation.check_consistent_length(y, y_pred)
    return np.lexsort((y, -y_pred))

def get_sorted_y(y, y_pred, check=True):
    """Returns a copy of `y` sorted by position in `y_pred`.

    Parameters
    ----------
    y : array_like of shape = [n_samples_in_query]
        List of sample scores for a query.
    y_pred : array_like of shape = [n_samples_in_query]
        List of predicted scores for a query.

    Returns
    -------
    y_sorted : array_like of shape = [n_samples_in_query]
        Copy of `y` sorted by descending order of `y_pred`.
        Ties are broken in ascending order of `y`.

    """
    return y[get_sorted_y_positions(y, y_pred, check=check)]
<|endoftext|>


<commit_before># Copyright 2015 Yahoo.com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import collections
import contextlib
import sys
import threading

import six

class ObjectPool(object):
    """A pool of objects that release/creates/destroys as needed."""

    def __init__(self, obj_creator,
                 after_remove=None, max_size=None,
                 ):
        self._used_objs = collections.deque()
        self._free_objs = collections.deque()
        self._obj_creator = obj_creator
        self._lock = threading.Lock()
        self._after_remove = after_remove
        max_size = max_size or 2 ** 31
        if not isinstance(max_size, six.integer_types) or max_size < 0:
            raise ValueError('"max_size" must be a positive integer')
        self.max_size = max_size

    @property
    def used(self):
        return tuple(self._used_objs)

    @property
    def free(self):
        return tuple(self._free_objs)

    @contextlib.contextmanager
    def get_and_release(self, destroy_on_fail=False):
        obj = self.get()
        try:
            yield obj
        except Exception:
            exc_info = sys.exc_info()
            if not destroy_on_fail:
                self.release(obj)
            else:
                self.destroy(obj)
            six.reraise(exc_info[0], exc_info[1], exc_info[2])
        self.release(obj)

    def get(self):
        with self._lock:
            if not self._free_objs:
                curr_count = len(self._used_objs)
                if curr_count >= self.max_size:
                    raise RuntimeError("Too many objects,"
                                       " %s >= %s" % (curr_count,
                                                      self.max_size))
                obj = self._obj_creator()
                self._used_objs.append(obj)
                return obj
            else:
                obj = self._free_objs.pop()
                self._used_objs.append(obj)
                return obj

    def destroy(self, obj, silent=True):
        with self._lock:
            try:
                self._used_objs.remove(obj)
                if self._after_remove is not None:
                    self._after_remove(obj)
            except ValueError:
                if not silent:
                    raise

    def release(self, obj, silent=True):
        with self._lock:
            try:
                self._used_objs.remove(obj)
                self._free_objs.append(obj)
            except ValueError:
                if not silent:
                    raise

    def clear(self):
        with self._lock:
            if self._after_remove is not None:
                while self._used_objs:
                    self._after_remove(self._used_objs.pop())
                while self._free_objs:
                    self._after_remove(self._free_objs.pop())
            else:
                self._free_objs.clear()
                self._used_objs.clear()
<commit_msg>Fix/remove extra newline<commit_after># Copyright 2015 Yahoo.com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import collections
import contextlib
import sys
import threading

import six

class ObjectPool(object):
    """A pool of objects that release/creates/destroys as needed."""

    def __init__(self, obj_creator,
                 after_remove=None, max_size=None):
        self._used_objs = collections.deque()
        self._free_objs = collections.deque()
        self._obj_creator = obj_creator
        self._lock = threading.Lock()
        self._after_remove = after_remove
        max_size = max_size or 2 ** 31
        if not isinstance(max_size, six.integer_types) or max_size < 0:
            raise ValueError('"max_size" must be a positive integer')
        self.max_size = max_size

    @property
    def used(self):
        return tuple(self._used_objs)

    @property
    def free(self):
        return tuple(self._free_objs)

    @contextlib.contextmanager
    def get_and_release(self, destroy_on_fail=False):
        obj = self.get()
        try:
            yield obj
        except Exception:
            exc_info = sys.exc_info()
            if not destroy_on_fail:
                self.release(obj)
            else:
                self.destroy(obj)
            six.reraise(exc_info[0], exc_info[1], exc_info[2])
        self.release(obj)

    def get(self):
        with self._lock:
            if not self._free_objs:
                curr_count = len(self._used_objs)
                if curr_count >= self.max_size:
                    raise RuntimeError("Too many objects,"
                                       " %s >= %s" % (curr_count,
                                                      self.max_size))
                obj = self._obj_creator()
                self._used_objs.append(obj)
                return obj
            else:
                obj = self._free_objs.pop()
                self._used_objs.append(obj)
                return obj

    def destroy(self, obj, silent=True):
        with self._lock:
            try:
                self._used_objs.remove(obj)
                if self._after_remove is not None:
                    self._after_remove(obj)
            except ValueError:
                if not silent:
                    raise

    def release(self, obj, silent=True):
        with self._lock:
            try:
                self._used_objs.remove(obj)
                self._free_objs.append(obj)
            except ValueError:
                if not silent:
                    raise

    def clear(self):
        with self._lock:
            if self._after_remove is not None:
                while self._used_objs:
                    self._after_remove(self._used_objs.pop())
                while self._free_objs:
                    self._after_remove(self._free_objs.pop())
            else:
                self._free_objs.clear()
                self._used_objs.clear()
<|endoftext|>


<commit_before>"""__init__ file for Pynuts."""

import os
import flask
from werkzeug.utils import cached_property
from flask_sqlalchemy import SQLAlchemy
from flask.ext.uploads import configure_uploads, patch_request_class
from dulwich.repo import Repo

from . import document, rights, view
from .environment import create_environment

class Pynuts(flask.Flask):
    """Create the Pynuts class, inheriting from flask.Flask

    :param import_name: Flask application name
    :type import_name: str
    :param config: Flask application configuration
    :type config: dict
    :param config_file: path of the application configuration file
    :type config_file: str
    :param reflect: Create models with database data
    :type reflect: bool
    :param args: flask.Flask args
    :param kwargs: flask.Flask kwargs

    .. seealso::
      `Flask Application <http://flask.pocoo.org/docs/api/>`_

    """
    def __init__(self, import_name, config=None, config_file=None,
                 reflect=False, *args, **kwargs):
        super(Pynuts, self).__init__(import_name, *args, **kwargs)

        self.config['CSRF_ENABLED'] = False
        if config_file:
            # generate app config from file
            self.config.from_pyfile(config_file)
        if config:
            self.config.update(config)  # generate app config from dict

        self.db = SQLAlchemy(self)  # bind the SQLAlchemy controller to the Pynuts app
        self.documents = {}
        self.views = {}

        if reflect:
            # Automatically create models from the database existent data
            self.db.metadata.reflect(bind=self.db.get_engine(self))

        # Set the document repository path
        self.document_repository_path = (
            os.path.join(
                self.instance_path,
                self.config.get('PYNUTS_DOCUMENT_REPOSITORY')
                    or 'documents.git')
            )

        # If self.document_repository_path does not exist,
        # create it (and possible parent folders) and initialize the repo
        if not os.path.exists(self.document_repository_path):
            os.makedirs(self.document_repository_path)
            Repo.init_bare(self.document_repository_path)

        # Serve files from the Pynuts static folder
        # at the /_pynuts/static/<path:filename> URL
        self.add_url_rule('/_pynuts/static/<path:filename>',
                          '_pynuts-static', static)

        # Uploads root directory
        self.config['UPLOADS_DEFAULT_DEST'] = os.path.join(
            self.instance_path, 'uploads')

        class Document(document.Document):
            """Document base class of the application."""
            _pynuts = self

        self.Document = Document

        class Context(object):
            """Context base class of the application.

            You can get or set any element in the context stored in
            the `g` flask object.

            Example : Set the current time of the request in the context, using
            datetime :

            @app.before_request
            def set_request_time():
                g.context.request_time = datetime.now().strftime('%Y/%m/%d')

            """
            __metaclass__ = rights.MetaContext
            _pynuts = self

            def __getitem__(self, key):
                return getattr(self, key)

            def __setitem__(self, key, value):
                setattr(self, key, value)

            def get(self, key, default=None):
                return getattr(self, key, default)

        self.Context = Context

        class ModelView(view.ModelView):
            """Model view base class of the application."""
            _pynuts = self
            # Create a new Jinja2 environment with Pynuts helpers
            environment = create_environment(_pynuts.jinja_env.loader)

        self.ModelView = ModelView

        self.before_request(self.create_context)

    @cached_property
    def document_repository(self):
        """Return the path to the document repository."""
        return Repo(self.document_repository_path)

    def render_rest(self, document_type, part='index.rst.jinja2',
                    **kwargs):
        """Return the generated ReST version of the document."""
        return self.documents[document_type].generate_rest(part, **kwargs)

    def create_context(self):
        """Create the request context."""
        flask.g.context = self._context_class()

    def add_upload_sets(self, upload_sets):
        """Configure the app with the argument upload sets."""
        configure_uploads(self, upload_sets)
        patch_request_class(self)  # limit the size of uploads to 16MB

    @property
    def uploads_default_dest(self):
        """Access to the UPLOADS_DEFAULT_DEST configuration."""
        return self.config.get('UPLOADS_DEFAULT_DEST')

def static(filename):
    """ Return files from Pynuts static folder.

    :param filename: the basename of the file contained in Pynuts static folder
    """
    return flask.send_from_directory(
        os.path.join(os.path.dirname(__file__), 'static'), filename)
<commit_msg>Bugfix: the maximum upload size as app config Signed-off-by: Balthazar Rouberol <balthazar.rouberol@kozea.fr><commit_after>"""__init__ file for Pynuts."""

import os
import flask
from werkzeug.utils import cached_property
from flask_sqlalchemy import SQLAlchemy
from flask.ext.uploads import configure_uploads, patch_request_class
from dulwich.repo import Repo

from . import document, rights, view
from .environment import create_environment

class Pynuts(flask.Flask):
    """Create the Pynuts class, inheriting from flask.Flask

    :param import_name: Flask application name
    :type import_name: str
    :param config: Flask application configuration
    :type config: dict
    :param config_file: path of the application configuration file
    :type config_file: str
    :param reflect: Create models with database data
    :type reflect: bool
    :param args: flask.Flask args
    :param kwargs: flask.Flask kwargs

    .. seealso::
      `Flask Application <http://flask.pocoo.org/docs/api/>`_

    """
    def __init__(self, import_name, config=None, config_file=None,
                 reflect=False, *args, **kwargs):
        super(Pynuts, self).__init__(import_name, *args, **kwargs)

        self.config['CSRF_ENABLED'] = False
        if config_file:
            # generate app config from file
            self.config.from_pyfile(config_file)
        if config:
            self.config.update(config)  # generate app config from dict

        self.db = SQLAlchemy(self)  # bind the SQLAlchemy controller to the Pynuts app
        self.documents = {}
        self.views = {}

        if reflect:
            # Automatically create models from the database existent data
            self.db.metadata.reflect(bind=self.db.get_engine(self))

        # Set the document repository path
        self.document_repository_path = (
            os.path.join(
                self.instance_path,
                self.config.get('PYNUTS_DOCUMENT_REPOSITORY')
                    or 'documents.git')
            )

        # If self.document_repository_path does not exist,
        # create it (and possible parent folders) and initialize the repo
        if not os.path.exists(self.document_repository_path):
            os.makedirs(self.document_repository_path)
            Repo.init_bare(self.document_repository_path)

        # Serve files from the Pynuts static folder
        # at the /_pynuts/static/<path:filename> URL
        self.add_url_rule('/_pynuts/static/<path:filename>',
                          '_pynuts-static', static)

        # Uploads root directory
        self.config['UPLOADS_DEFAULT_DEST'] = os.path.join(
            self.instance_path, 'uploads')

        class Document(document.Document):
            """Document base class of the application."""
            _pynuts = self

        self.Document = Document

        class Context(object):
            """Context base class of the application.

            You can get or set any element in the context stored in
            the `g` flask object.

            Example : Set the current time of the request in the context, using
            datetime :

            @app.before_request
            def set_request_time():
                g.context.request_time = datetime.now().strftime('%Y/%m/%d')

            """
            __metaclass__ = rights.MetaContext
            _pynuts = self

            def __getitem__(self, key):
                return getattr(self, key)

            def __setitem__(self, key, value):
                setattr(self, key, value)

            def get(self, key, default=None):
                return getattr(self, key, default)

        self.Context = Context

        class ModelView(view.ModelView):
            """Model view base class of the application."""
            _pynuts = self
            # Create a new Jinja2 environment with Pynuts helpers
            environment = create_environment(_pynuts.jinja_env.loader)

        self.ModelView = ModelView

        self.before_request(self.create_context)

    @cached_property
    def document_repository(self):
        """Return the path to the document repository."""
        return Repo(self.document_repository_path)

    def render_rest(self, document_type, part='index.rst.jinja2',
                    **kwargs):
        """Return the generated ReST version of the document."""
        return self.documents[document_type].generate_rest(part, **kwargs)

    def create_context(self):
        """Create the request context."""
        flask.g.context = self._context_class()

    def add_upload_sets(self, upload_sets, upload_max_size=16777216):
        """Configure the app with the argument upload sets."""
        configure_uploads(self, upload_sets)
        patch_request_class(self, upload_max_size)  # limit the size of uploads to 16MB

    @property
    def uploads_default_dest(self):
        """Access to the UPLOADS_DEFAULT_DEST configuration."""
        return self.config.get('UPLOADS_DEFAULT_DEST')

def static(filename):
    """ Return files from Pynuts static folder.

    :param filename: the basename of the file contained in Pynuts static folder
    """
    return flask.send_from_directory(
        os.path.join(os.path.dirname(__file__), 'static'), filename)
<|endoftext|>


<commit_before>#!/usr/bin/env python

__author__ = 'Adam R. Smith,Luke'

import gevent
import os
import fcntl
from gevent.event import Event
from collections import Iterable
from functools import wraps
import time

spawn = gevent.spawn

def spawnf(f):
    """ Decorator to spawn this function in a greenlet. """
    @wraps(f)
    def wrapper(*args, **kwargs):
        return gevent.spawn(f, *args, **kwargs)
    return wrapper

def asyncf(f):
    """ Decorator to spawn this function in a greenlet and return the result inline. """
    @wraps(f)
    def wrapper(*args, **kwargs):
        g = gevent.spawn(f, *args, **kwargs)
        return g.get()
    return wrapper

def switch():
    """ Shortcut to give control from the current greenlet back to the gevent hub. """
    gevent.getcurrent().switch()

def join(green_stuff):
    """ Universal way to join on either a single greenlet or a list of them. """
    if isinstance(green_stuff, Iterable):
        return gevent.joinall(green_stuff)
    return gevent.join(green_stuff)

def wait(green_stuff):
    """ Universal way to join on either a single greenlet or a list of them and get return value inline. """
    if isinstance(green_stuff, Iterable):
        gevent.joinall(green_stuff)
        return [g.get() for g in green_stuff]
    return green_stuff.get()

def blocking_cb(func, cb_arg, *args, **kwargs):
    """
    Wrap a function that takes a callback as a named parameter, to block and return its arguments as the result.
    Really handy for working with callback-based APIs. Do not use in really frequently-called code.
    If keyword args are supplied, they come through in a single dictionary to avoid out-of-order issues.
    """
    ev = Event()
    ret_vals = []
    def cb(*args, **kwargs):
        ret_vals.extend(args)
        if len(kwargs): ret_vals.append(kwargs)
        ev.set()
    kwargs[cb_arg] = cb
    func(*args, **kwargs)
    ev.wait(timeout=10)
    if len(ret_vals) == 0:
        return None
    elif len(ret_vals) == 1:
        return ret_vals[0]
    return tuple(ret_vals)

#--------------------------------------------------------------------------------

_pythread = None

def get_pythread():
    '''
    Loads the 'thread' module, free of monkey patching.
    '''
    global _pythread
    if _pythread:
        return _pythread # Cache the module so we don't have to use imp every time

    import imp
    fp, path, desc = imp.find_module('thread')
    try:
        _pythread = imp.load_module('pythread', fp, path, desc)
    finally:
        if fp:
            fp.close() # Close the file
    return _pythread

_pytime = None
def get_pytime():
    global _pytime
    if _pytime:
        return _pytime

    import imp
    fp, path, desc = imp.find_module('time')
    try:
        _pytime = imp.load_module('pytime', fp, path, desc)
    finally:
        if fp:
            fp.close()
    return _pytime

def nonblock_pipe():
    r, w = os.pipe()
    fcntl.fcntl(r, fcntl.F_SETFD, os.O_NONBLOCK)
    fcntl.fcntl(w, fcntl.F_SETFD, os.O_NONBLOCK)
    return r,w

_async_pipe_read, _async_pipe_write = nonblock_pipe()

def _async_pipe_read_callback(event, evtype):
    '''
    libevent callback to read from the pipe
    '''
    try:
        os.read(event.fd, 1)
    except EnvironmentError:
        pass # EAGAIN

# create a libevent callback to read from the pipe
gevent.core.event(gevent.core.EV_READ | gevent.core.EV_PERSIST,
                  _async_pipe_read,
                  _async_pipe_read_callback).add()

class AsyncResult(gevent.event.AsyncResult):

    def __init__(self):
        gevent.event.AsyncResult.__init__(self)

    def get(self, *args, **kwargs):
        return gevent.event.AsyncResult.get(self, *args, **kwargs)

    def set_exception(self, exception):
        gevent.event.AsyncResult.set_exception(self, exception)
        os.write(_async_pipe_write, '\0')

    def set(self, value):
        gevent.event.AsyncResult.set(self, value)
        os.write(_async_pipe_write, '\0')

class AsyncEvent(gevent.event.Event):
    '''
    A gevent-friendly event to be signaled by a posix thread and respected by
    the gevent context manager.

    Essentially the posix thread signals gevent through a nonblocking pipe
    '''

    def __init__(self):
        gevent.event.Event.__init__(self)
        self._r, self._w = self._pipe() # non blocking pipe

        # Create a new gevent core object that will observe the
        # nonblocking pipe file descriptor for a value
        self._core_event = gevent.core.event(
                gevent.core.EV_READ | gevent.core.EV_PERSIST,
                self._r,
                self._pipe_read)
        self._core_event.add()

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        self._close()

    def _pipe(self):
        return nonblock_pipe()

    def _pipe_read(self, event, eventtype):
        '''
        Non-blocking gevent friendly core event callback
        http://www.gevent.org/gevent.core.html#events
        '''
        try:
            os.read(event.fd, 1)
        except EnvironmentError:
            # file descriptors set with O_NONBLOCK return -1 and set errno to
            # EAGAIN, we just want to ignore it and try again later
            pass

    def set(self):
        '''
        Sets the event value and writes a value to the pipe to
        trigger the gevent core event
        '''

        gevent.event.Event.set(self)
        os.write(self._w, '\0') # Empty C-string '\0'

    def _close(self):
        '''
        Closes the core event, the read and write file descriptors
        '''
        if getattr(self, '_core_event', None):
            try:
                self._core_event.cancel()
            except:
                pass
        if getattr(self, '_r', None):
            try:
                os.close(self._r)
            except:
                pass
        if getattr(self, '_w', None):
            try:
                os.close(self._w)
            except:
                pass

class AsyncQueue(object):
    '''
    An extremely primitive queue that has a mutex over pthreads
    '''
    def __init__(self):
        self.__queue = []
        pythread = get_pythread()
        self.lock = pythread.allocate_lock()

    def put(self, item):
        with self.lock:
            self.__queue.append(item)

    def get(self):
        retval = None
        with self.lock:
            if self.__queue:
                retval = self.__queue.pop(0)
        return retval

class AsyncTask(object):
    def __init__(self, callback, *args, **kwargs):
        self.ar = AsyncResult()
        self.callback = callback
        self.args = args
        self.kwargs = kwargs

class ThreadExit(Exception):
    def __init__(self):
        self.ar = AsyncResult()

class ThreadJob(object):
    def __init__(self, queue):
        self.queue = queue

    def sleep(self, n):
        '''
        Sleeps n seconds using the un-patched time module
        '''
        pytime = get_pytime()
        pytime.sleep(n)

    def run(self):
        '''
        Runs a loop, listening for tasks
        '''
        while True:
            entry = self.queue.get()
            # If there is no task available sleep for 0.02s
            if not entry:
                self.sleep(0.02)
                continue
            # If the entry is a ThreadExit, break the loop and exit from the thread
            elif isinstance(entry, ThreadExit):
                entry.ar.set(True)
                break

            # An unrecognized task
            elif not isinstance(entry, AsyncTask):
                raise Exception("Invalid task")

            ar = entry.ar
            callback = entry.callback
            args = entry.args
            kwargs = entry.kwargs
            # If a task raises or fails then just print the stack trace and
            # continue processing
            try:
                retval = callback(*args, **kwargs)
                ar.set(retval)
            except Exception as e:
                from traceback import print_exc
                print_exc()
                # Note: the AR has an exception set so clients can observe task status
                ar.set_exception(e)

'''
Thread Pool Limitations:
    - Any thread task that imports a gevent monkey-patched module will probably
      raise an exception if gevent tries to context switch while in the child
      thread.
    - When closing threads, it is VERY prudent to synchronize the threads,
      otherwise the threads will be dangling.
    - Currently, there is no way to force a thread to quit, especially since
      most of these threads probably won't be running in the python
      interpreter. We could use the pthread library to forcefully quit a
      thread, but the memory could leak and it could cause some serious issues
      in the interpreter.
    - If a timeout is placed on synchronizing closing threads, a SystemError is
      raised, the intent with this is to fast fail a container. If threads
      aren't synchronized in time and a timeout is applied, then there is a
      bigger problem.
      - If only a subset of threads are closed, but the alive ones are still
        listening on the queue, you wind up with an inconsistent thread pool,
        and reliably executing tasks on the pool and managing what the pool
        believes to be the correct number of threads, becomes a stochastic
        system.

Advisements:
    - Keep the thread tasks as small as possible.
    - Try to limit the uses to only the code that absolutely blocks gevent.
    - If a thread blocks too long, it may be advisable to increase the threadpool temporarily.
'''

class ThreadPool(object):
    '''
    A pool of threads that can be used to run gevent-blocking code

    pool = ThreadPool(10)
    '''
    def __init__(self, poolsize=5):
        self.poolsize = poolsize
        self.queue = AsyncQueue()
        self.pythread = get_pythread()
        self.active = False
        # sets active to True
        self._spawn_threads(self.poolsize)

    def _spawn_threads(self, num):
        for i in xrange(num):
            job_worker = ThreadJob(self.queue)
            self.pythread.start_new_thread(job_worker.run, tuple())
        self.active = True

    def _check_exit(self, n, sync=False, timeout=None):
        exits = []
        for i in xrange(n):
            exit = ThreadExit()
            exits.append(exit)
            self.queue.put(exit)

        if not sync:
            return # No synchronization probably not good...

        self._inner_check(exits, timeout)

    def _inner_check(self, exits, timeout=None):
        if timeout is not None:
            t0 = time.time()
        i = len(exits)
        while i > 0:
            if timeout is not None and time.time() > (t0 + timeout):
                raise SystemError("Failed to close and synchronize threads. Thread pool is now inconsistent")
            for exit in exits:
                if exit.ar.ready():
                    i-=1

    def resize(self, newsize, sync=False, timeout=None):
        '''
        Resizes the available pool
        '''
        assert newsize > 0, "Must have a positive number of threads"
        if newsize > self.poolsize:
            n = newsize - self.poolsize
            self._spawn_threads(n)
        else:
            n = self.poolsize - newsize
            self._check_exit(n, sync, timeout)

        self.poolsize = newsize

    def apply_async(self, func, *args, **kwargs):
        '''Run func, using the given args and kwargs in the thread pool.
        Returns :class:`AsyncResult`
        '''

        assert self.active, "Thread pool is deactivated"

        task = AsyncTask(func, *args, **kwargs)
        self.queue.put(task)
        return task.ar

    def apply(self, func, *args, **kwargs):
        '''Run func, using the given args and kwargs in the thread pool,
        but block the current greenlet until the result is ready.
        Returns the value from func after execution'''

        assert self.active, "Thread pool is deactivated"

        task = AsyncTask(func, *args, **kwargs)
        self.queue.put(task)
        retval = task.ar.get()
        return retval

    def close(self, sync=False, timeout=None):
        '''
        Sends each thread an Exit exception and deactivates the thread pool.

        Note: There is no synchronization to verify that the threads each exited
        '''
        self._check_exit(self.poolsize, sync, timeout)
        self.active = False

    def map(self, func, arg_list):
        '''
        Takes a function and an iterable argument list.
        Runs func against each argument list in the iterable and returns a list
        of results, the results list will block the current greenlet in a
        gevent-safe way.
        '''
        jobs = self.map_async(func, arg_list)
        gevent.event.waitall(jobs)
        return jobs

    def map_async(self, func, arg_list):
        '''
        Takes a function and an iterable argument list.
        Runs func using each argument list in the iterable and returns a list
        of AsyncResult objects.
        '''
        async_jobs = [self.apply_async(func, *i) for i in arg_list]
        return async_jobs

class AsyncDispatcher(object):
    '''
    Used to synchronize a result obtained in a pythread to a gevent thread
    To use:

    with AsyncDispatcher(callback, arg1, arg2, keyword=argument) as dispatcher:
        v = dispatcher.wait(10)

    The concurrency is NOT thread safe, do not attempt to modify shared regions inside the
    dispatcher's context, it is solely meant to allow os threads to operate concurently
    with greenlets.
    '''
    _value     = None
    _exception = None
    _set       = None
    event      = None

    def __init__(self, callback, *args, **kwargs):
        self.callback = callback
        self.args     = args
        self.kwargs   = kwargs

    def __enter__(self):
        self.event = AsyncEvent()
        self.event.__enter__()
        pythread = get_pythread()
        self._thread = pythread.start_new_thread(self.dispatch, (self.callback,) + self.args, self.kwargs)
        return self

    def __exit__(self, type, value, traceback):
        self.event.__exit__(type, value, traceback)

    def dispatch(self, callback, *args, **kwargs):
        '''
        Runs a callback, either sets an asynchronous value or an exception
        When the os thread completes, the event is set to signal gevent that it's complete
        '''
        try:
            retval = callback(*args, **kwargs)
            self._value = retval
        except Exception as e:
            self._exception = e
        self.event.set()

    def wait(self, timeout=None):
        '''
        Blocks the current gevent greenlet until a value is set or the timeout expires.
        If the timeout expires a Timeout is raised.
        If the callback raised an exception in the os thread, an exception is raised here.
        '''
        if self.event.wait(timeout):
            if self._exception:
                raise self._exception
            else:
                return self._value
        else:
            raise gevent.timeout.Timeout(timeout)

<commit_msg>Fixes comments and docstrings<commit_after>#!/usr/bin/env python

__author__ = 'Adam R. Smith,Luke'

import gevent
import os
import fcntl
from gevent.event import Event
from collections import Iterable
from functools import wraps
import time

spawn = gevent.spawn

def spawnf(f):
    """ Decorator to spawn this function in a greenlet. """
    @wraps(f)
    def wrapper(*args, **kwargs):
        return gevent.spawn(f, *args, **kwargs)
    return wrapper

def asyncf(f):
    """ Decorator to spawn this function in a greenlet and return the result inline. """
    @wraps(f)
    def wrapper(*args, **kwargs):
        g = gevent.spawn(f, *args, **kwargs)
        return g.get()
    return wrapper

def switch():
    """ Shortcut to give control from the current greenlet back to the gevent hub. """
    gevent.getcurrent().switch()

def join(green_stuff):
    """ Universal way to join on either a single greenlet or a list of them. """
    if isinstance(green_stuff, Iterable):
        return gevent.joinall(green_stuff)
    return gevent.join(green_stuff)

def wait(green_stuff):
    """ Universal way to join on either a single greenlet or a list of them and get return value inline. """
    if isinstance(green_stuff, Iterable):
        gevent.joinall(green_stuff)
        return [g.get() for g in green_stuff]
    return green_stuff.get()

def blocking_cb(func, cb_arg, *args, **kwargs):
    """
    Wrap a function that takes a callback as a named parameter, to block and return its arguments as the result.
    Really handy for working with callback-based APIs. Do not use in really frequently-called code.
    If keyword args are supplied, they come through in a single dictionary to avoid out-of-order issues.
    """
    ev = Event()
    ret_vals = []
    def cb(*args, **kwargs):
        ret_vals.extend(args)
        if len(kwargs): ret_vals.append(kwargs)
        ev.set()
    kwargs[cb_arg] = cb
    func(*args, **kwargs)
    ev.wait(timeout=10)
    if len(ret_vals) == 0:
        return None
    elif len(ret_vals) == 1:
        return ret_vals[0]
    return tuple(ret_vals)

#--------------------------------------------------------------------------------

_pythread = None

def get_pythread():
    '''
    Loads the 'thread' module, free of monkey patching.
    '''
    global _pythread
    if _pythread:
        return _pythread # Cache the module so we don't have to use imp every time

    import imp
    fp, path, desc = imp.find_module('thread')
    try:
        _pythread = imp.load_module('pythread', fp, path, desc)
    finally:
        if fp:
            fp.close() # Close the file
    return _pythread

_pytime = None
def get_pytime():
    global _pytime
    if _pytime:
        return _pytime

    import imp
    fp, path, desc = imp.find_module('time')
    try:
        _pytime = imp.load_module('pytime', fp, path, desc)
    finally:
        if fp:
            fp.close()
    return _pytime

def nonblock_pipe():
    r, w = os.pipe()
    fcntl.fcntl(r, fcntl.F_SETFD, os.O_NONBLOCK)
    fcntl.fcntl(w, fcntl.F_SETFD, os.O_NONBLOCK)
    return r,w

_async_pipe_read, _async_pipe_write = nonblock_pipe()

def _async_pipe_read_callback(event, evtype):
    '''
    libevent callback to read from the pipe
    '''
    try:
        os.read(event.fd, 1)
    except EnvironmentError:
        pass # EAGAIN

# create a libevent callback to read from the pipe
gevent.core.event(gevent.core.EV_READ | gevent.core.EV_PERSIST,
                  _async_pipe_read,
                  _async_pipe_read_callback).add()

class AsyncResult(gevent.event.AsyncResult):
    def __init__(self):
        gevent.event.AsyncResult.__init__(self)

    def get(self, *args, **kwargs):
        return gevent.event.AsyncResult.get(self, *args, **kwargs)

    def set_exception(self, exception):
        gevent.event.AsyncResult.set_exception(self, exception)
        os.write(_async_pipe_write, '\0')

    def set(self, value):
        gevent.event.AsyncResult.set(self, value)
        os.write(_async_pipe_write, '\0')

class AsyncEvent(gevent.event.Event):
    '''
    A gevent-friendly event to be signaled by a posix thread and respected by
    the gevent context manager.

    Essentially the posix thread signals gevent through a nonblocking pipe
    '''

    def __init__(self):
        gevent.event.Event.__init__(self)
        self._r, self._w = self._pipe() # non blocking pipe

        # Create a new gevent core object that will observe the
        # nonblocking pipe file descriptor for a value
        self._core_event = gevent.core.event(
                gevent.core.EV_READ | gevent.core.EV_PERSIST,
                self._r,
                self._pipe_read)
        self._core_event.add()

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        self._close()

    def _pipe(self):
        return nonblock_pipe()

    def _pipe_read(self, event, eventtype):
        '''
        Non-blocking gevent friendly core event callback
        http://www.gevent.org/gevent.core.html#events
        '''
        try:
            os.read(event.fd, 1)
        except EnvironmentError:
            # file descriptors set with O_NONBLOCK return -1 and set errno to
            # EAGAIN, we just want to ignore it and try again later
            pass

    def set(self):
        '''
        Sets the event value and writes a value to the pipe to
        trigger the gevent core event
        '''

        gevent.event.Event.set(self)
        os.write(self._w, '\0') # Empty C-string '\0'

    def _close(self):
        '''
        Closes the core event, the read and write file descriptors
        '''
        if getattr(self, '_core_event', None):
            try:
                self._core_event.cancel()
            except:
                pass
        if getattr(self, '_r', None):
            try:
                os.close(self._r)
            except:
                pass
        if getattr(self, '_w', None):
            try:
                os.close(self._w)
            except:
                pass

class AsyncQueue(object):
    '''
    An extremely primitive queue that has a mutex over pthreads
    '''
    def __init__(self):
        self.__queue = []
        pythread = get_pythread()
        self.lock = pythread.allocate_lock()

    def put(self, item):
        with self.lock:
            self.__queue.append(item)

    def get(self):
        retval = None
        with self.lock:
            if self.__queue:
                retval = self.__queue.pop(0)
        return retval

class AsyncTask(object):
    '''
    A structure to represent the elements of a threadable task
    '''
    def __init__(self, callback, *args, **kwargs):
        self.ar = AsyncResult()
        self.callback = callback
        self.args = args
        self.kwargs = kwargs

class ThreadExit(Exception):
    def __init__(self):
        self.ar = AsyncResult()

class ThreadJob(object):
    '''
    A thread pool worker
    '''
    def __init__(self, queue):
        self.queue = queue

    def sleep(self, n):
        '''
        Sleeps n seconds using the un-patched time module
        '''
        pytime = get_pytime()
        pytime.sleep(n)

    def run(self):
        '''
        Runs a loop, listening for tasks
        '''
        while True:
            entry = self.queue.get()
            # If there is no task available sleep for 0.02s
            if not entry:
                self.sleep(0.02)
                continue
            # If the entry is a ThreadExit, break the loop and exit from the thread
            elif isinstance(entry, ThreadExit):
                entry.ar.set(True)
                break

            # An unrecognized task
            elif not isinstance(entry, AsyncTask):
                raise Exception("Invalid task")

            ar = entry.ar
            callback = entry.callback
            args = entry.args
            kwargs = entry.kwargs
            # If a task raises or fails then just print the stack trace and
            # continue processing
            try:
                retval = callback(*args, **kwargs)
                ar.set(retval)
            except Exception as e:
                from traceback import print_exc
                print_exc()
                # Note: the AR has an exception set so clients can observe task status
                ar.set_exception(e)

'''
Thread Pool Limitations:
    - Any thread task that imports a gevent monkey-patched module will probably
      raise an exception if gevent tries to context switch while in the child
      thread.
    - When closing threads, it is VERY prudent to synchronize the threads,
      otherwise the threads will be dangling.
    - Currently, there is no way to force a thread to quit, especially since
      most of these threads probably won't be running in the python
      interpreter. We could use the pthread library to forcefully quit a
      thread, but the memory could leak and it could cause some serious issues
      in the interpreter.
    - If a timeout is placed on synchronizing closing threads, a SystemError is
      raised, the intent with this is to fast fail a container. If threads
      aren't synchronized in time and a timeout is applied, then there is a
      bigger problem.
      - If only a subset of threads are closed, but the alive ones are still
        listening on the queue, you wind up with an inconsistent thread pool,
        and reliably executing tasks on the pool and managing what the pool
        believes to be the correct number of threads, becomes a stochastic
        system.

Advisements:
    - Keep the thread tasks as small as possible.
    - Try to limit the uses to only the code that absolutely blocks gevent.
    - If a thread blocks too long, it may be advisable to increase the threadpool temporarily.
'''

class ThreadPool(object):
    '''
    A pool of threads that can be used to run gevent-blocking code

    pool = ThreadPool(10)
    '''
    def __init__(self, poolsize=5):
        self.poolsize = poolsize
        self.queue = AsyncQueue()
        self.pythread = get_pythread()
        self.active = False
        # sets active to True
        self._spawn_threads(self.poolsize)

    def _spawn_threads(self, num):
        for i in xrange(num):
            job_worker = ThreadJob(self.queue)
            self.pythread.start_new_thread(job_worker.run, tuple())
        self.active = True

    def _check_exit(self, n, sync=False, timeout=None):
        exits = []
        for i in xrange(n):
            exit = ThreadExit()
            exits.append(exit)
            self.queue.put(exit)

        if not sync:
            return # No synchronization probably not good...

        self._inner_check(exits, timeout)

    def _inner_check(self, exits, timeout=None):
        if timeout is not None:
            t0 = time.time()
        i = len(exits)
        while i > 0:
            if timeout is not None and time.time() > (t0 + timeout):
                raise SystemError("Failed to close and synchronize threads. Thread pool is now inconsistent")
            for exit in exits:
                if exit.ar.ready():
                    i-=1

    def resize(self, newsize, sync=False, timeout=None):
        '''
        Resizes the available pool

        If `sync` is set and the thread pool is reduced, the method will
        synchronize with each exiting thread to ensure an exit was successful.
        If `timeout` is set with the `sync` a SystemError will be raised if the
        timeout is exceeded.
        '''
        assert newsize > 0, "Must have a positive number of threads"
        if newsize > self.poolsize:
            n = newsize - self.poolsize
            self._spawn_threads(n)
        else:
            n = self.poolsize - newsize
            self._check_exit(n, sync, timeout)

        self.poolsize = newsize

    def apply_async(self, func, *args, **kwargs):
        '''Run func, using the given args and kwargs in the thread pool.
        Returns :class:`AsyncResult`
        '''

        assert self.active, "Thread pool is deactivated"

        task = AsyncTask(func, *args, **kwargs)
        self.queue.put(task)
        return task.ar

    def apply(self, func, *args, **kwargs):
        '''Run func, using the given args and kwargs in the thread pool,
        but block the current greenlet until the result is ready.
        Returns the value from func after execution'''

        assert self.active, "Thread pool is deactivated"

        task = AsyncTask(func, *args, **kwargs)
        self.queue.put(task)
        retval = task.ar.get()
        return retval

    def close(self, sync=False, timeout=None):
        '''
        Sends each thread an Exit exception and deactivates the thread pool. If
        `sync` is set to True, the ThreadPool with synchronize with each thread
        before close returns. If `timeout` is set to a postive number and
        `sync` is set, then a SystemError will be raised if the timeout is
        exceeded before the threads have exited.

        Raises SystemError if timeout is exceeded before all the threads have exited.
        '''
        self._check_exit(self.poolsize, sync, timeout)
        self.active = False

    def map(self, func, arg_list):
        '''
        Takes a function and an iterable argument list.
        Runs func against each argument list in the iterable and returns a list
        of results, the results list will block the current greenlet in a
        gevent-safe way.
        '''
        jobs = self.map_async(func, arg_list)
        gevent.event.waitall(jobs)
        return jobs

    def map_async(self, func, arg_list):
        '''
        Takes a function and an iterable argument list.
        Runs func using each argument list in the iterable and returns a list
        of AsyncResult objects.
        '''
        async_jobs = [self.apply_async(func, *i) for i in arg_list]
        return async_jobs

class AsyncDispatcher(object):
    '''
    Used to synchronize a result obtained in a pythread to a gevent thread
    To use:

    with AsyncDispatcher(callback, arg1, arg2, keyword=argument) as dispatcher:
        v = dispatcher.wait(10)

    The concurrency is NOT thread safe, do not attempt to modify shared regions inside the
    dispatcher's context, it is solely meant to allow os threads to operate concurently
    with greenlets.
    '''
    _value     = None
    _exception = None
    _set       = None
    event      = None

    def __init__(self, callback, *args, **kwargs):
        self.callback = callback
        self.args     = args
        self.kwargs   = kwargs

    def __enter__(self):
        self.event = AsyncEvent()
        self.event.__enter__()
        pythread = get_pythread()
        self._thread = pythread.start_new_thread(self.dispatch, (self.callback,) + self.args, self.kwargs)
        return self

    def __exit__(self, type, value, traceback):
        self.event.__exit__(type, value, traceback)

    def dispatch(self, callback, *args, **kwargs):
        '''
        Runs a callback, either sets an asynchronous value or an exception
        When the os thread completes, the event is set to signal gevent that it's complete
        '''
        try:
            retval = callback(*args, **kwargs)
            self._value = retval
        except Exception as e:
            self._exception = e
        self.event.set()

    def wait(self, timeout=None):
        '''
        Blocks the current gevent greenlet until a value is set or the timeout expires.
        If the timeout expires a Timeout is raised.
        If the callback raised an exception in the os thread, an exception is raised here.
        '''
        if self.event.wait(timeout):
            if self._exception:
                raise self._exception
            else:
                return self._value
        else:
            raise gevent.timeout.Timeout(timeout)

<|endoftext|>