<commit_before># @(#) $Id: Simulator.py,v 1.33 2006/05/02 21:43:30 dfugate Exp $
#
# Copyright (C) 2001
# Associated Universities, Inc. Washington DC, USA.
#
# Produced for the ALMA project
#
# This library is free software; you can redistribute it and/or modify it under
# the terms of the GNU Library General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option) any
# later version.
#
# This library is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more
# details.
#
# You should have received a copy of the GNU Library General Public License
# along with this library; if not, write to the Free Software Foundation, Inc.,
# 675 Massachusetts Ave, Cambridge, MA 02139, USA.  Correspondence concerning
# ALMA should be addressed as follows:
#
# Internet email: alma-sw-admin@nrao.edu
# "@(#) $Id: Simulator.py,v 1.33 2006/05/02 21:43:30 dfugate Exp $"
#
# who       when        what
# --------  ----------  -------------------------------------------------------
# dfugate   2003/12/09  Created.
#------------------------------------------------------------------------------
'''
This module contains the implementation of a generic IDL interface simulator
component.  Currently broken.

TODO LIST:
- all
'''
#--REGULAR IMPORTS-------------------------------------------------------------
from traceback import print_exc
#--CORBA STUBS-----------------------------------------------------------------
import CORBA
import omniORB
#--ACS Imports-----------------------------------------------------------------
from Acspy.Servants.ContainerServices        import ContainerServices
from Acspy.Servants.ComponentLifecycle       import ComponentLifecycle
from Acspy.Servants.CharacteristicComponent  import CharacteristicComponent

from Acspy.Util.BaciHelper   import addProperty
from ACSImpl.DevIO           import DevIO
from Acspy.Util.ACSCorba     import interfaceRepository

from Acssim.Corba.DynamicImplementation    import DynamicImplementation
from Acssim.Servants.Executor              import _execute
from Acssim.Goodies               import getSimProxy
from Acssim.Servants.Executor              import _executeDict
from Acssim.Corba.Generator             import tryCallbackParams
from Acssim.Corba.Generator             import getRandomEnum
from Acssim.Goodies               import addComponent
from Acssim.Goodies               import removeComponent
from Acssim.Goodies               import getCompLocalNS
from Acssim.Corba.Utilities                import getSuperIDs
from Acssim.Servants.Representations.BehaviorProxy import BehaviorProxy
from Acssim.Corba.EventDispatcher import EventDispatcher
#--GLOBALS---------------------------------------------------------------------
_DEBUG = 0
#------------------------------------------------------------------------------
class BaseSimulator(DynamicImplementation):
    '''
    '''
    def __init__(self, ir, name=None):
        '''
        '''
        self.__ir = ir

        DynamicImplementation.__init__(self, ir)

        if name != None:
            self.__name = str(name)

        else:
            self.__name = ir
    #------------------------------------------------------------------------------
    def invoke(self, args, moreargs):
        '''
        This method has all calls to CORBA operations forwarded to it.  Definitly
        needs to be overriden in subclasses.

        Parameters:
        - args is a tuple of arguments...args[0] is the calling methods name and
        everything that follows were parameters to the calling method.
        - moreargs is a dictionary of yet more arguments

        Returns: Anything

        Raises: Anything
        '''
        if _DEBUG == 1:
            print self.__name, " - DynamicImplementation.invoke(", args, moreargs, ")"

        #simulate doing something with callbacks...
        tryCallbackParams(args[1:], self)

        meth_name = args[0]

        if len(args)>1:
            #create the list of args
            args = list(args[1:])
            #append the component reference
            args.append(self)
            new_args = args

        else:
            new_args = [self]

        new_args = tuple(new_args)

        return _execute(self.__name,
                        meth_name,
                        new_args,
                        getCompLocalNS(self.__name))

#------------------------------------------------------------------------------
class Simulator(CharacteristicComponent,  #Base IDL interface
                BaseSimulator, #CORBA stubs for IDL interface
                ContainerServices,  #Developer niceties
                ComponentLifecycle):  #HLA stuff
    '''
    '''
    #------------------------------------------------------------------------------
    def __init__(self):
        '''
        Just call superclass constructors here.
        '''
        CharacteristicComponent.__init__(self)
        ContainerServices.__init__(self)
        return
    #------------------------------------------------------------------------------
    #--Override ComponentLifecycle methods-----------------------------------------
    #------------------------------------------------------------------------------
    def initialize (self):
        '''
        Overriden from baseclass.
        '''
        #we can finally access the IR location and call BaseSimulator's
        #constructor...dynamically changing this object's inheritance
        BaseSimulator.__init__(self, self.ir, self._get_name())

        #handle attributes that should NOT be generically simulated
        self.__setupSpecialCases()

        #add myself to the global lis
        addComponent(self._get_name(), self)

        #possible for developers to configure an initialize method
        #for the simulated component.
        _execute(self._get_name(),
                 "initialize",
                 (self),
                 getCompLocalNS(self._get_name()))

        #create the object used to dispatch events automatically
        self.event_dispatcher = EventDispatcher(self)

    #------------------------------------------------------------------------------
    def cleanUp(self):
        '''
        Overriden from baseclass.
        '''
        self.event_dispatcher.destroy()

        #possible for developers to configure cleanUp method
        #for the simulated component.
        _execute(self._get_name(),
                 "cleanUp",
                 (self),
                 getCompLocalNS(self._get_name()))

        ComponentLifecycle.cleanUp(self)
        removeComponent(self._get_name())
    #------------------------------------------------------------------------------
    def __setupSpecialCases(self):
        '''
        Helper method designed to handle special cases that we do not necessarily
        want being 100% simulated like BACI properties, callbacks, etc.
        '''
        #look in the CDB for instructions on how to setup the special cases. This
        #is mainly used to see if the end-user has specified some devIO class to
        #be used with a BACI property.
        if_list = getSuperIDs(self.ir)
        if_list.append(self.ir)
        simCDB = getSimProxy(self._get_name()).cdb_handler

        #IFR
        ir = interfaceRepository()

        #_executeXyz methods need an argument list
        args = (self)

        #get an interface description for this component
        interf = ir.lookup_id(self._NP_RepositoryId)
        interf = interf._narrow(CORBA.InterfaceDef)
        interf = interf.describe_interface()

        #use the IFR descrip to go searching for BACI properties
        for attribute in interf.attributes:

            #if the typecode is NOT an object reference it cannot be a BACI property.
            #that implies it's OK to skip
            if not isinstance(attribute.type, omniORB.tcInternal.TypeCode_objref):
                continue

            #save the short version of the attribute's ID (i.e., ROdouble)
            tempType = attribute.type.id().split(":")[1].split("/").pop()

            #sequence BACI property
            if (tempType=="ROstringSeq")or(tempType=="ROdoubleSeq")or(tempType=="RWdoubleSeq")or(tempType=="ROlongSeq")or(tempType=="RWlongSeq"):
                cdbAttrDict = simCDB.getMethod(attribute.name)
                if  cdbAttrDict!= None:
                    devio = _executeDict(cdbAttrDict, args, getCompLocalNS(self._get_name()))
                else:
                    devio = DevIO([])

                addProperty(self, attribute.name, devio_ref=devio)
                continue

            #double BACI property
            elif (tempType=="ROdouble")or(tempType=="RWdouble"):
                cdbAttrDict = simCDB.getMethod(attribute.name)
                if  cdbAttrDict!= None:
                    devio = _executeDict(cdbAttrDict, args, getCompLocalNS(self._get_name()))
                else:
                    devio = DevIO(float(0))

                addProperty(self, attribute.name, devio_ref=devio)
                continue

            #long BACI property
            elif (tempType=="ROlong")or(tempType=="RWlong"):
                cdbAttrDict = simCDB.getMethod(attribute.name)
                if  cdbAttrDict!= None:
                    devio = _executeDict(cdbAttrDict, args, getCompLocalNS(self._get_name()))
                else:
                    devio = DevIO(0)

                addProperty(self, attribute.name, devio_ref=devio)
                continue

            #long (Python long also) BACI property
            elif (tempType=="ROpattern")or(tempType=="RWpattern")or(tempType=="ROlongLong")or(tempType=="RWlongLong")or(tempType=="ROuLongLong")or(tempType=="ROuLongLong"):
                cdbAttrDict = simCDB.getMethod(attribute.name)
                if  cdbAttrDict!= None:
                    devio = _executeDict(cdbAttrDict, args, getCompLocalNS(self._get_name()))
                else:
                    devio = DevIO(0L)

                addProperty(self, attribute.name, devio_ref=devio)
                continue

            #string BACI property
            elif (tempType=="ROstring")or(tempType=="RWstring"):
                cdbAttrDict = simCDB.getMethod(attribute.name)
                if  cdbAttrDict!= None:
                    devio = _executeDict(cdbAttrDict, args, getCompLocalNS(self._get_name()))
                else:
                    devio = DevIO("")

                addProperty(self, attribute.name, devio_ref=devio)
                continue

            else:

                ifrName = attribute.type.id()

                try:
                    #get an interface description for this property
                    tIfr = ir.lookup_id(ifrName)
                    tIfr = tIfr._narrow(CORBA.InterfaceDef)
                    tIfr = tIfr.describe_interface()

                    for tAttr in tIfr.attributes:
                        #check if it's a default_value AND an enum!
                        if (tAttr.name=="default_value") and (tAttr.type.kind()==CORBA.tk_enum):
                            #GREAT! It's completely safe to add!
                            cdbAttrDict = simCDB.getMethod(attribute.name)
                            if  cdbAttrDict!= None:
                                devio = _executeDict(cdbAttrDict, args, getCompLocalNS(self._get_name()))
                            else:
                                devio = DevIO(getRandomEnum(tAttr.type))
                            addProperty(self, attribute.name, devio_ref=devio)

                            break

                except:
                    print_exc()
                    continue
    #------------------------------------------------------------------------------
    #--Implementation of IDL methods-----------------------------------------------
    #------------------------------------------------------------------------------

#------------------------------------------------------------------------------
#--Main defined only for generic testing---------------------------------------
#------------------------------------------------------------------------------
if __name__ == "__main__":
    print "Creating an object"
    g = Simulator()
    print "Done..."

<commit_msg>Casting argument parameters to tuple type was causing problems. Using list instead<commit_after># @(#) $Id: Simulator.py,v 1.34 2006/05/02 22:55:44 dfugate Exp $
#
# Copyright (C) 2001
# Associated Universities, Inc. Washington DC, USA.
#
# Produced for the ALMA project
#
# This library is free software; you can redistribute it and/or modify it under
# the terms of the GNU Library General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option) any
# later version.
#
# This library is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more
# details.
#
# You should have received a copy of the GNU Library General Public License
# along with this library; if not, write to the Free Software Foundation, Inc.,
# 675 Massachusetts Ave, Cambridge, MA 02139, USA.  Correspondence concerning
# ALMA should be addressed as follows:
#
# Internet email: alma-sw-admin@nrao.edu
# "@(#) $Id: Simulator.py,v 1.34 2006/05/02 22:55:44 dfugate Exp $"
#
# who       when        what
# --------  ----------  -------------------------------------------------------
# dfugate   2003/12/09  Created.
#------------------------------------------------------------------------------
'''
This module contains the implementation of a generic IDL interface simulator
component.  Currently broken.

TODO LIST:
- all
'''
#--REGULAR IMPORTS-------------------------------------------------------------
from traceback import print_exc
#--CORBA STUBS-----------------------------------------------------------------
import CORBA
import omniORB
#--ACS Imports-----------------------------------------------------------------
from Acspy.Servants.ContainerServices        import ContainerServices
from Acspy.Servants.ComponentLifecycle       import ComponentLifecycle
from Acspy.Servants.CharacteristicComponent  import CharacteristicComponent

from Acspy.Util.BaciHelper   import addProperty
from ACSImpl.DevIO           import DevIO
from Acspy.Util.ACSCorba     import interfaceRepository

from Acssim.Corba.DynamicImplementation    import DynamicImplementation
from Acssim.Servants.Executor              import _execute
from Acssim.Goodies               import getSimProxy
from Acssim.Servants.Executor              import _executeDict
from Acssim.Corba.Generator             import tryCallbackParams
from Acssim.Corba.Generator             import getRandomEnum
from Acssim.Goodies               import addComponent
from Acssim.Goodies               import removeComponent
from Acssim.Goodies               import getCompLocalNS
from Acssim.Corba.Utilities                import getSuperIDs
from Acssim.Servants.Representations.BehaviorProxy import BehaviorProxy
from Acssim.Corba.EventDispatcher import EventDispatcher
#--GLOBALS---------------------------------------------------------------------
_DEBUG = 0
#------------------------------------------------------------------------------
class BaseSimulator(DynamicImplementation):
    '''
    '''
    def __init__(self, ir, name=None):
        '''
        '''
        self.__ir = ir

        DynamicImplementation.__init__(self, ir)

        if name != None:
            self.__name = str(name)

        else:
            self.__name = ir
    #------------------------------------------------------------------------------
    def invoke(self, args, moreargs):
        '''
        This method has all calls to CORBA operations forwarded to it.  Definitly
        needs to be overriden in subclasses.

        Parameters:
        - args is a tuple of arguments...args[0] is the calling methods name and
        everything that follows were parameters to the calling method.
        - moreargs is a dictionary of yet more arguments

        Returns: Anything

        Raises: Anything
        '''
        if _DEBUG == 1:
            print self.__name, " - DynamicImplementation.invoke(", args, moreargs, ")"

        #simulate doing something with callbacks...
        tryCallbackParams(args[1:], self)

        meth_name = args[0]

        if len(args)>1:
            #create the list of args
            args = list(args[1:])
            #append the component reference
            args.append(self)
            new_args = args

        else:
            new_args = [self]

        return _execute(self.__name,
                        meth_name,
                        new_args,
                        getCompLocalNS(self.__name))

#------------------------------------------------------------------------------
class Simulator(CharacteristicComponent,  #Base IDL interface
                BaseSimulator, #CORBA stubs for IDL interface
                ContainerServices,  #Developer niceties
                ComponentLifecycle):  #HLA stuff
    '''
    '''
    #------------------------------------------------------------------------------
    def __init__(self):
        '''
        Just call superclass constructors here.
        '''
        CharacteristicComponent.__init__(self)
        ContainerServices.__init__(self)
        return
    #------------------------------------------------------------------------------
    #--Override ComponentLifecycle methods-----------------------------------------
    #------------------------------------------------------------------------------
    def initialize (self):
        '''
        Overriden from baseclass.
        '''
        #we can finally access the IR location and call BaseSimulator's
        #constructor...dynamically changing this object's inheritance
        BaseSimulator.__init__(self, self.ir, self._get_name())

        #handle attributes that should NOT be generically simulated
        self.__setupSpecialCases()

        #add myself to the global lis
        addComponent(self._get_name(), self)

        #possible for developers to configure an initialize method
        #for the simulated component.
        _execute(self._get_name(),
                 "initialize",
                 [self],
                 getCompLocalNS(self._get_name()))

        #create the object used to dispatch events automatically
        self.event_dispatcher = EventDispatcher(self)

    #------------------------------------------------------------------------------
    def cleanUp(self):
        '''
        Overriden from baseclass.
        '''
        self.event_dispatcher.destroy()

        #possible for developers to configure cleanUp method
        #for the simulated component.
        _execute(self._get_name(),
                 "cleanUp",
                 [self],
                 getCompLocalNS(self._get_name()))

        ComponentLifecycle.cleanUp(self)
        removeComponent(self._get_name())
    #------------------------------------------------------------------------------
    def __setupSpecialCases(self):
        '''
        Helper method designed to handle special cases that we do not necessarily
        want being 100% simulated like BACI properties, callbacks, etc.
        '''
        #look in the CDB for instructions on how to setup the special cases. This
        #is mainly used to see if the end-user has specified some devIO class to
        #be used with a BACI property.
        if_list = getSuperIDs(self.ir)
        if_list.append(self.ir)
        simCDB = getSimProxy(self._get_name()).cdb_handler

        #IFR
        ir = interfaceRepository()

        #_executeXyz methods need an argument list
        args = [self]

        #get an interface description for this component
        interf = ir.lookup_id(self._NP_RepositoryId)
        interf = interf._narrow(CORBA.InterfaceDef)
        interf = interf.describe_interface()

        #use the IFR descrip to go searching for BACI properties
        for attribute in interf.attributes:

            #if the typecode is NOT an object reference it cannot be a BACI property.
            #that implies it's OK to skip
            if not isinstance(attribute.type, omniORB.tcInternal.TypeCode_objref):
                continue

            #save the short version of the attribute's ID (i.e., ROdouble)
            tempType = attribute.type.id().split(":")[1].split("/").pop()

            #sequence BACI property
            if (tempType=="ROstringSeq")or(tempType=="ROdoubleSeq")or(tempType=="RWdoubleSeq")or(tempType=="ROlongSeq")or(tempType=="RWlongSeq"):
                cdbAttrDict = simCDB.getMethod(attribute.name)
                if  cdbAttrDict!= None:
                    devio = _executeDict(cdbAttrDict, args, getCompLocalNS(self._get_name()))
                else:
                    devio = DevIO([])

                addProperty(self, attribute.name, devio_ref=devio)
                continue

            #double BACI property
            elif (tempType=="ROdouble")or(tempType=="RWdouble"):
                cdbAttrDict = simCDB.getMethod(attribute.name)
                if  cdbAttrDict!= None:
                    devio = _executeDict(cdbAttrDict, args, getCompLocalNS(self._get_name()))
                else:
                    devio = DevIO(float(0))

                addProperty(self, attribute.name, devio_ref=devio)
                continue

            #long BACI property
            elif (tempType=="ROlong")or(tempType=="RWlong"):
                cdbAttrDict = simCDB.getMethod(attribute.name)
                if  cdbAttrDict!= None:
                    devio = _executeDict(cdbAttrDict, args, getCompLocalNS(self._get_name()))
                else:
                    devio = DevIO(0)

                addProperty(self, attribute.name, devio_ref=devio)
                continue

            #long (Python long also) BACI property
            elif (tempType=="ROpattern")or(tempType=="RWpattern")or(tempType=="ROlongLong")or(tempType=="RWlongLong")or(tempType=="ROuLongLong")or(tempType=="ROuLongLong"):
                cdbAttrDict = simCDB.getMethod(attribute.name)
                if  cdbAttrDict!= None:
                    devio = _executeDict(cdbAttrDict, args, getCompLocalNS(self._get_name()))
                else:
                    devio = DevIO(0L)

                addProperty(self, attribute.name, devio_ref=devio)
                continue

            #string BACI property
            elif (tempType=="ROstring")or(tempType=="RWstring"):
                cdbAttrDict = simCDB.getMethod(attribute.name)
                if  cdbAttrDict!= None:
                    devio = _executeDict(cdbAttrDict, args, getCompLocalNS(self._get_name()))
                else:
                    devio = DevIO("")

                addProperty(self, attribute.name, devio_ref=devio)
                continue

            else:

                ifrName = attribute.type.id()

                try:
                    #get an interface description for this property
                    tIfr = ir.lookup_id(ifrName)
                    tIfr = tIfr._narrow(CORBA.InterfaceDef)
                    tIfr = tIfr.describe_interface()

                    for tAttr in tIfr.attributes:
                        #check if it's a default_value AND an enum!
                        if (tAttr.name=="default_value") and (tAttr.type.kind()==CORBA.tk_enum):
                            #GREAT! It's completely safe to add!
                            cdbAttrDict = simCDB.getMethod(attribute.name)
                            if  cdbAttrDict!= None:
                                devio = _executeDict(cdbAttrDict, args, getCompLocalNS(self._get_name()))
                            else:
                                devio = DevIO(getRandomEnum(tAttr.type))
                            addProperty(self, attribute.name, devio_ref=devio)

                            break

                except:
                    print_exc()
                    continue
    #------------------------------------------------------------------------------
    #--Implementation of IDL methods-----------------------------------------------
    #------------------------------------------------------------------------------

#------------------------------------------------------------------------------
#--Main defined only for generic testing---------------------------------------
#------------------------------------------------------------------------------
if __name__ == "__main__":
    print "Creating an object"
    g = Simulator()
    print "Done..."

<|endoftext|>


<commit_before>'''
Created on 2016/3/8

:author: hubo
'''
from vlcp.config import defaultconfig
from vlcp.service.connection.tcpserver import TcpServerBase
from vlcp.protocol.redis import Redis
from vlcp.server.module import api
from vlcp.utils.redisclient import RedisClient
from vlcp.event.runnable import RoutineContainer
import pickle
from _ctypes import set_conversion_mode
try:
    import cPickle
except ImportError:
    pass
import json
from vlcp.utils.jsonencoder import encode_default, decode_object
import itertools
import random

class RedisWriteConflictException(Exception):
    pass

@defaultconfig
class RedisDB(TcpServerBase):
    '''
    Create redis clients to connect to redis server
    '''
    _default_url = None
    _default_db = None
    _default_serialize = 'json'
    _default_pickleversion = 'default'
    _default_cpickle = True
    _default_maxretry = 16
    _default_maxspin = 10
    client = True
    def __init__(self, server):
        TcpServerBase.__init__(self, server, Redis)
        self._redis_clients = {}
        self.apiroutine = RoutineContainer(self.scheduler)
        if self.serialize == 'pickle' or self.serialize == 'cpickle' or self.serialize == 'cPickle':
            if self.serialize == 'pickle':
                if not self.cpickle:
                    p = pickle
                else:
                    p = cPickle
            else:
                p = cPickle
            if self.pickleversion is None or self.pickleversion == 'default':
                pickleversion = None
            elif self.pickleversion == 'highest':
                pickleversion = p.HIGHEST_PROTOCOL
            else:
                pickleversion = self.pickleversion
            def _encode(obj):
                return p.dumps(obj, pickleversion)
            self._encode = _encode
            def _decode(data):
                return p.loads(data)
            self._decode = _decode
        else:
            def _encode(obj):
                return json.dumps(obj, default=encode_default)
            self._encode = _encode
            def _decode(data):
                return json.loads(data, object_hook=decode_object)
            self._decode = _decode
        self.appendAPI(api(self.getclient),
                       api(self.get, self.apiroutine),
                       api(self.set, self.apiroutine),
                       api(self.delete, self.apiroutine),
                       api(self.mget, self.apiroutine),
                       api(self.mset, self.apiroutine),
                       api(self.update, self.apiroutine),
                       api(self.mupdate, self.apiroutine))
    def _client_class(self, config, protocol, vhost):
        db = getattr(config, 'db', None)
        def _create_client(url, protocol, scheduler = None, key = None, certificate = None, ca_certs = None, bindaddress = None):
            c = RedisClient(url, db, protocol)
            if key:
                c.key = key
            if certificate:
                c.certificate = certificate
            if ca_certs:
                c.ca_certs = ca_certs
            self._redis_clients[vhost] = c
            return c.make_connobj(self.apiroutine)
        return _create_client
    def getclient(self, vhost = ''):
        return self._redis_clients.get(vhost)
    def get(self, key, timeout = None, vhost = ''):
        "Get value from key"
        c = self.getclient(vhost)
        if c is None:
            raise ValueError('vhost ' + repr(vhost) + ' is not defined')
        if timeout is not None:
            if timeout <= 0:
                for m in c.batch_execute(self.apiroutine, ('MULTI',),
                                                        ('GET', key),
                                                        ('DEL', key),
                                                        ('EXEC',)):
                    yield m
            else:
                for m in c.batch_execute(self.apiroutine, ('MULTI',),
                                                        ('GET', key),
                                                        ('PEXPIRE', key, int(timeout * 1000)),
                                                        ('EXEC',)):
                    yield m
            r = self.apiroutine.retvalue[3][0]
            if isinstance(r, Exception):
                raise r
            self.apiroutine.retvalue = self._decode(r)
        else:
            for m in c.execute_command(self.apiroutine, 'GET', key):
                yield m
            self.apiroutine.retvalue = self._decode(self.apiroutine.retvalue)
    def set(self, key, value, timeout = None, vhost = ''):
        "Set value to key, with an optional timeout"
        c = self.getclient(vhost)
        if timeout is None:
            for m in c.execute_command(self.apiroutine, 'SET', key, self._encode(value)):
                yield m
        elif timeout <= 0:
            for m in c.execute_command(self.apiroutine, 'DEL', key):
                yield m
        else:
            for m in c.execute_command(self.apiroutine, 'PSETEX', key, int(timeout * 1000), self._encode(value)):
                yield m
        self.apiroutine.retvalue = None
    def delete(self, key, vhost = ''):
        c = self.getclient(vhost)
        for m in c.execute_command(self.apiroutine, 'DEL', key):
            yield m
        self.apiroutine.retvalue = None
    def mget(self, keys, vhost = ''):
        "Get multiple values from multiple keys"
        c = self.getclient(vhost)
        for m in c.execute_command(self.apiroutine, 'MGET', *keys):
            yield m
        self.apiroutine.retvalue = [self._decode(r) for r in self.apiroutine.retvalue]
    def mset(self, kvpairs, timeout = None, vhost = ''):
        "Set multiple values on multiple keys"
        c = self.getclient(vhost)
        d = kvpairs
        if hasattr(d, 'items'):
            d = d.items()
        if timeout is not None and timeout <= 0:
            for m in c.execute_command(self.apiroutine, 'DEL', *[k for k,_ in d]):
                yield m
        else:
            if timeout is None:
                for m in c.execute_command(self.apiroutine, 'MSET', *list(itertools.chain(d))):
                    yield m
            else:
                # Use a transact
                ptimeout = int(timeout * 1000)

                for m in c.batch_execute(self.apiroutine, *((('MULTI',),
                                                             ('MSET',) + tuple(itertools.chain(d))) + \
                                                            tuple(('PEXPIRE', k, ptimeout) for k, _ in d) + \
                                                            (('EXEC',),))
                                                        ):
                    yield m
        self.apiroutine.retvalue = None
    def update(self, key, updater, timeout = None, vhost = ''):
        '''
        Update in-place with a custom function

        :param key: key to update

        :param updater: func(k,v), should return a new value to update, or return None to delete. The function
                        may be call more than once.

        :param timeout: new timeout

        :returns: the updated value, or None if deleted
        '''
        c = self.getclient(vhost)
        for m in c.get_connection(self.apiroutine):
            yield m
        newconn = self.apiroutine.retvalue
        spin = self.maxspin
        with newconn.context(self.apiroutine):
            for i in range(0, self.maxretry):
                for m in newconn.execute_command(self.apiroutine, 'WATCH', key):
                    yield m
                for m in newconn.execute_command(self.apiroutine, 'GET', key):
                    yield m
                v = self.apiroutine.retvalue
                v = updater(v)
                if timeout is None:
                    set_command = ('SET', key, v)
                elif timeout <= 0:
                    set_command = ('DEL', key)
                else:
                    set_command = ('PSETEX', key, int(timeout * 1000), v)
                for m in newconn.batch_execute(self.apiroutine, ('MULTI',),
                                                                set_command,
                                                                ('EXEC',)):
                    yield m
                r = self.apiroutine.retvalue[2]
                if r is not None:
                    # Succeeded
                    self.apiroutine.retvalue = v
                    raise StopIteration
                else:
                    # Watch keys changed, retry
                    if i > spin:
                        # Wait for a random small while
                        for m in self.apiroutine.waitWithTimeout(random.randrange(0, 1<<(i-spin)) * 0.05):
                            yield m
            raise RedisWriteConflictException('Transaction still fails after many retries: key=' + repr(key))

    def mupdate(self, keys, updater, timeout = None, vhost = ''):
        "Update multiple keys in-place with a custom function, see update"
        c = self.getclient(vhost)
        for m in c.get_connection(self.apiroutine):
            yield m
        newconn = self.apiroutine.retvalue
        spin = self.maxspin
        with newconn.context(self.apiroutine):
            for i in range(0, self.maxretry):
                for m in newconn.execute_command(self.apiroutine, 'WATCH', *keys):
                    yield m
                for m in newconn.execute_command(self.apiroutine, 'MGET', *keys):
                    yield m
                values = self.apiroutine.retvalue
                values = [updater(v) for v in values]
                if timeout is None:
                    set_commands = (('MSET',) + tuple(itertools.chain.from_iterable(itertools.izip(keys, values))),)
                elif timeout <= 0:
                    set_commands = (('DEL',) + tuple(keys),)
                else:
                    ptimeout = int(timeout * 1000)
                    set_commands = (('MSET',) + tuple(itertools.chain.from_iterable(itertools.izip(keys, values))),) \
                                + tuple(('PEXPIRE', k, ptimeout) for k in keys)
                for m in newconn.batch_execute(self.apiroutine, ((('MULTI',),) + \
                                                                set_commands + \
                                                                ('EXEC',))):
                    yield m
                r = self.apiroutine.retvalue[-1]
                if r is not None:
                    # Succeeded
                    self.apiroutine.retvalue = values
                    raise StopIteration
                else:
                    # Watch keys changed, retry
                    if i > spin:
                        # Wait for a random small while
                        for m in self.apiroutine.waitWithTimeout(random.randrange(0, 1<<(i-spin)) * 0.05):
                            yield m
            raise RedisWriteConflictException('Transaction still fails after many retries: keys=' + repr(keys))
<commit_msg>remove auto-generated imports...<commit_after>'''
Created on 2016/3/8

:author: hubo
'''
from vlcp.config import defaultconfig
from vlcp.service.connection.tcpserver import TcpServerBase
from vlcp.protocol.redis import Redis
from vlcp.server.module import api
from vlcp.utils.redisclient import RedisClient
from vlcp.event.runnable import RoutineContainer
import pickle
try:
    import cPickle
except ImportError:
    pass
import json
from vlcp.utils.jsonencoder import encode_default, decode_object
import itertools
import random

class RedisWriteConflictException(Exception):
    pass

@defaultconfig
class RedisDB(TcpServerBase):
    '''
    Create redis clients to connect to redis server
    '''
    _default_url = None
    _default_db = None
    _default_serialize = 'json'
    _default_pickleversion = 'default'
    _default_cpickle = True
    _default_maxretry = 16
    _default_maxspin = 10
    client = True
    def __init__(self, server):
        TcpServerBase.__init__(self, server, Redis)
        self._redis_clients = {}
        self.apiroutine = RoutineContainer(self.scheduler)
        if self.serialize == 'pickle' or self.serialize == 'cpickle' or self.serialize == 'cPickle':
            if self.serialize == 'pickle':
                if not self.cpickle:
                    p = pickle
                else:
                    p = cPickle
            else:
                p = cPickle
            if self.pickleversion is None or self.pickleversion == 'default':
                pickleversion = None
            elif self.pickleversion == 'highest':
                pickleversion = p.HIGHEST_PROTOCOL
            else:
                pickleversion = self.pickleversion
            def _encode(obj):
                return p.dumps(obj, pickleversion)
            self._encode = _encode
            def _decode(data):
                return p.loads(data)
            self._decode = _decode
        else:
            def _encode(obj):
                return json.dumps(obj, default=encode_default)
            self._encode = _encode
            def _decode(data):
                return json.loads(data, object_hook=decode_object)
            self._decode = _decode
        self.appendAPI(api(self.getclient),
                       api(self.get, self.apiroutine),
                       api(self.set, self.apiroutine),
                       api(self.delete, self.apiroutine),
                       api(self.mget, self.apiroutine),
                       api(self.mset, self.apiroutine),
                       api(self.update, self.apiroutine),
                       api(self.mupdate, self.apiroutine))
    def _client_class(self, config, protocol, vhost):
        db = getattr(config, 'db', None)
        def _create_client(url, protocol, scheduler = None, key = None, certificate = None, ca_certs = None, bindaddress = None):
            c = RedisClient(url, db, protocol)
            if key:
                c.key = key
            if certificate:
                c.certificate = certificate
            if ca_certs:
                c.ca_certs = ca_certs
            self._redis_clients[vhost] = c
            return c.make_connobj(self.apiroutine)
        return _create_client
    def getclient(self, vhost = ''):
        return self._redis_clients.get(vhost)
    def get(self, key, timeout = None, vhost = ''):
        "Get value from key"
        c = self.getclient(vhost)
        if c is None:
            raise ValueError('vhost ' + repr(vhost) + ' is not defined')
        if timeout is not None:
            if timeout <= 0:
                for m in c.batch_execute(self.apiroutine, ('MULTI',),
                                                        ('GET', key),
                                                        ('DEL', key),
                                                        ('EXEC',)):
                    yield m
            else:
                for m in c.batch_execute(self.apiroutine, ('MULTI',),
                                                        ('GET', key),
                                                        ('PEXPIRE', key, int(timeout * 1000)),
                                                        ('EXEC',)):
                    yield m
            r = self.apiroutine.retvalue[3][0]
            if isinstance(r, Exception):
                raise r
            self.apiroutine.retvalue = self._decode(r)
        else:
            for m in c.execute_command(self.apiroutine, 'GET', key):
                yield m
            self.apiroutine.retvalue = self._decode(self.apiroutine.retvalue)
    def set(self, key, value, timeout = None, vhost = ''):
        "Set value to key, with an optional timeout"
        c = self.getclient(vhost)
        if timeout is None:
            for m in c.execute_command(self.apiroutine, 'SET', key, self._encode(value)):
                yield m
        elif timeout <= 0:
            for m in c.execute_command(self.apiroutine, 'DEL', key):
                yield m
        else:
            for m in c.execute_command(self.apiroutine, 'PSETEX', key, int(timeout * 1000), self._encode(value)):
                yield m
        self.apiroutine.retvalue = None
    def delete(self, key, vhost = ''):
        c = self.getclient(vhost)
        for m in c.execute_command(self.apiroutine, 'DEL', key):
            yield m
        self.apiroutine.retvalue = None
    def mget(self, keys, vhost = ''):
        "Get multiple values from multiple keys"
        c = self.getclient(vhost)
        for m in c.execute_command(self.apiroutine, 'MGET', *keys):
            yield m
        self.apiroutine.retvalue = [self._decode(r) for r in self.apiroutine.retvalue]
    def mset(self, kvpairs, timeout = None, vhost = ''):
        "Set multiple values on multiple keys"
        c = self.getclient(vhost)
        d = kvpairs
        if hasattr(d, 'items'):
            d = d.items()
        if timeout is not None and timeout <= 0:
            for m in c.execute_command(self.apiroutine, 'DEL', *[k for k,_ in d]):
                yield m
        else:
            if timeout is None:
                for m in c.execute_command(self.apiroutine, 'MSET', *list(itertools.chain(d))):
                    yield m
            else:
                # Use a transact
                ptimeout = int(timeout * 1000)

                for m in c.batch_execute(self.apiroutine, *((('MULTI',),
                                                             ('MSET',) + tuple(itertools.chain(d))) + \
                                                            tuple(('PEXPIRE', k, ptimeout) for k, _ in d) + \
                                                            (('EXEC',),))
                                                        ):
                    yield m
        self.apiroutine.retvalue = None
    def update(self, key, updater, timeout = None, vhost = ''):
        '''
        Update in-place with a custom function

        :param key: key to update

        :param updater: func(k,v), should return a new value to update, or return None to delete. The function
                        may be call more than once.

        :param timeout: new timeout

        :returns: the updated value, or None if deleted
        '''
        c = self.getclient(vhost)
        for m in c.get_connection(self.apiroutine):
            yield m
        newconn = self.apiroutine.retvalue
        spin = self.maxspin
        with newconn.context(self.apiroutine):
            for i in range(0, self.maxretry):
                for m in newconn.execute_command(self.apiroutine, 'WATCH', key):
                    yield m
                for m in newconn.execute_command(self.apiroutine, 'GET', key):
                    yield m
                v = self.apiroutine.retvalue
                v = updater(v)
                if timeout is None:
                    set_command = ('SET', key, v)
                elif timeout <= 0:
                    set_command = ('DEL', key)
                else:
                    set_command = ('PSETEX', key, int(timeout * 1000), v)
                for m in newconn.batch_execute(self.apiroutine, ('MULTI',),
                                                                set_command,
                                                                ('EXEC',)):
                    yield m
                r = self.apiroutine.retvalue[2]
                if r is not None:
                    # Succeeded
                    self.apiroutine.retvalue = v
                    raise StopIteration
                else:
                    # Watch keys changed, retry
                    if i > spin:
                        # Wait for a random small while
                        for m in self.apiroutine.waitWithTimeout(random.randrange(0, 1<<(i-spin)) * 0.05):
                            yield m
            raise RedisWriteConflictException('Transaction still fails after many retries: key=' + repr(key))

    def mupdate(self, keys, updater, timeout = None, vhost = ''):
        "Update multiple keys in-place with a custom function, see update"
        c = self.getclient(vhost)
        for m in c.get_connection(self.apiroutine):
            yield m
        newconn = self.apiroutine.retvalue
        spin = self.maxspin
        with newconn.context(self.apiroutine):
            for i in range(0, self.maxretry):
                for m in newconn.execute_command(self.apiroutine, 'WATCH', *keys):
                    yield m
                for m in newconn.execute_command(self.apiroutine, 'MGET', *keys):
                    yield m
                values = self.apiroutine.retvalue
                values = [updater(v) for v in values]
                if timeout is None:
                    set_commands = (('MSET',) + tuple(itertools.chain.from_iterable(itertools.izip(keys, values))),)
                elif timeout <= 0:
                    set_commands = (('DEL',) + tuple(keys),)
                else:
                    ptimeout = int(timeout * 1000)
                    set_commands = (('MSET',) + tuple(itertools.chain.from_iterable(itertools.izip(keys, values))),) \
                                + tuple(('PEXPIRE', k, ptimeout) for k in keys)
                for m in newconn.batch_execute(self.apiroutine, ((('MULTI',),) + \
                                                                set_commands + \
                                                                ('EXEC',))):
                    yield m
                r = self.apiroutine.retvalue[-1]
                if r is not None:
                    # Succeeded
                    self.apiroutine.retvalue = values
                    raise StopIteration
                else:
                    # Watch keys changed, retry
                    if i > spin:
                        # Wait for a random small while
                        for m in self.apiroutine.waitWithTimeout(random.randrange(0, 1<<(i-spin)) * 0.05):
                            yield m
            raise RedisWriteConflictException('Transaction still fails after many retries: keys=' + repr(keys))
<|endoftext|>


<commit_before>'''
Created on 2016/3/21

:author: hubo
'''

from vlcp.server.module import Module, callAPI, depend, ModuleNotification,\
    ModuleLoadStateChanged, api, proxy
from vlcp.config import defaultconfig
import vlcp.service.connection.redisdb as redisdb
from vlcp.event.event import withIndices, Event
from vlcp.event.runnable import RoutineContainer
from vlcp.event.connection import ConnectionResetException
from vlcp.event.core import syscall_removequeue, QuitException
import json
from time import time
import functools
import uuid
import logging

@withIndices('notifier', 'transactid', 'keys', 'reason', 'fromself')
class UpdateNotification(Event):
    UPDATED = 'updated'
    RESTORED = 'restored'

@withIndices('notifier', 'stage')
class ModifyListen(Event):
    SUBSCRIBE = 'subscribe'
    LISTEN = 'listen'

def _delegate(func):
    @functools.wraps(func)
    def f(self, *args, **kwargs):
        for m in self.delegate(func(self, *args, **kwargs)):
            yield m
    return f

def _bytes(s):
    if isinstance(s, bytes):
        return s
    else:
        return s.encode('utf-8')

class _Notifier(RoutineContainer):
    _logger = logging.getLogger(__name__ + '.Notifier')
    def __init__(self, vhostbind, prefix, scheduler=None, daemon=False):
        RoutineContainer.__init__(self, scheduler=scheduler, daemon=daemon)
        self.vhostbind = vhostbind
        self.prefix = _bytes(prefix)
        self._matchers = {}
        self._publishkey = uuid.uuid1().hex
        self._publishno = 1
        self._publish_wait = set()
        self._matchadd_wait = set()
        self._matchremove_wait = set()
    def main(self):
        try:
            timestamp = '%012x' % (int(time() * 1000),) + '-'
            transactno = 1
            for m in callAPI(self, 'redisdb', 'getclient', {'vhost':self.vhostbind}):
                yield m
            client, encoder, decoder = self.retvalue
            self.subroutine(self._modifier(client), True, "modifierroutine")
            listen_modify = ModifyListen.createMatcher(self, ModifyListen.LISTEN)
            connection_down = client.subscribe_state_matcher(self, False)
            connection_up = client.subscribe_state_matcher(self, True)
            module_loaded = ModuleLoadStateChanged.createMatcher(state = ModuleLoadStateChanged.LOADED,
                                                 _ismatch = lambda x: x._instance.getServiceName() == 'redisdb')
            matchers = tuple(self._matchers.values()) + (listen_modify, connection_down)
            last_transact = None
            while True:
                yield matchers
                if self.matcher is listen_modify:
                    matchers = tuple(self._matchers.values()) + (listen_modify, connection_down)
                elif self.matcher is connection_down:
                    # Connection is down, wait for restore
                    # The module may be reloaded
                    recreate_matchers = False
                    last_transact = None
                    while True:
                        yield (connection_up, module_loaded)
                        if self.matcher is module_loaded:
                            self.terminate(self.modifierroutine)
                            for m in callAPI(self, 'redisdb', 'getclient', {'vhost':self.vhostbind}):
                                yield m
                            client, encoder, decoder = self.retvalue
                            # Recreate listeners
                            connection_down = client.subscribe_state_matcher(self, False)
                            connection_up = client.subscribe_state_matcher(self, True)
                            try:
                                for m in client.subscribe(self, *tuple(self._matchers.keys())):
                                    yield m
                            except Exception:
                                recreate_matchers = True
                                continue
                            else:
                                self._matchers = dict(zip(self._matchers.keys(), self.retvalue))
                                self.subroutine(self._modifier(client), True, "modifierroutine")
                                matchers = tuple(self._matchers.values()) + (listen_modify, connection_down)
                                break
                        else:
                            if recreate_matchers:
                                try:
                                    for m in client.subscribe(self, *[self.prefix + k for k in self._matchers.keys()]):
                                        yield m
                                except Exception:
                                    recreate_matchers = True
                                    continue
                                else:
                                    self._matchers = dict(zip(self._matchers.keys(), self.retvalue))
                                    self.subroutine(self._modifier(client), True, "modifierroutine")
                                    matchers = tuple(self._matchers.values()) + (listen_modify, connection_down)
                                    break
                            else:
                                matchers = tuple(self._matchers.values()) + (listen_modify, connection_down)
                                break
                    if self._publish_wait:
                        self.subroutine(self.publish())
                    transactid = '%s%016x' % (timestamp, transactno)
                    transactno += 1
                    self.subroutine(self.waitForSend(
                                UpdateNotification(self, transactid, tuple(self._matchers.keys()), UpdateNotification.RESTORED, False, extrainfo = None)
                                                                           ), False)
                else:
                    transact = decoder(self.event.message)
                    if transact['id'] == last_transact:
                        # Ignore duplicated updates
                        continue
                    last_transact = transact['id']
                    pubkey, sep, pubno = last_transact.partition('-')
                    fromself = (sep and pubkey == self._publishkey)
                    transactid = '%s%016x' % (timestamp, transactno)
                    transactno += 1
                    self.subroutine(self.waitForSend(
                                UpdateNotification(self, transactid, tuple(transact['keys']), UpdateNotification.UPDATED, fromself, extrainfo = transact.get('extrainfo'))
                                                                           ), False)
        finally:
            self.terminate(self.modifierroutine)
    def _modifier(self, client):
        try:
            modify_matcher = ModifyListen.createMatcher(self, ModifyListen.SUBSCRIBE)
            while True:
                try:
                    while self._matchadd_wait or self._matchremove_wait:
                        if self._matchadd_wait:
                            # Subscribe new keys
                            current_add = set(self._matchadd_wait)
                            self._matchadd_wait.clear()
                            add_keys = list(current_add.difference(self._matchers.keys()))
                            try:
                                for m in client.subscribe(self, *[self.prefix + k for k in add_keys]):
                                    yield m
                            except:
                                # Return to matchadd
                                self._matchadd_wait.update(current_add.difference(self._matchremove_wait))
                                raise
                            else:
                                self._matchers.update(zip(add_keys, self.retvalue))
                                for m in self.waitForSend(ModifyListen(self, ModifyListen.LISTEN)):
                                    yield m
                        if self._matchremove_wait:
                            # Unsubscribe keys
                            current_remove = set(self._matchremove_wait)
                            self._matchremove_wait.clear()
                            del_keys = list(current_remove.intersection(self._matchers.keys()))
                            try:
                                for m in client.unsubscribe(self, *[self.prefix + k for k in del_keys]):
                                    yield m
                            except:
                                # Return to matchremove
                                self._matchremove_wait.update(current_remove.difference(self._matchadd_wait))
                                raise
                            else:
                                for k in del_keys:
                                    del self._matchers[k]
                                for m in self.waitForSend(ModifyListen(self, ModifyListen.LISTEN)):
                                    yield m
                    yield (modify_matcher,)
                except (IOError, ConnectionResetException):
                    # Wait for connection resume
                    connection_up = client.subscribe_state_matcher(self)
                    yield (connection_up,)
        finally:
            self.subroutine(self._clearup(client, list(self._matchers.keys())))
    def _clearup(self, client, keys):
        try:
            if not self.scheduler.quitting:
                for m in client.unsubscribe(self, *[self.prefix + k for k in keys]):
                    yield m
        except Exception:
            pass
    def add_listen(self, *keys):
        keys = [_bytes(k) for k in keys]
        self._matchremove_wait.difference_update(keys)
        self._matchadd_wait.update(keys)
        for m in self.waitForSend(ModifyListen(self, ModifyListen.SUBSCRIBE)):
            yield m
    def remove_listen(self, *keys):
        keys = [_bytes(k) for k in keys]
        self._matchadd_wait.difference_update(keys)
        self._matchremove_wait.update(keys)
        for m in self.waitForSend(ModifyListen(self, ModifyListen.SUBSCRIBE)):
            yield m
    @_delegate
    def publish(self, keys = (), extrainfo = None):
        keys = [_bytes(k) for k in keys]
        if self._publish_wait:
            merged_keys = list(self._publish_wait.union(keys))
            self._publish_wait.clear()
        else:
            merged_keys = list(keys)
        if not merged_keys:
            raise StopIteration
        for m in callAPI(self, 'redisdb', 'getclient', {'vhost':self.vhostbind}):
            yield m
        client, encoder, _ = self.retvalue
        transactid = '%s-%016x' % (self._publishkey, self._publishno)
        self._publishno += 1
        msg = encoder({'id':transactid, 'keys':merged_keys, 'extrainfo': extrainfo})
        try:
            for m in client.batch_execute(self, *((('MULTI',),) +
                                                tuple(('PUBLISH', self.prefix + k, msg) for k in merged_keys) +
                                                (('EXEC',),))):
                yield m
        except (IOError, ConnectionResetException):
            self._logger.warning('Following keys are not published because exception occurred, delay to next publish: %r', merged_keys, exc_info = True)
            self._publish_wait.update(merged_keys)
        else:
            self._publish_wait.clear()
    def notification_matcher(self, fromself = None):
        if fromself is None:
            return UpdateNotification.createMatcher(self)
        else:
            return UpdateNotification.createMatcher(notifier = self, fromself = fromself)

@defaultconfig
@depend(redisdb.RedisDB)
class RedisNotifier(Module):
    """
    Update notification with Redis Pub/Sub
    """
    _default_vhostbind = ''
    _default_prefix = 'vlcp.updatenotifier.'
    def __init__(self, server):
        Module.__init__(self, server)
        self.createAPI(api(self.createnotifier))
    def load(self, container):
        self.scheduler.queue.addSubQueue(999, ModifyListen.createMatcher(), "redisnotifier_modifylisten")
        for m in Module.load(self, container):
            yield m
    def unload(self, container, force=False):
        for m in Module.unload(self, container, force=force):
            yield m
        for m in container.syscall_noreturn(syscall_removequeue(self.scheduler.queue, "redisnotifier_modifylisten")):
            yield m
    def createnotifier(self):
        "Create a new notifier object"
        n = _Notifier(self.vhostbind, self.prefix, self.scheduler)
        n.start()
        return n

UpdateNotifier = proxy('UpdateNotifier', RedisNotifier)
<commit_msg>use zlib compress<commit_after>'''
Created on 2016/3/21

:author: hubo
'''

from vlcp.server.module import Module, callAPI, depend, ModuleNotification,\
    ModuleLoadStateChanged, api, proxy
from vlcp.config import defaultconfig
import vlcp.service.connection.redisdb as redisdb
from vlcp.event.event import withIndices, Event
from vlcp.event.runnable import RoutineContainer
from vlcp.event.connection import ConnectionResetException
from vlcp.event.core import syscall_removequeue, QuitException
import json
from time import time
from zlib import compress, decompress
import functools
import uuid
import logging

@withIndices('notifier', 'transactid', 'keys', 'reason', 'fromself')
class UpdateNotification(Event):
    UPDATED = 'updated'
    RESTORED = 'restored'

@withIndices('notifier', 'stage')
class ModifyListen(Event):
    SUBSCRIBE = 'subscribe'
    LISTEN = 'listen'

def _delegate(func):
    @functools.wraps(func)
    def f(self, *args, **kwargs):
        for m in self.delegate(func(self, *args, **kwargs)):
            yield m
    return f

def _bytes(s):
    if isinstance(s, bytes):
        return s
    else:
        return s.encode('utf-8')

class _Notifier(RoutineContainer):
    _logger = logging.getLogger(__name__ + '.Notifier')
    def __init__(self, vhostbind, prefix, scheduler=None, daemon=False):
        RoutineContainer.__init__(self, scheduler=scheduler, daemon=daemon)
        self.vhostbind = vhostbind
        self.prefix = _bytes(prefix)
        self._matchers = {}
        self._publishkey = uuid.uuid1().hex
        self._publishno = 1
        self._publish_wait = set()
        self._matchadd_wait = set()
        self._matchremove_wait = set()
    def main(self):
        try:
            timestamp = '%012x' % (int(time() * 1000),) + '-'
            transactno = 1
            for m in callAPI(self, 'redisdb', 'getclient', {'vhost':self.vhostbind}):
                yield m
            client, encoder, decoder = self.retvalue
            self.subroutine(self._modifier(client), True, "modifierroutine")
            listen_modify = ModifyListen.createMatcher(self, ModifyListen.LISTEN)
            connection_down = client.subscribe_state_matcher(self, False)
            connection_up = client.subscribe_state_matcher(self, True)
            module_loaded = ModuleLoadStateChanged.createMatcher(state = ModuleLoadStateChanged.LOADED,
                                                 _ismatch = lambda x: x._instance.getServiceName() == 'redisdb')
            matchers = tuple(self._matchers.values()) + (listen_modify, connection_down)
            last_transact = None
            while True:
                yield matchers
                if self.matcher is listen_modify:
                    matchers = tuple(self._matchers.values()) + (listen_modify, connection_down)
                elif self.matcher is connection_down:
                    # Connection is down, wait for restore
                    # The module may be reloaded
                    recreate_matchers = False
                    last_transact = None
                    while True:
                        yield (connection_up, module_loaded)
                        if self.matcher is module_loaded:
                            self.terminate(self.modifierroutine)
                            for m in callAPI(self, 'redisdb', 'getclient', {'vhost':self.vhostbind}):
                                yield m
                            client, encoder, decoder = self.retvalue
                            # Recreate listeners
                            connection_down = client.subscribe_state_matcher(self, False)
                            connection_up = client.subscribe_state_matcher(self, True)
                            try:
                                for m in client.subscribe(self, *tuple(self._matchers.keys())):
                                    yield m
                            except Exception:
                                recreate_matchers = True
                                continue
                            else:
                                self._matchers = dict(zip(self._matchers.keys(), self.retvalue))
                                self.subroutine(self._modifier(client), True, "modifierroutine")
                                matchers = tuple(self._matchers.values()) + (listen_modify, connection_down)
                                break
                        else:
                            if recreate_matchers:
                                try:
                                    for m in client.subscribe(self, *[self.prefix + k for k in self._matchers.keys()]):
                                        yield m
                                except Exception:
                                    recreate_matchers = True
                                    continue
                                else:
                                    self._matchers = dict(zip(self._matchers.keys(), self.retvalue))
                                    self.subroutine(self._modifier(client), True, "modifierroutine")
                                    matchers = tuple(self._matchers.values()) + (listen_modify, connection_down)
                                    break
                            else:
                                matchers = tuple(self._matchers.values()) + (listen_modify, connection_down)
                                break
                    if self._publish_wait:
                        self.subroutine(self.publish())
                    transactid = '%s%016x' % (timestamp, transactno)
                    transactno += 1
                    self.subroutine(self.waitForSend(
                                UpdateNotification(self, transactid, tuple(self._matchers.keys()), UpdateNotification.RESTORED, False, extrainfo = None)
                                                                           ), False)
                else:
                    transact = decoder(decompress(self.event.message))
                    if transact['id'] == last_transact:
                        # Ignore duplicated updates
                        continue
                    last_transact = transact['id']
                    pubkey, sep, pubno = last_transact.partition('-')
                    fromself = (sep and pubkey == self._publishkey)
                    transactid = '%s%016x' % (timestamp, transactno)
                    transactno += 1
                    self.subroutine(self.waitForSend(
                                UpdateNotification(self, transactid, tuple(transact['keys']), UpdateNotification.UPDATED, fromself, extrainfo = transact.get('extrainfo'))
                                                                           ), False)
        finally:
            self.terminate(self.modifierroutine)
    def _modifier(self, client):
        try:
            modify_matcher = ModifyListen.createMatcher(self, ModifyListen.SUBSCRIBE)
            while True:
                try:
                    while self._matchadd_wait or self._matchremove_wait:
                        if self._matchadd_wait:
                            # Subscribe new keys
                            current_add = set(self._matchadd_wait)
                            self._matchadd_wait.clear()
                            add_keys = list(current_add.difference(self._matchers.keys()))
                            try:
                                for m in client.subscribe(self, *[self.prefix + k for k in add_keys]):
                                    yield m
                            except:
                                # Return to matchadd
                                self._matchadd_wait.update(current_add.difference(self._matchremove_wait))
                                raise
                            else:
                                self._matchers.update(zip(add_keys, self.retvalue))
                                for m in self.waitForSend(ModifyListen(self, ModifyListen.LISTEN)):
                                    yield m
                        if self._matchremove_wait:
                            # Unsubscribe keys
                            current_remove = set(self._matchremove_wait)
                            self._matchremove_wait.clear()
                            del_keys = list(current_remove.intersection(self._matchers.keys()))
                            try:
                                for m in client.unsubscribe(self, *[self.prefix + k for k in del_keys]):
                                    yield m
                            except:
                                # Return to matchremove
                                self._matchremove_wait.update(current_remove.difference(self._matchadd_wait))
                                raise
                            else:
                                for k in del_keys:
                                    del self._matchers[k]
                                for m in self.waitForSend(ModifyListen(self, ModifyListen.LISTEN)):
                                    yield m
                    yield (modify_matcher,)
                except (IOError, ConnectionResetException):
                    # Wait for connection resume
                    connection_up = client.subscribe_state_matcher(self)
                    yield (connection_up,)
        finally:
            self.subroutine(self._clearup(client, list(self._matchers.keys())))
    def _clearup(self, client, keys):
        try:
            if not self.scheduler.quitting:
                for m in client.unsubscribe(self, *[self.prefix + k for k in keys]):
                    yield m
        except Exception:
            pass
    def add_listen(self, *keys):
        keys = [_bytes(k) for k in keys]
        self._matchremove_wait.difference_update(keys)
        self._matchadd_wait.update(keys)
        for m in self.waitForSend(ModifyListen(self, ModifyListen.SUBSCRIBE)):
            yield m
    def remove_listen(self, *keys):
        keys = [_bytes(k) for k in keys]
        self._matchadd_wait.difference_update(keys)
        self._matchremove_wait.update(keys)
        for m in self.waitForSend(ModifyListen(self, ModifyListen.SUBSCRIBE)):
            yield m
    @_delegate
    def publish(self, keys = (), extrainfo = None):
        keys = [_bytes(k) for k in keys]
        if self._publish_wait:
            merged_keys = list(self._publish_wait.union(keys))
            self._publish_wait.clear()
        else:
            merged_keys = list(keys)
        if not merged_keys:
            raise StopIteration
        for m in callAPI(self, 'redisdb', 'getclient', {'vhost':self.vhostbind}):
            yield m
        client, encoder, _ = self.retvalue
        transactid = '%s-%016x' % (self._publishkey, self._publishno)
        self._publishno += 1
        msg = compress(encoder({'id':transactid, 'keys':merged_keys, 'extrainfo': extrainfo}))
        try:
            for m in client.batch_execute(self, *((('MULTI',),) +
                                                tuple(('PUBLISH', self.prefix + k, msg) for k in merged_keys) +
                                                (('EXEC',),))):
                yield m
        except (IOError, ConnectionResetException):
            self._logger.warning('Following keys are not published because exception occurred, delay to next publish: %r', merged_keys, exc_info = True)
            self._publish_wait.update(merged_keys)
        else:
            self._publish_wait.clear()
    def notification_matcher(self, fromself = None):
        if fromself is None:
            return UpdateNotification.createMatcher(self)
        else:
            return UpdateNotification.createMatcher(notifier = self, fromself = fromself)

@defaultconfig
@depend(redisdb.RedisDB)
class RedisNotifier(Module):
    """
    Update notification with Redis Pub/Sub
    """
    _default_vhostbind = ''
    _default_prefix = 'vlcp.updatenotifier.'
    def __init__(self, server):
        Module.__init__(self, server)
        self.createAPI(api(self.createnotifier))
    def load(self, container):
        self.scheduler.queue.addSubQueue(999, ModifyListen.createMatcher(), "redisnotifier_modifylisten")
        for m in Module.load(self, container):
            yield m
    def unload(self, container, force=False):
        for m in Module.unload(self, container, force=force):
            yield m
        for m in container.syscall_noreturn(syscall_removequeue(self.scheduler.queue, "redisnotifier_modifylisten")):
            yield m
    def createnotifier(self):
        "Create a new notifier object"
        n = _Notifier(self.vhostbind, self.prefix, self.scheduler)
        n.start()
        return n

UpdateNotifier = proxy('UpdateNotifier', RedisNotifier)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Author: Jayden Navarro

from profile_tool import profile_tool

class ProjectNameError( Exception ):
   pass

def check_groups( func ):
   def inner( self, *args, **kwargs ):
      if self.groups_ is None:
         raise ValueError( 'groups is None' )
      else:
         return func( self, *args, **kwargs )
   return inner

class proj_helper(object):
   def __init__( self, profile, groups=None ):
      if profile:
         self.ptool_ = profile_tool( profile )
      else:
         self.ptool_ = profile_tool()
      self.groups_ = groups

   def read_profile( self ):
      self.groups_ = self.ptool_.readGroups()

   @check_groups
   def write_profile( self ):
      self.ptool_.writeGroups( self.groups_ )

   @property
   @check_groups
   def default_project( self ):
      try:
         return self.groups_[ 'default_project' ][ 0 ][ 0 ]
      except:
         return ''

   @default_project.setter
   @check_groups
   def default_project( self, proj_name ):
      self.groups_[ 'default_project' ] = [ ( proj_name, ) ]

   @property
   @check_groups
   def projects( self ):
      try:
         return dict( [ ( a, ( b, c ) ) for ( a, b, c ) in
                        self.groups_[ 'projects' ] ] )
      except:
         return {}

   @projects.setter
   @check_groups
   def projects( self, projects ):
      self.groups_[ 'projects' ] = [ ( a, b, c ) for ( a, ( b, c ) )
                                     in projects.items() ]

   @check_groups
   def get_full_project_name( self, proj_name ):
      projects = self.projects

      # check for exact matches first
      exact_matches = [ name for name in projects if proj_name == name ]
      if len( exact_matches ) == 1:
         return exact_matches[ 0 ]

      # check for substring matches
      full_names = [ name for name in projects if proj_name in name ]
      if len( full_names ) == 1:
         return full_names[ 0 ]
      elif len( full_names ) == 0:
         raise ProjectNameError( 'No matching projects found' )
      else:
         raise ProjectNameError( 'Ambigous project name, following match: %s' \
                                 % ', '.join( full_names ) )
<commit_msg>Changed default project tools file path<commit_after>#!/usr/bin/env python
# Author: Jayden Navarro

import os
from profile_tool import profile_tool

class ProjectNameError( Exception ):
   pass

def check_groups( func ):
   def inner( self, *args, **kwargs ):
      if self.groups_ is None:
         raise ValueError( 'groups is None' )
      else:
         return func( self, *args, **kwargs )
   return inner

class proj_helper(object):
   def __init__( self, profile, groups=None ):
      if profile:
         self.ptool_ = profile_tool( profile_path=profile )
      else:
         # default to `~/.projects.conf`
         homeDir = os.path.expanduser( '~' )
         self.ptool_ = profile_tool( profile_path=( homeDir + '/.projects.conf' ) )
      self.groups_ = groups

   def read_profile( self ):
      self.groups_ = self.ptool_.readGroups()

   @check_groups
   def write_profile( self ):
      self.ptool_.writeGroups( self.groups_ )

   @property
   @check_groups
   def default_project( self ):
      try:
         return self.groups_[ 'default_project' ][ 0 ][ 0 ]
      except:
         return ''

   @default_project.setter
   @check_groups
   def default_project( self, proj_name ):
      self.groups_[ 'default_project' ] = [ ( proj_name, ) ]

   @property
   @check_groups
   def projects( self ):
      try:
         return dict( [ ( a, ( b, c ) ) for ( a, b, c ) in
                        self.groups_[ 'projects' ] ] )
      except:
         return {}

   @projects.setter
   @check_groups
   def projects( self, projects ):
      self.groups_[ 'projects' ] = [ ( a, b, c ) for ( a, ( b, c ) )
                                     in projects.items() ]

   @check_groups
   def get_full_project_name( self, proj_name ):
      projects = self.projects

      # check for exact matches first
      exact_matches = [ name for name in projects if proj_name == name ]
      if len( exact_matches ) == 1:
         return exact_matches[ 0 ]

      # check for substring matches
      full_names = [ name for name in projects if proj_name in name ]
      if len( full_names ) == 1:
         return full_names[ 0 ]
      elif len( full_names ) == 0:
         raise ProjectNameError( 'No matching projects found' )
      else:
         raise ProjectNameError( 'Ambigous project name, following match: %s' \
                                 % ', '.join( full_names ) )
<|endoftext|>


<commit_before># -*- test-case-name: vumi.transports.devnull.tests.test_devnull -*-
import random
import uuid

from vumi import log

from vumi.transports.base import Transport
from vumi.message import TransportUserMessage

from twisted.internet.defer import inlineCallbacks

class DevNullTransport(Transport):
    """
    DevNullTransport for messages that need fake delivery to networks.
    Useful for testing.

    Configuration parameters:

    :param transport_type str:
        The transport type to emulate, defaults to sms.
    :param failure_rate float:
        How many messages should be treated as failures.
        Float value between 0.0 and 1.0.
    :param reply_rate float:
        For how many messages should we generate a reply?
        Float value between 0.0 and 1.0.
    :param reply_copy str:
        What copy should be sent as the reply, defaults to echo-ing the content
        of the outbound message.
    """

    def validate_config(self):
        self.transport_type = self.config.get('transport_type', 'sms')
        self.failure_rate = float(self.config['failure_rate'])
        self.reply_rate = float(self.config['reply_rate'])
        self.reply_copy = self.config.get('reply_copy')

    def setup_transport(self):
        pass

    def teardown_transport(self):
        pass

    @inlineCallbacks
    def handle_outbound_message(self, message):
        dr = ('failed' if random.random() < self.failure_rate
                else 'delivered')
        log.info('MT %(dr)s: %(from_addr)s -> %(to_addr)s: %(content)s' % {
            'dr': dr,
            'from_addr': message['from_addr'],
            'to_addr': message['to_addr'],
            'content': message['content'],
            })
        yield self.publish_ack(message['message_id'],
            sent_message_id=uuid.uuid4().hex)
        yield self.publish_delivery_report(message['message_id'], dr)
        if random.random() < self.reply_rate:
            reply_copy = self.reply_copy or message['content']
            log.info('MO %(from_addr)s -> %(to_addr)s: %(content)s' % {
                'from_addr': message['to_addr'],
                'to_addr': message['from_addr'],
                'content': reply_copy,
                })
            yield self.publish_message(
                message_id=uuid.uuid4().hex,
                content=reply_copy,
                to_addr=message['from_addr'],
                from_addr=message['to_addr'],
                provider='devnull',
                session_event=TransportUserMessage.SESSION_NONE,
                transport_type=self.transport_type,
                transport_metadata={})
<commit_msg>docstring fixes<commit_after># -*- test-case-name: vumi.transports.devnull.tests.test_devnull -*-
import random
import uuid

from vumi import log

from vumi.transports.base import Transport
from vumi.message import TransportUserMessage

from twisted.internet.defer import inlineCallbacks

class DevNullTransport(Transport):
    """
    DevNullTransport for messages that need fake delivery to networks.
    Useful for testing.

    Configuration parameters:

    :type transport_type: str
    :param transport_type:
        The transport type to emulate, defaults to sms.
    :type failure_rate: float
    :param failure_rate:
        How many messages should be treated as failures.
        Float value between 0.0 and 1.0.
    :type reply_rate: float
    :param reply_rate:
        For how many messages should we generate a reply?
        Float value between 0.0 and 1.0.
    :type reply_copy: str
    :param reply_copy:
        What copy should be sent as the reply, defaults to echo-ing the content
        of the outbound message.
    """

    def validate_config(self):
        self.transport_type = self.config.get('transport_type', 'sms')
        self.failure_rate = float(self.config['failure_rate'])
        self.reply_rate = float(self.config['reply_rate'])
        self.reply_copy = self.config.get('reply_copy')

    def setup_transport(self):
        pass

    def teardown_transport(self):
        pass

    @inlineCallbacks
    def handle_outbound_message(self, message):
        dr = ('failed' if random.random() < self.failure_rate
                else 'delivered')
        log.info('MT %(dr)s: %(from_addr)s -> %(to_addr)s: %(content)s' % {
            'dr': dr,
            'from_addr': message['from_addr'],
            'to_addr': message['to_addr'],
            'content': message['content'],
            })
        yield self.publish_ack(message['message_id'],
            sent_message_id=uuid.uuid4().hex)
        yield self.publish_delivery_report(message['message_id'], dr)
        if random.random() < self.reply_rate:
            reply_copy = self.reply_copy or message['content']
            log.info('MO %(from_addr)s -> %(to_addr)s: %(content)s' % {
                'from_addr': message['to_addr'],
                'to_addr': message['from_addr'],
                'content': reply_copy,
                })
            yield self.publish_message(
                message_id=uuid.uuid4().hex,
                content=reply_copy,
                to_addr=message['from_addr'],
                from_addr=message['to_addr'],
                provider='devnull',
                session_event=TransportUserMessage.SESSION_NONE,
                transport_type=self.transport_type,
                transport_metadata={})
<|endoftext|>


<commit_before>from __future__ import absolute_import, unicode_literals

from django.conf import settings
from django.contrib.auth import get_user_model
from django.db import connections
from django.shortcuts import render
from django.template.loader import render_to_string

from wagtail.wagtailadmin.navigation import get_explorable_root_page
from wagtail.wagtailadmin.site_summary import SiteSummaryPanel
from wagtail.wagtailcore import hooks
from wagtail.wagtailcore.models import Page, PageRevision, UserPagePermissionsProxy

User = get_user_model()

# Panels for the homepage

class UpgradeNotificationPanel(object):
    name = 'upgrade_notification'
    order = 100

    def __init__(self, request):
        self.request = request

    def render(self):
        if self.request.user.is_superuser and getattr(settings, "WAGTAIL_ENABLE_UPDATE_CHECK", True):
            return render_to_string('wagtailadmin/home/upgrade_notification.html', {}, request=self.request)
        else:
            return ""

class PagesForModerationPanel(object):
    name = 'pages_for_moderation'
    order = 200

    def __init__(self, request):
        self.request = request
        user_perms = UserPagePermissionsProxy(request.user)
        self.page_revisions_for_moderation = (user_perms.revisions_for_moderation()
                                              .select_related('page', 'user').order_by('-created_at'))

    def render(self):
        return render_to_string('wagtailadmin/home/pages_for_moderation.html', {
            'page_revisions_for_moderation': self.page_revisions_for_moderation,
        }, request=self.request)

class RecentEditsPanel(object):
    name = 'recent_edits'
    order = 300

    def __init__(self, request):
        self.request = request

        # Last n edited pages
        last_edits = PageRevision.objects.raw(
            """
            SELECT wp.* FROM
                wagtailcore_pagerevision wp JOIN (
                    SELECT max(created_at) AS max_created_at, page_id FROM
                        wagtailcore_pagerevision WHERE user_id = %s GROUP BY page_id ORDER BY max_created_at DESC LIMIT %s
                ) AS max_rev ON max_rev.max_created_at = wp.created_at ORDER BY wp.created_at DESC
             """, [
                User._meta.pk.get_db_prep_value(self.request.user.pk, connections['default']),
                getattr(settings, 'WAGTAILADMIN_RECENT_EDITS_LIMIT', 5)
            ]
        )
        last_edits = list(last_edits)
        page_keys = [pr.page.pk for pr in last_edits]
        specific_pages = Page.objects.filter(pk__in=page_keys).specific()
        pages = {p.pk: p for p in specific_pages}
        self.last_edits = [
            [review, pages.get(review.page.pk)] for review in last_edits
        ]

    def render(self):
        return render_to_string('wagtailadmin/home/recent_edits.html', {
            'last_edits': list(self.last_edits),
        }, request=self.request)

def home(request):

    panels = [
        SiteSummaryPanel(request),
        UpgradeNotificationPanel(request),
        PagesForModerationPanel(request),
        RecentEditsPanel(request),
    ]

    for fn in hooks.get_hooks('construct_homepage_panels'):
        fn(request, panels)

    root_page = get_explorable_root_page(request.user)
    if root_page:
        root_site = root_page.get_site()
    else:
        root_site = None

    real_site_name = None
    if root_site:
        real_site_name = root_site.site_name if root_site.site_name else root_site.hostname

    return render(request, "wagtailadmin/home.html", {
        'root_page': root_page,
        'root_site': root_site,
        'site_name': real_site_name if real_site_name else settings.WAGTAIL_SITE_NAME,
        'panels': sorted(panels, key=lambda p: p.order),
        'user': request.user
    })

def error_test(request):
    raise Exception("This is a test of the emergency broadcast system.")
<commit_msg>Replace 'recent edits' query with a pure ORM version<commit_after>from __future__ import absolute_import, unicode_literals

from django.conf import settings
from django.contrib.auth import get_user_model
from django.db.models import Max
from django.shortcuts import render
from django.template.loader import render_to_string

from wagtail.wagtailadmin.navigation import get_explorable_root_page
from wagtail.wagtailadmin.site_summary import SiteSummaryPanel
from wagtail.wagtailcore import hooks
from wagtail.wagtailcore.models import Page, PageRevision, UserPagePermissionsProxy

User = get_user_model()

# Panels for the homepage

class UpgradeNotificationPanel(object):
    name = 'upgrade_notification'
    order = 100

    def __init__(self, request):
        self.request = request

    def render(self):
        if self.request.user.is_superuser and getattr(settings, "WAGTAIL_ENABLE_UPDATE_CHECK", True):
            return render_to_string('wagtailadmin/home/upgrade_notification.html', {}, request=self.request)
        else:
            return ""

class PagesForModerationPanel(object):
    name = 'pages_for_moderation'
    order = 200

    def __init__(self, request):
        self.request = request
        user_perms = UserPagePermissionsProxy(request.user)
        self.page_revisions_for_moderation = (user_perms.revisions_for_moderation()
                                              .select_related('page', 'user').order_by('-created_at'))

    def render(self):
        return render_to_string('wagtailadmin/home/pages_for_moderation.html', {
            'page_revisions_for_moderation': self.page_revisions_for_moderation,
        }, request=self.request)

class RecentEditsPanel(object):
    name = 'recent_edits'
    order = 300

    def __init__(self, request):
        self.request = request

        # Last n edited pages
        edit_count = getattr(settings, 'WAGTAILADMIN_RECENT_EDITS_LIMIT', 5)
        last_edits_dates = (PageRevision.objects.filter(user=self.request.user)
                            .values('page_id').annotate(latest_date=Max('created_at'))
                            .order_by('-latest_date').values('latest_date')[:edit_count])
        last_edits = PageRevision.objects.filter(created_at__in=last_edits_dates).order_by('-created_at')
        page_keys = [pr.page_id for pr in last_edits]
        pages = Page.objects.specific().in_bulk(page_keys)
        self.last_edits = [
            [review, pages.get(review.page.pk)] for review in last_edits
        ]

    def render(self):
        return render_to_string('wagtailadmin/home/recent_edits.html', {
            'last_edits': list(self.last_edits),
        }, request=self.request)

def home(request):

    panels = [
        SiteSummaryPanel(request),
        UpgradeNotificationPanel(request),
        PagesForModerationPanel(request),
        RecentEditsPanel(request),
    ]

    for fn in hooks.get_hooks('construct_homepage_panels'):
        fn(request, panels)

    root_page = get_explorable_root_page(request.user)
    if root_page:
        root_site = root_page.get_site()
    else:
        root_site = None

    real_site_name = None
    if root_site:
        real_site_name = root_site.site_name if root_site.site_name else root_site.hostname

    return render(request, "wagtailadmin/home.html", {
        'root_page': root_page,
        'root_site': root_site,
        'site_name': real_site_name if real_site_name else settings.WAGTAIL_SITE_NAME,
        'panels': sorted(panels, key=lambda p: p.order),
        'user': request.user
    })

def error_test(request):
    raise Exception("This is a test of the emergency broadcast system.")
<|endoftext|>


<commit_before>#!/usr/bin/env python
import MySQLdb
import sys
import os

db_user = os.getenv('DB_USER')
db_pass = os.getenv('DB_PASS')
db_host = os.getenv('DB_HOST')
db_name = os.getenv('DB_NAME')

db = MySQLdb.connect(db_host, db_user, db_pass, db_name)

# days to keep
days = 14
# batch size for deletes
limit = 1000

print "Cleaning instances older than %s days , in groups of %s" % (days, limit)

finished = 0
cleaned = 0

while (finished < 1):
    db.autocommit(False)
    cursor = db.cursor(MySQLdb.cursors.DictCursor)
    s = """SELECT * FROM instances WHERE deleted !=0 and deleted_at <
    DATE_SUB(NOW(), INTERVAL %s day) order by deleted_at
    LIMIT %s""" % (days, limit)
    cursor.execute(s)
    results = cursor.fetchall()
    ids = []
    for row in results:
        ids.append(row["uuid"])

    if len(ids) < 1:
        print "finished!"
        finished = 1
        break
    try:
        print "Removing %s " % (', ').join(ids)
        sql_str = ','.join(['%s'] * len(ids))
        cursor.execute("""SELECT id from instance_actions where
                        instance_uuid  in (%s)""" % (sql_str), ids)
        iae = []
        results = cursor.fetchall()
        for event in results:
                iae.append(event["id"])

        if len(iae):
                iae_str = ','.join(['%s'] * len(iae))
                cursor.execute("""DELETE from instance_actions_events
                                WHERE action_id in (%s)""" % (iae_str), iae)

        cursor.execute("DELETE from instance_actions where instance_uuid in (%s)" % (sql_str), ids)
        cursor.execute("DELETE from instance_info_caches where instance_uuid in (%s)" % (sql_str), ids)
        cursor.execute("DELETE from instance_system_metadata where instance_uuid in (%s)" % (sql_str), ids)
        cursor.execute("DELETE from instance_faults where instance_uuid in (%s)" % (sql_str), ids)
        cursor.execute("DELETE from instance_metadata where instance_uuid in (%s)" % (sql_str), ids)
        cursor.execute("DELETE from block_device_mapping where instance_uuid in (%s)" % (sql_str), ids)
        cursor.execute("DELETE from migrations where instance_uuid in (%s)" % (sql_str), ids)
        cursor.execute("DELETE from instances where uuid in (%s)" % (sql_str), ids)
        db.commit()
    except db.Error, e:
        print "Rollback %s %s" % (e[0], e[1])
        db.rollback()
        sys.exit(1)

    cleaned += limit
    print "Finished cleaning %s " % cleaned

print "Cleaning up reservations / stats"
try:
    db.autocommit(False)
    cursor = db.cursor(MySQLdb.cursors.DictCursor)
    cursor.execute("""DELETE FROM reservations WHERE expire
                    < DATE_SUB(NOW(), INTERVAL %s day)""" % days)
    cursor.execute("""DELETE FROM compute_node_stats WHERE deleted_at
                    < DATE_SUB(NOW(), INTERVAL %s day)""" % days)
    db.commit()
except db.Error, e:
    print "Rollback %s %s" % (e[0], e[1])
    db.rollback()
    sys.exit(1)

db.close()
<commit_msg>Remove compute_node_stats pruning<commit_after>#!/usr/bin/env python
import MySQLdb
import sys
import os

db_user = os.getenv('DB_USER')
db_pass = os.getenv('DB_PASS')
db_host = os.getenv('DB_HOST')
db_name = os.getenv('DB_NAME')

db = MySQLdb.connect(db_host, db_user, db_pass, db_name)

# days to keep
days = 14
# batch size for deletes
limit = 1000

print "Cleaning instances older than %s days , in groups of %s" % (days, limit)

finished = 0
cleaned = 0

while (finished < 1):
    db.autocommit(False)
    cursor = db.cursor(MySQLdb.cursors.DictCursor)
    s = """SELECT * FROM instances WHERE deleted !=0 and deleted_at <
    DATE_SUB(NOW(), INTERVAL %s day) order by deleted_at
    LIMIT %s""" % (days, limit)
    cursor.execute(s)
    results = cursor.fetchall()
    ids = []
    for row in results:
        ids.append(row["uuid"])

    if len(ids) < 1:
        print "finished!"
        finished = 1
        break
    try:
        print "Removing %s " % (', ').join(ids)
        sql_str = ','.join(['%s'] * len(ids))
        cursor.execute("""SELECT id from instance_actions where
                        instance_uuid  in (%s)""" % (sql_str), ids)
        iae = []
        results = cursor.fetchall()
        for event in results:
                iae.append(event["id"])

        if len(iae):
                iae_str = ','.join(['%s'] * len(iae))
                cursor.execute("""DELETE from instance_actions_events
                                WHERE action_id in (%s)""" % (iae_str), iae)

        cursor.execute("DELETE from instance_actions where instance_uuid in (%s)" % (sql_str), ids)
        cursor.execute("DELETE from instance_info_caches where instance_uuid in (%s)" % (sql_str), ids)
        cursor.execute("DELETE from instance_system_metadata where instance_uuid in (%s)" % (sql_str), ids)
        cursor.execute("DELETE from instance_faults where instance_uuid in (%s)" % (sql_str), ids)
        cursor.execute("DELETE from instance_metadata where instance_uuid in (%s)" % (sql_str), ids)
        cursor.execute("DELETE from block_device_mapping where instance_uuid in (%s)" % (sql_str), ids)
        cursor.execute("DELETE from migrations where instance_uuid in (%s)" % (sql_str), ids)
        cursor.execute("DELETE from instances where uuid in (%s)" % (sql_str), ids)
        db.commit()
    except db.Error, e:
        print "Rollback %s %s" % (e[0], e[1])
        db.rollback()
        sys.exit(1)

    cleaned += limit
    print "Finished cleaning %s " % cleaned

print "Cleaning up reservations / stats"
try:
    db.autocommit(False)
    cursor = db.cursor(MySQLdb.cursors.DictCursor)
    cursor.execute("""DELETE FROM reservations WHERE expire
                    < DATE_SUB(NOW(), INTERVAL %s day)""" % days)
    db.commit()
except db.Error, e:
    print "Rollback %s %s" % (e[0], e[1])
    db.rollback()
    sys.exit(1)

db.close()
<|endoftext|>