<commit_before># -*- coding: utf-8 -*-
# Copyright (C) 2012-2015 MUJIN Inc

import threading

from . import zmq
from . import TimeoutError, GetMonotonicTime

import logging
log = logging.getLogger(__name__)

class ZmqSocketPool(object):

    _url = None  # url that sockets should connect to
    _ctx = None  # the context to use
    _ctxown = None  # the context owned exclusively
    _timeout = None  # timeout waiting for either send on receive while a socket is in the polling state
    _limit = None  # limit on number of socket alive at any time, None means unlimited

    _isok = False  # whether it is time to terminate
    _poller = None  # znq poller that tracks sockets in self._pollingsockets
    _sockets = None  # all sockets alive, a dictionary mapping from socket to True
    _pollingsockets = None  # sockets that are in the polling state, waiting for send or receive, a dictionary mapping from socket to timestamp when added to the poller
    _availablesockets = None  # list of sockets that are ready for use immediately

    _acquirecount = 0  # number of times a socket is acuired
    _releasecount = 0  # number of times a socket is released
    _opencount = 0  # number of times we opened a new socket
    _closecount = 0  # number of times we closed a socket

    def __init__(self, url, ctx=None, timeout=10.0, limit=None):
        """creates a socket pool, the pool can lease out socket for send and recv.
        if caller only does send but not recv, the socket will not be reused until internally the pool recv data and discard them.

        :param url: url for sockets to connect to
        :param ctx: optionally force socket to use provided zmq context instead of creating a new zmq context
        :param timeout: defaults to 10 seconds, specifies how long should we be polling the socket, until we consider it to be dead, note this is not a blocking timeout.
        :param limit: limit on number of socket alive at any time, None means unlimited
        """
        self._url = url
        self._ctx = ctx
        if self._ctx is None:
            self._ctxown = zmq.Context()
            self._ctx = self._ctxown
        self._timeout = timeout
        self._limit = limit

        self._isok = True
        self._poller = zmq.Poller()
        self._sockets = {}
        self._pollingsockets = {}
        self._availablesockets = []

        self._acquirecount = 0
        self._releasecount = 0
        self._opencount = 0
        self._closecount = 0

    def __del__(self):
        self.Destroy()

    def Destroy(self):
        self.SetDestroy()

        # at this point, all sockets should have been released
        assert(self._acquirecount == self._releasecount)
        sockets = list(self._pollingsockets.keys())
        while len(sockets) > 0:
            self._StopPollingSocket(sockets.pop())

        # close all socket ever created
        sockets = list(self._sockets.keys())
        while len(sockets) > 0:
            self._CloseSocket(sockets.pop())
        assert(self._opencount == self._closecount)

        # log.debug('sockets: opened = %d, closed = %d, acquired = %d, released = %d', self._opencount, self._closecount, self._acquirecount, self._releasecount)

        if self._ctxown is not None:
            try:
                self._ctxown.destroy()
            except Exception:
                log.exception('caught exception when destroying context')
            self._ctxown = None
        self._ctx = None

    def SetDestroy(self):
        # make sure no new socket can be created
        self._isok = False

        # make sure no one can acquire a socket now
        self._availablesockets = []

    def _OpenSocket(self):
        if not self._isok:
            return None

        socket = self._ctx.socket(zmq.REQ)
        socket.connect(self._url)
        assert(socket not in self._sockets)
        self._sockets[socket] = True
        self._opencount += 1

        # log.debug('opened a socket, url = %s opened = %d, closed = %d', self._url, self._opencount, self._closecount)
        return socket

    def _CloseSocket(self, socket):
        assert(socket in self._sockets)
        self._closecount += 1
        del self._sockets[socket]
        try:
            # make sure we do not linger when closing socket
            socket.close(linger=0)
        except Exception as e:
            log.exception('caught exception when closing socket: %s', e)

    def _StartPollingSocket(self, socket):
        assert(socket not in self._pollingsockets)
        self._pollingsockets[socket] = GetMonotonicTime()
        self._poller.register(socket, zmq.POLLIN | zmq.POLLOUT)

    def _StopPollingSocket(self, socket):
        assert(socket in self._pollingsockets)
        self._poller.unregister(socket)
        del self._pollingsockets[socket]

    def _Poll(self, timeout=0):
        """spin once and does internal polling of sockets
        """
        now = GetMonotonicTime()

        # poll for receives, non blocking if timeout is 0
        for socket, event in self._poller.poll(timeout):
            if (event & zmq.POLLIN) == zmq.POLLIN:
                # log.debug('a socket is ready for receive, url = %s, polling = %d, availble = %d', self._url, len(self._pollingsockets), len(self._availablesockets))

                # at least one message can be received without blocking
                try:
                    socket.recv(zmq.NOBLOCK)
                except Exception as e:
                    # when error occur, throw the socket away
                    log.exception('caught exception when recv: %s', e)
                    self._StopPollingSocket(socket)
                    self._CloseSocket(socket)
                    continue

                # reset the timestamp since we just called recv
                self._pollingsockets[socket] = now

        # poll again for send, non blocking if timeout is 0, some previously in recv state socket may now be available for sending
        for socket, event in self._poller.poll(timeout):
            if (event & zmq.POLLOUT) == zmq.POLLOUT:
                # log.debug('a socket is ready for send, url = %s, polling = %d, availble = %d', self._url, len(self._pollingsockets), len(self._availablesockets))

                # at least one message can be sent to socket without blocking
                self._StopPollingSocket(socket)
                self._availablesockets.append(socket)

        # check for socket that stayed in the polling state for too long
        timedoutsockets = []
        for socket, timestamp in self._pollingsockets.iteritems():
            if now - timestamp > self._timeout:
                timedoutsockets.append(socket)

        # close timed out sockets
        while len(timedoutsockets) > 0:
            socket = timedoutsockets.pop()
            self._StopPollingSocket(socket)
            self._CloseSocket(socket)

    def AcquireSocket(self, timeout=None, checkpreemptfn=None):
        """acquire a socket from the list of availble sockets for sending
        """
        # first we try a non blocking poll
        self._Poll(timeout=0)

        starttime = GetMonotonicTime()
        while self._isok:

            # if a socket is available, use it
            if len(self._availablesockets) > 0:
                socket = self._availablesockets.pop()
                self._acquirecount += 1
                return socket

            # if we don't have available socket but we have quota to create new socket, create a new one
            if self._limit is None or len(self._availablesockets) + len(self._pollingsockets) < self._limit:
                socket = self._OpenSocket()
                self._acquirecount += 1
                return socket

            # check for timeout
            elapsedtime = GetMonotonicTime() - starttime
            if timeout is not None and elapsedtime > timeout:
                raise TimeoutError(u'Timed out waiting for a socket to %s to become available after %f seconds' % (self._url, elapsedtime))

            if checkpreemptfn is not None:
                checkpreemptfn()

            # otherwise, wait by a small blocking poll
            self._Poll(timeout=50)

        # TODO: raise UserInterrupt
        return None

    def ReleaseSocket(self, socket, reuse=True):
        """release a socket after use, if caller did not call recv, the pool will take care of that
        """
        if socket is not None:
            self._releasecount += 1
            if self._isok and reuse:
                self._StartPollingSocket(socket)
            else:
                self._CloseSocket(socket)

class ZmqClient(object):

    _hostname = None
    _port = None
    _url = None
    _pool = None
    _socket = None
    _isok = False
    _callerthread = None  # last caller thread

    def __init__(self, hostname='', port=0, ctx=None, limit=100, url=None, checkpreemptfn=None, reusetimeout=10.0):
        """creates a new zmq client, uses zmq req socket over tcp

        :param hostname: hostname or ip to connect to
        :param port: port to connect to
        :param ctx: optionally specifies a zmq context to use, if not provided, a new zmq context will be created
        :param limit: defaults to 100, limit the number of underlying zmq socket to create, usually one socket will actually be used, but if server times out frequently and you use fireandforget, then this limit caps the number of sockets we can use
        :param url: allow passing of zmq socket url instead of hostname and port
        """

        self._hostname = hostname
        self._port = int(port)
        self._url = url
        if self._url is None:
            self._url = 'tcp://%s:%d' % (self._hostname, self._port)

        self._pool = ZmqSocketPool(self._url, ctx=ctx, limit=limit, timeout=reusetimeout)
        self._socket = None
        self._isok = True
        self._checkpreemptfn = checkpreemptfn

    def __del__(self):
        self.Destroy()

    def Destroy(self):
        self.SetDestroy()

        if self._pool is not None:
            self._ReleaseSocket()
            self._pool.Destroy()
            self._pool = None

    def SetDestroy(self):
        self._isok = False
        if self._pool is not None:
            self._pool.SetDestroy()

    def GetHostname(self):
        """returns the hostname given when constructing the client
        """
        return self._hostname

    def GetPort(self):
        """returns the port given when constructing the client
        """
        return self._port

    # TODO: this is for backward compatibility, remove once all callers are updated
    @property
    def hostname(self):
        return self._hostname

    # TODO: this is for backward compatibility, remove once all callers are updated
    @property
    def port(self):
        return self._port

    def _CheckCallerThread(self, context=None):
        """catch bad caller who use zmq client from multiple threads and causes random race conditions.
        """
        callerthread = repr(threading.current_thread())
        oldcallerthread = self._callerthread
        self._callerthread = callerthread
        if oldcallerthread is not None:
            # assert oldcallerthread == callerthread, 'zmqclient used from multiple threads: previously = %s, now = %s' % (oldcallerthread, callerthread)
            if oldcallerthread != callerthread:
                log.error('zmqclient used from multiple threads in %s, this is a bug in the caller: previously = %s, now = %s' % (context, oldcallerthread, callerthread))

    def _AcquireSocket(self, timeout=None, checkpreempt=True):
        # if we were holding on to a socket before, release it before acquiring one
        self._ReleaseSocket()
        self._socket = self._pool.AcquireSocket(timeout=timeout, checkpreemptfn=self._checkpreemptfn if checkpreempt else None)

    def _ReleaseSocket(self):
        if self._socket is not None:
            self._pool.ReleaseSocket(self._socket)
            self._socket = None

    def SetPreemptFn(self, checkpreemptfn):
        self._checkpreemptfn = checkpreemptfn

    def SendCommand(self, command, timeout=10.0, blockwait=True, fireandforget=False, sendjson=True, recvjson=True, checkpreempt=True):
        """sends command via established zmq socket

        :param command: command in json format
        :param timeout: if None, block. If >= 0, use as timeout
        :param blockwait: if True (default), will call receive also, otherwise, caller needs to call ReceiveCommand later
        :param fireandforget: if True, will send command and immediately return without trying to receive, blockwait will be set to False
        :param sendjson: if True (default), will send data as json
        :param recvjson: if True (default), will parse received data as json

        :return: returns the response from the zmq server in json format if blockwait is True
        """
        # log.debug('Sending command via ZMQ: %s', command)

        self._CheckCallerThread(command)

        if fireandforget:
            blockwait = False

        # acquire a socket for sending
        self._AcquireSocket(timeout=timeout, checkpreempt=checkpreempt)

        # we may be exiting, the pool refused to give us a socket
        if not self._isok or self._socket is None:
            # TODO: raise UserInterrupt
            return None

        releasesocket = True
        try:
            # send phase
            starttime = GetMonotonicTime()
            while self._isok:
                # timeout checking
                elapsedtime = GetMonotonicTime() - starttime
                if timeout is not None and elapsedtime > timeout:
                    raise TimeoutError(u'Timed out trying to send to %s after %f seconds' % (self._url, elapsedtime))

                if checkpreempt and self._checkpreemptfn is not None:
                    self._checkpreemptfn()

                # poll to see if we can send, if not, loop
                waitingevents = self._socket.poll(50, zmq.POLLOUT)
                if (waitingevents & zmq.POLLOUT) != zmq.POLLOUT:
                    continue

                if sendjson:
                    self._socket.send_json(command, zmq.NOBLOCK)
                else:
                    self._socket.send(command, zmq.NOBLOCK)

                # break when successfully sent
                break

            # for fire and forget, no need to receive
            if fireandforget:
                return None

            # keep the socket and let caller call receive
            if not blockwait:
                releasesocket = False
                return None

            # receive
            return self.ReceiveCommand(timeout=timeout, recvjson=recvjson, checkpreempt=checkpreempt)

        finally:
            # release socket
            if releasesocket:
                self._ReleaseSocket()

        # TODO: raise UserInterrupt
        return None

    def IsWaitingReply(self):
        return self._socket is not None

    def ReceiveCommand(self, timeout=10.0, recvjson=True, checkpreempt=True):
        """receive response to a previous SendCommand call, SendCommand must be called with blockwait=False and fireandforget=False

        :param timeout: if None, block. If >= 0, use as timeout
        :param recvjson: if True (default), will parse received data as json

        :return: returns the recv or recv_json response
        """
        self._CheckCallerThread('ReceiveCommand')

        # should have called SendCommand with blockwait=False first
        assert(self._socket is not None)

        try:
            # receive phase
            starttime = GetMonotonicTime()
            while self._isok:
                # timeout checking
                elapsedtime = GetMonotonicTime() - starttime
                if timeout is not None and elapsedtime > timeout:
                    raise TimeoutError(u'Timed out to get response from %s after %f seconds (timeout=%f)' % (self._url, elapsedtime, timeout))

                if checkpreempt and self._checkpreemptfn is not None:
                    self._checkpreemptfn()

                # poll to see if something has been received, if received nothing, loop
                startpolltime = GetMonotonicTime()
                waitingevents = self._socket.poll(50, zmq.POLLIN)
                endpolltime = GetMonotonicTime()
                if endpolltime - startpolltime > 0.2:  # due to python delays sometimes this can be 0.11s
                    log.critical('polling time took %fs!', endpolltime - startpolltime)
                if (waitingevents & zmq.POLLIN) != zmq.POLLIN:
                    continue

                if recvjson:
                    return self._socket.recv_json(zmq.NOBLOCK)
                else:
                    return self._socket.recv(zmq.NOBLOCK)

        finally:
            # release socket
            self._ReleaseSocket()

        # TODO: raise UserInterrupt
        return None
<commit_msg>save context for tracking problems faster<commit_after># -*- coding: utf-8 -*-
# Copyright (C) 2012-2015 MUJIN Inc

import threading

from . import zmq
from . import TimeoutError, GetMonotonicTime

import logging
log = logging.getLogger(__name__)

class ZmqSocketPool(object):

    _url = None  # url that sockets should connect to
    _ctx = None  # the context to use
    _ctxown = None  # the context owned exclusively
    _timeout = None  # timeout waiting for either send on receive while a socket is in the polling state
    _limit = None  # limit on number of socket alive at any time, None means unlimited

    _isok = False  # whether it is time to terminate
    _poller = None  # znq poller that tracks sockets in self._pollingsockets
    _sockets = None  # all sockets alive, a dictionary mapping from socket to True
    _pollingsockets = None  # sockets that are in the polling state, waiting for send or receive, a dictionary mapping from socket to timestamp when added to the poller
    _availablesockets = None  # list of sockets that are ready for use immediately

    _acquirecount = 0  # number of times a socket is acuired
    _releasecount = 0  # number of times a socket is released
    _opencount = 0  # number of times we opened a new socket
    _closecount = 0  # number of times we closed a socket

    def __init__(self, url, ctx=None, timeout=10.0, limit=None):
        """creates a socket pool, the pool can lease out socket for send and recv.
        if caller only does send but not recv, the socket will not be reused until internally the pool recv data and discard them.

        :param url: url for sockets to connect to
        :param ctx: optionally force socket to use provided zmq context instead of creating a new zmq context
        :param timeout: defaults to 10 seconds, specifies how long should we be polling the socket, until we consider it to be dead, note this is not a blocking timeout.
        :param limit: limit on number of socket alive at any time, None means unlimited
        """
        self._url = url
        self._ctx = ctx
        if self._ctx is None:
            self._ctxown = zmq.Context()
            self._ctx = self._ctxown
        self._timeout = timeout
        self._limit = limit

        self._isok = True
        self._poller = zmq.Poller()
        self._sockets = {}
        self._pollingsockets = {}
        self._availablesockets = []

        self._acquirecount = 0
        self._releasecount = 0
        self._opencount = 0
        self._closecount = 0

    def __del__(self):
        self.Destroy()

    def Destroy(self):
        self.SetDestroy()

        # at this point, all sockets should have been released
        assert(self._acquirecount == self._releasecount)
        sockets = list(self._pollingsockets.keys())
        while len(sockets) > 0:
            self._StopPollingSocket(sockets.pop())

        # close all socket ever created
        sockets = list(self._sockets.keys())
        while len(sockets) > 0:
            self._CloseSocket(sockets.pop())
        assert(self._opencount == self._closecount)

        # log.debug('sockets: opened = %d, closed = %d, acquired = %d, released = %d', self._opencount, self._closecount, self._acquirecount, self._releasecount)

        if self._ctxown is not None:
            try:
                self._ctxown.destroy()
            except Exception:
                log.exception('caught exception when destroying context')
            self._ctxown = None
        self._ctx = None

    def SetDestroy(self):
        # make sure no new socket can be created
        self._isok = False

        # make sure no one can acquire a socket now
        self._availablesockets = []

    def _OpenSocket(self):
        if not self._isok:
            return None

        socket = self._ctx.socket(zmq.REQ)
        socket.connect(self._url)
        assert(socket not in self._sockets)
        self._sockets[socket] = True
        self._opencount += 1

        # log.debug('opened a socket, url = %s opened = %d, closed = %d', self._url, self._opencount, self._closecount)
        return socket

    def _CloseSocket(self, socket):
        assert(socket in self._sockets)
        self._closecount += 1
        del self._sockets[socket]
        try:
            # make sure we do not linger when closing socket
            socket.close(linger=0)
        except Exception as e:
            log.exception('caught exception when closing socket: %s', e)

    def _StartPollingSocket(self, socket):
        assert(socket not in self._pollingsockets)
        self._pollingsockets[socket] = GetMonotonicTime()
        self._poller.register(socket, zmq.POLLIN | zmq.POLLOUT)

    def _StopPollingSocket(self, socket):
        assert(socket in self._pollingsockets)
        self._poller.unregister(socket)
        del self._pollingsockets[socket]

    def _Poll(self, timeout=0):
        """spin once and does internal polling of sockets
        """
        now = GetMonotonicTime()

        # poll for receives, non blocking if timeout is 0
        for socket, event in self._poller.poll(timeout):
            if (event & zmq.POLLIN) == zmq.POLLIN:
                # log.debug('a socket is ready for receive, url = %s, polling = %d, availble = %d', self._url, len(self._pollingsockets), len(self._availablesockets))

                # at least one message can be received without blocking
                try:
                    socket.recv(zmq.NOBLOCK)
                except Exception as e:
                    # when error occur, throw the socket away
                    log.exception('caught exception when recv: %s', e)
                    self._StopPollingSocket(socket)
                    self._CloseSocket(socket)
                    continue

                # reset the timestamp since we just called recv
                self._pollingsockets[socket] = now

        # poll again for send, non blocking if timeout is 0, some previously in recv state socket may now be available for sending
        for socket, event in self._poller.poll(timeout):
            if (event & zmq.POLLOUT) == zmq.POLLOUT:
                # log.debug('a socket is ready for send, url = %s, polling = %d, availble = %d', self._url, len(self._pollingsockets), len(self._availablesockets))

                # at least one message can be sent to socket without blocking
                self._StopPollingSocket(socket)
                self._availablesockets.append(socket)

        # check for socket that stayed in the polling state for too long
        timedoutsockets = []
        for socket, timestamp in self._pollingsockets.iteritems():
            if now - timestamp > self._timeout:
                timedoutsockets.append(socket)

        # close timed out sockets
        while len(timedoutsockets) > 0:
            socket = timedoutsockets.pop()
            self._StopPollingSocket(socket)
            self._CloseSocket(socket)

    def AcquireSocket(self, timeout=None, checkpreemptfn=None):
        """acquire a socket from the list of availble sockets for sending
        """
        # first we try a non blocking poll
        self._Poll(timeout=0)

        starttime = GetMonotonicTime()
        while self._isok:

            # if a socket is available, use it
            if len(self._availablesockets) > 0:
                socket = self._availablesockets.pop()
                self._acquirecount += 1
                return socket

            # if we don't have available socket but we have quota to create new socket, create a new one
            if self._limit is None or len(self._availablesockets) + len(self._pollingsockets) < self._limit:
                socket = self._OpenSocket()
                self._acquirecount += 1
                return socket

            # check for timeout
            elapsedtime = GetMonotonicTime() - starttime
            if timeout is not None and elapsedtime > timeout:
                raise TimeoutError(u'Timed out waiting for a socket to %s to become available after %f seconds' % (self._url, elapsedtime))

            if checkpreemptfn is not None:
                checkpreemptfn()

            # otherwise, wait by a small blocking poll
            self._Poll(timeout=50)

        # TODO: raise UserInterrupt
        return None

    def ReleaseSocket(self, socket, reuse=True):
        """release a socket after use, if caller did not call recv, the pool will take care of that
        """
        if socket is not None:
            self._releasecount += 1
            if self._isok and reuse:
                self._StartPollingSocket(socket)
            else:
                self._CloseSocket(socket)

class ZmqClient(object):

    _hostname = None
    _port = None
    _url = None
    _pool = None
    _socket = None
    _isok = False
    _callerthread = None  # last caller thread
    _callercontext = None # the context of the last caller

    def __init__(self, hostname='', port=0, ctx=None, limit=100, url=None, checkpreemptfn=None, reusetimeout=10.0):
        """creates a new zmq client, uses zmq req socket over tcp

        :param hostname: hostname or ip to connect to
        :param port: port to connect to
        :param ctx: optionally specifies a zmq context to use, if not provided, a new zmq context will be created
        :param limit: defaults to 100, limit the number of underlying zmq socket to create, usually one socket will actually be used, but if server times out frequently and you use fireandforget, then this limit caps the number of sockets we can use
        :param url: allow passing of zmq socket url instead of hostname and port
        """

        self._hostname = hostname
        self._port = int(port)
        self._url = url
        if self._url is None:
            self._url = 'tcp://%s:%d' % (self._hostname, self._port)

        self._pool = ZmqSocketPool(self._url, ctx=ctx, limit=limit, timeout=reusetimeout)
        self._socket = None
        self._isok = True
        self._checkpreemptfn = checkpreemptfn

    def __del__(self):
        self.Destroy()

    def Destroy(self):
        self.SetDestroy()

        if self._pool is not None:
            self._ReleaseSocket()
            self._pool.Destroy()
            self._pool = None

    def SetDestroy(self):
        self._isok = False
        if self._pool is not None:
            self._pool.SetDestroy()

    def GetHostname(self):
        """returns the hostname given when constructing the client
        """
        return self._hostname

    def GetPort(self):
        """returns the port given when constructing the client
        """
        return self._port

    # TODO: this is for backward compatibility, remove once all callers are updated
    @property
    def hostname(self):
        return self._hostname

    # TODO: this is for backward compatibility, remove once all callers are updated
    @property
    def port(self):
        return self._port

    def _CheckCallerThread(self, context=None):
        """catch bad caller who use zmq client from multiple threads and causes random race conditions.
        """
        callerthread = repr(threading.current_thread())
        oldcallerthread = self._callerthread
        oldcallercontext = self._callercontext
        self._callerthread = callerthread
        self._callercontext = context
        if oldcallerthread is not None:
            # assert oldcallerthread == callerthread, 'zmqclient used from multiple threads: previously = %s, now = %s' % (oldcallerthread, callerthread)
            if oldcallerthread != callerthread:
                log.error('zmqclient used from multiple threads in %s, this is a bug in the caller: previously = %s, now = %s. oldcontext=%r', context, oldcallerthread, callerthread, oldcallercontext)

    def _AcquireSocket(self, timeout=None, checkpreempt=True):
        # if we were holding on to a socket before, release it before acquiring one
        self._ReleaseSocket()
        self._socket = self._pool.AcquireSocket(timeout=timeout, checkpreemptfn=self._checkpreemptfn if checkpreempt else None)

    def _ReleaseSocket(self):
        if self._socket is not None:
            self._pool.ReleaseSocket(self._socket)
            self._socket = None

    def SetPreemptFn(self, checkpreemptfn):
        self._checkpreemptfn = checkpreemptfn

    def SendCommand(self, command, timeout=10.0, blockwait=True, fireandforget=False, sendjson=True, recvjson=True, checkpreempt=True):
        """sends command via established zmq socket

        :param command: command in json format
        :param timeout: if None, block. If >= 0, use as timeout
        :param blockwait: if True (default), will call receive also, otherwise, caller needs to call ReceiveCommand later
        :param fireandforget: if True, will send command and immediately return without trying to receive, blockwait will be set to False
        :param sendjson: if True (default), will send data as json
        :param recvjson: if True (default), will parse received data as json

        :return: returns the response from the zmq server in json format if blockwait is True
        """
        # log.debug('Sending command via ZMQ: %s', command)

        self._CheckCallerThread(command)

        if fireandforget:
            blockwait = False

        # acquire a socket for sending
        self._AcquireSocket(timeout=timeout, checkpreempt=checkpreempt)

        # we may be exiting, the pool refused to give us a socket
        if not self._isok or self._socket is None:
            # TODO: raise UserInterrupt
            return None

        releasesocket = True
        try:
            # send phase
            starttime = GetMonotonicTime()
            while self._isok:
                # timeout checking
                elapsedtime = GetMonotonicTime() - starttime
                if timeout is not None and elapsedtime > timeout:
                    raise TimeoutError(u'Timed out trying to send to %s after %f seconds' % (self._url, elapsedtime))

                if checkpreempt and self._checkpreemptfn is not None:
                    self._checkpreemptfn()

                # poll to see if we can send, if not, loop
                waitingevents = self._socket.poll(50, zmq.POLLOUT)
                if (waitingevents & zmq.POLLOUT) != zmq.POLLOUT:
                    continue

                if sendjson:
                    self._socket.send_json(command, zmq.NOBLOCK)
                else:
                    self._socket.send(command, zmq.NOBLOCK)

                # break when successfully sent
                break

            # for fire and forget, no need to receive
            if fireandforget:
                return None

            # keep the socket and let caller call receive
            if not blockwait:
                releasesocket = False
                return None

            # receive
            return self.ReceiveCommand(timeout=timeout, recvjson=recvjson, checkpreempt=checkpreempt)

        finally:
            # release socket
            if releasesocket:
                self._ReleaseSocket()

        # TODO: raise UserInterrupt
        return None

    def IsWaitingReply(self):
        return self._socket is not None

    def ReceiveCommand(self, timeout=10.0, recvjson=True, checkpreempt=True):
        """receive response to a previous SendCommand call, SendCommand must be called with blockwait=False and fireandforget=False

        :param timeout: if None, block. If >= 0, use as timeout
        :param recvjson: if True (default), will parse received data as json

        :return: returns the recv or recv_json response
        """
        self._CheckCallerThread('ReceiveCommand')

        # should have called SendCommand with blockwait=False first
        assert(self._socket is not None)

        try:
            # receive phase
            starttime = GetMonotonicTime()
            while self._isok:
                # timeout checking
                elapsedtime = GetMonotonicTime() - starttime
                if timeout is not None and elapsedtime > timeout:
                    raise TimeoutError(u'Timed out to get response from %s after %f seconds (timeout=%f)' % (self._url, elapsedtime, timeout))

                if checkpreempt and self._checkpreemptfn is not None:
                    self._checkpreemptfn()

                # poll to see if something has been received, if received nothing, loop
                startpolltime = GetMonotonicTime()
                waitingevents = self._socket.poll(50, zmq.POLLIN)
                endpolltime = GetMonotonicTime()
                if endpolltime - startpolltime > 0.2:  # due to python delays sometimes this can be 0.11s
                    log.critical('polling time took %fs!', endpolltime - startpolltime)
                if (waitingevents & zmq.POLLIN) != zmq.POLLIN:
                    continue

                if recvjson:
                    return self._socket.recv_json(zmq.NOBLOCK)
                else:
                    return self._socket.recv(zmq.NOBLOCK)

        finally:
            # release socket
            self._ReleaseSocket()

        # TODO: raise UserInterrupt
        return None
<|endoftext|>


<commit_before>#!/usr/bin/env python

import cgitb
cgitb.enable()

import cgi
import shelve
import html
import math

MESSAGES_PER_PAGE = 20
MIME_LINK = html.a("[Should I be excited about this?]",
                   href="https://www.reddit.com/r/Solving_A858/comments/"
                        "24vml1/mime_type/chb5k2e?context=3")

form = cgi.FieldStorage()
db = shelve.open("../archive.db", 'r')

def reddit_user(username):
	return html.a("/u/" + username,
	              href="https://www.reddit.com/u/" + username)

def credits():
	return "By %s and %s" % (reddit_user("fragglet"),
	                         reddit_user("kamalist"))

def expander(name, inner):
	control = html.span("►",
	                    id="control-%s" % name,
	                    onclick='expand("%s");' % name)
	div = html.div(inner,
	               id="inner-%s" % name,
	               style="display: none;")

	return control + div

def statistics(post):
	"""Print statistics about post."""
	data = post["analysis"]["data"]

	lenfield = "Length: %i bytes " % len(data)
	if ((len(data) - 8) % 32) == 0:
		lenfield += "(= 8 + %i * 32)" % ((len(data) - 8) / 32)
	else:
		lenfield += "(does NOT match pattern!)"

	return html.ul(
		lenfield,
		"Statistical distribution: %s" % \
		    post["analysis"]["distribution"],
		"Entropy: %s" % \
		    post["analysis"]["entropy"],
		"Mean: %0.2f" % \
		    post["analysis"]["mean"],
		"Variance: %0.2f" % \
		    post["analysis"]["variance"],
		"Stdev: %0.2f" % \
		    post["analysis"]["sigma"],
		"Skewness: %0.2f" % \
		    post["analysis"]["sk"],
		"Kurtosis: %0.2f" % \
		    post["analysis"]["ex"],
	)

def histogram_grid(post):
	histogram = post["analysis"]["histogram"]
	id = post["data"]["id"]

        n = len(post["analysis"]["data"])
	p = 1 / 256.0
	expected = n * p
	stddev = math.sqrt(n * p * (1 - p))
	if stddev == 0:
		stddev = 1

	rows = []
	for y in range(16):
		row = []
		for x in range(16):
			i = y * 16 + x
			value = histogram[i]
			offset = float(value - expected) / (stddev * 8)
			bright = min(max(int(offset * 128 + 127), 0), 255)
			bgcolor = "#%02x%02x%02x" % (bright, bright, bright)
			td = html.td(" " * 5, bgcolor=bgcolor,
			             title="0x%02x: %i" % (i, value))
			row.append(td)

		rows.append(html.tr(*row))

	return expander("histogram-grid-%s" % id, html.table(*rows))

def timezone_link(zone):
	return html.a(zone, href="https://en.wikipedia.org/wiki/%s" % zone)

def time_data(post):
	timedata = post["analysis"]["time"]
	return html.ul(
		"Time in post title: %s" % timedata["title_time_str"],
		"Posted to Reddit: %s UTC" % timedata["post_time_str"],
		"Identified time zone: %s" % \
		    timezone_link(timedata["timezone"]),
		"Post delay: %i seconds" % timedata["post_delay"],
	)

def file_type(post):
	mime = post["analysis"]["mime"]
	if mime == "data":
		return "unknown"
	else:
		return mime + " " + MIME_LINK

def plain_text(post):
	"""Print plain text of post."""
	id = post["data"]["id"]
	text = post["data"]["selftext"]
	# Format text so that it is (usually?) four groups wide.
	# This makes it 256 bits per line.
	plaintext = html.div(html.tt(html.escape(text)), style="width: 38em;")
	return expander("plaintext-%s" % id, plaintext)

def hex_dump(post):
	"""Print hex dump of post."""
	id = post["data"]["id"]
	text = post["analysis"]["hexdump"]
	# Format text so that it is (usually?) four groups wide.
	# This makes it 256 bits per line.
	plaintext = html.pre(html.escape(text))
	return expander("hexdump-%s" % id, plaintext)

decoders = [
	("Statistics",        statistics),
	("Date/time",         time_data),
	("File type (MIME)",  file_type),
	("Text",              plain_text),
	("Hex dump",          hex_dump),
	("Histogram grid",    histogram_grid),
]

def format_post(post):
	id = post["data"]["id"]
	title = post["data"]["title"]
	url = post["data"]["url"]

	def formatted(x):
		name, callback = x
		return "%s: %s" % (name, callback(post))

	items = map(formatted, decoders) + [
		html.a("Permalink", href="/?id=%s" % id)
	]

	return html.div(html.a(name=id),
	                html.h3(html.a(html.escape(title), href=url)),
	                html.ul(*items),
	                id="post-%s" % id)

def gen_pager(messages, position):
	num_pages = (len(messages) + MESSAGES_PER_PAGE - 1) / MESSAGES_PER_PAGE
	this_page = position / MESSAGES_PER_PAGE

	result = []
	for i in range(num_pages):
		label = "%i" % (i + 1)
		if this_page == i:
			result.append(label)
		else:
			if i == 0:
				url = "/"
			else:
				url = "/?start=%i" % (i * MESSAGES_PER_PAGE)

			result.append(html.a(label, href=url))

	return html.div("Page: ", *(" ".join(result)),
	                style="float: right; width: 50%; text-align: right;")

def list_messages():
	# Which messages to show?
	messages = list(reversed(sorted(db.keys())))
	if "start" in form:
		start = int(form["start"].value)
	else:
		start = 0
	page_messages = messages[start:start + MESSAGES_PER_PAGE]
	pager = gen_pager(messages, start)

	return html.body(
		pager,
		html.h1("a858 auto-analysis"),
		credits(),
		html.div(*map(lambda key: format_post(db[key]),
		              page_messages)),
		pager
	)

def single_message():
	id = form["id"].value
	for key in db.keys():
		if ("-%s-" % id) in key:
			post = db[key]
			break
	else:
		return "Unknown message: %s" % id

	return html.body(
		html.h1(html.escape(post["data"]["title"])),
		format_post(post),
	)

def gen_html():
	if "id" in form:
		body = single_message()
	else:
		body = list_messages()

	return html.html(
		html.head(
			html.title("a858 auto-analysis"),
			html.script(language='javascript', src='functions.js')
		),
		body
	)

print "Content-Type: text/html"
print

print gen_html()

<commit_msg>Limit number of pages shown.<commit_after>#!/usr/bin/env python

import cgitb
cgitb.enable()

import cgi
import shelve
import html
import math

MAX_PAGES = 20
MESSAGES_PER_PAGE = 20
MIME_LINK = html.a("[Should I be excited about this?]",
                   href="https://www.reddit.com/r/Solving_A858/comments/"
                        "24vml1/mime_type/chb5k2e?context=3")

form = cgi.FieldStorage()
db = shelve.open("../archive.db", 'r')

def reddit_user(username):
	return html.a("/u/" + username,
	              href="https://www.reddit.com/u/" + username)

def credits():
	return "By %s and %s" % (reddit_user("fragglet"),
	                         reddit_user("kamalist"))

def expander(name, inner):
	control = html.span("►",
	                    id="control-%s" % name,
	                    onclick='expand("%s");' % name)
	div = html.div(inner,
	               id="inner-%s" % name,
	               style="display: none;")

	return control + div

def statistics(post):
	"""Print statistics about post."""
	data = post["analysis"]["data"]

	lenfield = "Length: %i bytes " % len(data)
	if ((len(data) - 8) % 32) == 0:
		lenfield += "(= 8 + %i * 32)" % ((len(data) - 8) / 32)
	else:
		lenfield += "(does NOT match pattern!)"

	return html.ul(
		lenfield,
		"Statistical distribution: %s" % \
		    post["analysis"]["distribution"],
		"Entropy: %s" % \
		    post["analysis"]["entropy"],
		"Mean: %0.2f" % \
		    post["analysis"]["mean"],
		"Variance: %0.2f" % \
		    post["analysis"]["variance"],
		"Stdev: %0.2f" % \
		    post["analysis"]["sigma"],
		"Skewness: %0.2f" % \
		    post["analysis"]["sk"],
		"Kurtosis: %0.2f" % \
		    post["analysis"]["ex"],
	)

def histogram_grid(post):
	histogram = post["analysis"]["histogram"]
	id = post["data"]["id"]

        n = len(post["analysis"]["data"])
	p = 1 / 256.0
	expected = n * p
	stddev = math.sqrt(n * p * (1 - p))
	if stddev == 0:
		stddev = 1

	rows = []
	for y in range(16):
		row = []
		for x in range(16):
			i = y * 16 + x
			value = histogram[i]
			offset = float(value - expected) / (stddev * 8)
			bright = min(max(int(offset * 128 + 127), 0), 255)
			bgcolor = "#%02x%02x%02x" % (bright, bright, bright)
			td = html.td(" " * 5, bgcolor=bgcolor,
			             title="0x%02x: %i" % (i, value))
			row.append(td)

		rows.append(html.tr(*row))

	return expander("histogram-grid-%s" % id, html.table(*rows))

def timezone_link(zone):
	return html.a(zone, href="https://en.wikipedia.org/wiki/%s" % zone)

def time_data(post):
	timedata = post["analysis"]["time"]
	return html.ul(
		"Time in post title: %s" % timedata["title_time_str"],
		"Posted to Reddit: %s UTC" % timedata["post_time_str"],
		"Identified time zone: %s" % \
		    timezone_link(timedata["timezone"]),
		"Post delay: %i seconds" % timedata["post_delay"],
	)

def file_type(post):
	mime = post["analysis"]["mime"]
	if mime == "data":
		return "unknown"
	else:
		return mime + " " + MIME_LINK

def plain_text(post):
	"""Print plain text of post."""
	id = post["data"]["id"]
	text = post["data"]["selftext"]
	# Format text so that it is (usually?) four groups wide.
	# This makes it 256 bits per line.
	plaintext = html.div(html.tt(html.escape(text)), style="width: 38em;")
	return expander("plaintext-%s" % id, plaintext)

def hex_dump(post):
	"""Print hex dump of post."""
	id = post["data"]["id"]
	text = post["analysis"]["hexdump"]
	# Format text so that it is (usually?) four groups wide.
	# This makes it 256 bits per line.
	plaintext = html.pre(html.escape(text))
	return expander("hexdump-%s" % id, plaintext)

decoders = [
	("Statistics",        statistics),
	("Date/time",         time_data),
	("File type (MIME)",  file_type),
	("Text",              plain_text),
	("Hex dump",          hex_dump),
	("Histogram grid",    histogram_grid),
]

def format_post(post):
	id = post["data"]["id"]
	title = post["data"]["title"]
	url = post["data"]["url"]

	def formatted(x):
		name, callback = x
		return "%s: %s" % (name, callback(post))

	items = map(formatted, decoders) + [
		html.a("Permalink", href="/?id=%s" % id)
	]

	return html.div(html.a(name=id),
	                html.h3(html.a(html.escape(title), href=url)),
	                html.ul(*items),
	                id="post-%s" % id)

def page_link(label, offset):
	if offset == 0:
		url = "/"
	else:
		url = "/?start=%i" % offset

	return html.a(label, href=url)

def gen_pager(messages, position):
	total_pages = ((len(messages) + MESSAGES_PER_PAGE - 1)
	               / MESSAGES_PER_PAGE)
	this_page = position / MESSAGES_PER_PAGE

	start_page = min(max(this_page - (MAX_PAGES / 2), 0),
	                 total_pages - MAX_PAGES)
	num_pages = min(total_pages, MAX_PAGES)

	result = []
	for page in range(start_page, start_page + num_pages):
		label = "%i" % (page + 1)
		if this_page == page:
			result.append(label)
		else:
			result.append(page_link(label,
			                        page * MESSAGES_PER_PAGE))

	if start_page > 0:
		result.insert(0, "...")
		result.insert(0, page_link("« First", 0))
	if start_page + num_pages < total_pages:
		result.append("...")
		result.append(page_link("Last »",
		    (total_pages - 1) * MESSAGES_PER_PAGE))

	return html.div(*(" ".join(result)),
	                style="float: right; width: 70%; text-align: right;")

def list_messages():
	# Which messages to show?
	messages = list(reversed(sorted(db.keys())))
	if "start" in form:
		start = int(form["start"].value)
	else:
		start = 0
	page_messages = messages[start:start + MESSAGES_PER_PAGE]
	pager = gen_pager(messages, start)

	return html.body(
		pager,
		html.h1("a858 auto-analysis"),
		credits(),
		html.div(*map(lambda key: format_post(db[key]),
		              page_messages)),
		pager
	)

def single_message():
	id = form["id"].value
	for key in db.keys():
		if ("-%s-" % id) in key:
			post = db[key]
			break
	else:
		return "Unknown message: %s" % id

	return html.body(
		html.h1(html.escape(post["data"]["title"])),
		format_post(post),
	)

def gen_html():
	if "id" in form:
		body = single_message()
	else:
		body = list_messages()

	return html.html(
		html.head(
			html.title("a858 auto-analysis"),
			html.script(language='javascript', src='functions.js')
		),
		body
	)

print "Content-Type: text/html"
print

print gen_html()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import datetime
from django.core.urlresolvers import reverse
from django.db import models
from django.conf import settings
from django.contrib.auth.models import User
from django.utils import timezone
from uuslug import slugify
import thread

class PostQuerySet(models.QuerySet):
    def published(self):
        return self.active().filter(pub_date__lte=timezone.now())

    def active(self):
        return self.filter(is_active=True)

class Notice(models.Model):
    headline = models.CharField(max_length=100)
    content = models.TextField()
    pub_date = models.DateTimeField(editable=False, auto_now_add=True)

    def __unicode__(self):
        return self.headline

    class Meta:
        ordering = ('-pub_date',)
        get_latest_by = 'pub_date'

class Category(models.Model):
    name = models.CharField(max_length=128)
    create_data = models.DateTimeField(editable=False, auto_now_add=True)

    def get_absolute_url(self):
        kwargs = {
            'category': self.name,
        }
        return reverse('blog:archive-category', kwargs=kwargs)

    def __unicode__(self):
        return self.name

class Post(models.Model):
    headline = models.CharField(max_length=200)
    slug = models.SlugField(editable=False, unique_for_date='pub_date')
    is_active = models.BooleanField(default=False)
    category = models.ManyToManyField(Category, related_name='category_post')
    content = models.TextField()
    pub_date = models.DateTimeField(auto_now_add=True)
    create_date = models.DateTimeField(editable=False, auto_now_add=True)
    update_date = models.DateTimeField(editable=False, auto_now=True)
    author = models.ForeignKey(User,related_name='user_post')

    objects = PostQuerySet.as_manager()

    class Meta:
        ordering = ('-pub_date',)
        get_latest_by = 'pub_date'

    def save(self, *args, **kwargs):
        self.slug = slugify(self.headline)
        super(Post, self).save(*args, **kwargs)

    def __unicode__(self):
        return self.headline

    def get_absolute_url(self):
        kwargs = {
            'year': self.pub_date.year,
            'month': self.pub_date.strftime('%b').lower(),
            'day': self.pub_date.strftime('%d').lower(),
            'slug': self.slug,
        }
        return reverse('blog:post', kwargs=kwargs)

class Comment(models.Model):
    user = models.ForeignKey(User, related_name='user_comment')
    post = models.ForeignKey(Post, related_name='post_comment')
    context = models.TextField()
    create_date = models.DateTimeField(auto_now_add=True)

    class Meta:
        ordering = ('create_date',)
        get_latest_by = 'create_date'

    def generate_message(self):
        print self.post.get_absolute_url()
        message = """\

%s

                   %s

                   %s

            """ % (u'您好!',
                   u'你最近评价的文章有了新的评价',
                   u'http://blog.zouyapeng.website' + self.post.get_absolute_url(),
                   self.post.slug)
        return message

    def save(self, *args, **kwargs):
        super(Comment, self).save(*args, **kwargs)
        now = datetime.datetime.now()
        prev_day = now - datetime.timedelta(days=30)
        comments = self.post.post_comment.filter(create_date__range=(prev_day, now))
        users = []
        for comment in comments:
            if comment.user not in users:
                users.append(comment.user)

        if self.post.author not in users:
            users.append(self.post.author)

        users.remove(self.user)

        for user in users:
            if user.user_profile.enable_email is True:
                thread.start_new_thread(user.user_profile.email_user, ('[Zouyapeng 博客]你有一条新消息', self.generate_message()))
                # user.user_profile.email_user()

    def get_absolute_url(self):
        kwargs = {
            'year': self.post.pub_date.year,
            'month': self.post.pub_date.strftime('%b').lower(),
            'day': self.post.pub_date.strftime('%d').lower(),
            'slug': self.post.slug,
        }
        return reverse('blog:post', kwargs=kwargs)

    def __unicode__(self):
        return "%s:%s" % (self.post.slug, self.context)
<commit_msg>slug->headline<commit_after># -*- coding: utf-8 -*-

import datetime
from django.core.urlresolvers import reverse
from django.db import models
from django.conf import settings
from django.contrib.auth.models import User
from django.utils import timezone
from uuslug import slugify
import thread

class PostQuerySet(models.QuerySet):
    def published(self):
        return self.active().filter(pub_date__lte=timezone.now())

    def active(self):
        return self.filter(is_active=True)

class Notice(models.Model):
    headline = models.CharField(max_length=100)
    content = models.TextField()
    pub_date = models.DateTimeField(editable=False, auto_now_add=True)

    def __unicode__(self):
        return self.headline

    class Meta:
        ordering = ('-pub_date',)
        get_latest_by = 'pub_date'

class Category(models.Model):
    name = models.CharField(max_length=128)
    create_data = models.DateTimeField(editable=False, auto_now_add=True)

    def get_absolute_url(self):
        kwargs = {
            'category': self.name,
        }
        return reverse('blog:archive-category', kwargs=kwargs)

    def __unicode__(self):
        return self.name

class Post(models.Model):
    headline = models.CharField(max_length=200)
    slug = models.SlugField(editable=False, unique_for_date='pub_date')
    is_active = models.BooleanField(default=False)
    category = models.ManyToManyField(Category, related_name='category_post')
    content = models.TextField()
    pub_date = models.DateTimeField(auto_now_add=True)
    create_date = models.DateTimeField(editable=False, auto_now_add=True)
    update_date = models.DateTimeField(editable=False, auto_now=True)
    author = models.ForeignKey(User,related_name='user_post')

    objects = PostQuerySet.as_manager()

    class Meta:
        ordering = ('-pub_date',)
        get_latest_by = 'pub_date'

    def save(self, *args, **kwargs):
        self.slug = slugify(self.headline)
        super(Post, self).save(*args, **kwargs)

    def __unicode__(self):
        return self.headline

    def get_absolute_url(self):
        kwargs = {
            'year': self.pub_date.year,
            'month': self.pub_date.strftime('%b').lower(),
            'day': self.pub_date.strftime('%d').lower(),
            'slug': self.slug,
        }
        return reverse('blog:post', kwargs=kwargs)

class Comment(models.Model):
    user = models.ForeignKey(User, related_name='user_comment')
    post = models.ForeignKey(Post, related_name='post_comment')
    context = models.TextField()
    create_date = models.DateTimeField(auto_now_add=True)

    class Meta:
        ordering = ('create_date',)
        get_latest_by = 'create_date'

    def generate_message(self):
        message = """\

%s

                   %s

                   %s

            """ % (u'您好!',
                   u'你最近评价的文章有了新的评价',
                   u'http://blog.zouyapeng.website' + self.post.get_absolute_url(),
                   self.post.headline)
        return message

    def save(self, *args, **kwargs):
        super(Comment, self).save(*args, **kwargs)
        now = datetime.datetime.now()
        prev_day = now - datetime.timedelta(days=30)
        comments = self.post.post_comment.filter(create_date__range=(prev_day, now))
        users = []
        for comment in comments:
            if comment.user not in users:
                users.append(comment.user)

        if self.post.author not in users:
            users.append(self.post.author)

        users.remove(self.user)

        for user in users:
            if user.user_profile.enable_email is True:
                thread.start_new_thread(user.user_profile.email_user, ('[Zouyapeng 博客]你有一条新消息', self.generate_message()))
                # user.user_profile.email_user()

    def get_absolute_url(self):
        kwargs = {
            'year': self.post.pub_date.year,
            'month': self.post.pub_date.strftime('%b').lower(),
            'day': self.post.pub_date.strftime('%d').lower(),
            'slug': self.post.slug,
        }
        return reverse('blog:post', kwargs=kwargs)

    def __unicode__(self):
        return "%s:%s" % (self.post.slug, self.context)
<|endoftext|>


<commit_before>#/
#/login/
#/signup/
#/question/<123>/    # вместо <123> - произвольный ID
#/ask/
#/popular/
#/new/

from django.conf.urls import patterns, url

urlpatterns = patterns('',
    # url(r'^$', 'handler404')
    url(r'^404/$', 'django.views.defaults.page_not_found')
    url(r'^question/(?P<ID_qa>\d{6})/$', 'qa.views.quest'),

)
<commit_msg>запятые<commit_after>#/
#/login/
#/signup/
#/question/<123>/    # вместо <123> - произвольный ID
#/ask/
#/popular/
#/new/
# url(r'^$', 'handler404')

from django.conf.urls import patterns, url

urlpatterns = patterns('',
    url(r'^404/$', 'django.views.defaults.page_not_found'),
    url(r'^question/(?P<ID_qa>\d{6})/$', 'qa.views.quest'),

)
<|endoftext|>


<commit_before>def get(factor):
    factors = [1]
    while factors_product(factors) < factor:
        factors.append(
          lowest_prime_factor(factor/factors_product(factors)))
    return max(factors)

def factors_product(seq):
    return reduce(lambda x, y: x*y, seq)

def lowest_prime_factor(factor):
    for prime in primes():
        if factor_p(factor, prime):
            return prime

def factor_p(factor, divisor):
    return factor % divisor == 0

def get_kval_primes(k_val, primes):
    #filter primes only from 1+kval and 1-kval
    return [x for x in [k_val*6-1, k_val*6+1] if  prime_p(x, primes)]

def prime_p(num, primes):
    #filter primes smaller than num and see if they are factors
    for prime in primes:
        if num % prime == 0:
            return False
    return True

def at_end_of_primes(counter, primes):
    return counter == len(primes) - 1

def up_to(num, seq):
    """Generator given num yields all items from the sequence until
    num is exceeded

    Args:
      num: number that represents a limit the generator will stop
      at when exceeded

      seq: the sequence to iterate over
    """
    for item in seq:
        if item <= num:
            yield item
        else:
            return

def primes():
    counter = 0
    primes = [2, 3]
    k_val = 1
    while True:
        while at_end_of_primes(counter, primes):
            next_primes = get_kval_primes(k_val, list(primes))
            k_val += 1
            for prime in next_primes:
                primes.append(prime)
        yield primes[counter]
        counter += 1
<commit_msg>refactoring of largest_prime_factor for increased readability<commit_after>def get(dividend):
    """ given a dividend, finds all the prime factors for it
    and returns the largest of the prime factors

    args:
      dividend: the number you want to calculate the largest
      prime factor for
    """
    factors = []
    while product_of_factors(factors) < dividend:
        factors.append(
          lowest_prime_factor(reduced_dividend(dividend, factors)))
    return max(factors)

def reduced_dividend(dividend, factors):
    """The new dividend as a result of dividing the original
    dividend by the list of prime factors calculated so far."""
    return dividend/product_of_factors(factors)

def product_of_factors(seq):
    """return the product of numbers in the list or 1 for the
    empty list to prevent division by zero errors"""
    return reduce(lambda x, y: x*y, seq, 1)

def lowest_prime_factor(factor):
    for prime in primes():
        if factor_p(factor, prime):
            return prime

def factor_p(factor, divisor):
    return factor % divisor == 0

def get_kval_primes(k_val, primes):
    #filter primes only from 1+kval and 1-kval
    return [x for x in [k_val*6-1, k_val*6+1] if  prime_p(x, primes)]

def prime_p(num, primes):
    #filter primes smaller than num and see if they are factors
    for prime in primes:
        if num % prime == 0:
            return False
    return True

def at_end_of_primes(counter, primes):
    return counter == len(primes) - 1

def up_to(num, seq):
    """Generator given num yields all items from the sequence until
    num is exceeded

    Args:
      num: number that represents a limit the generator will stop
      at when exceeded

      seq: the sequence to iterate over
    """
    for item in seq:
        if item <= num:
            yield item
        else:
            return

def primes():
    counter = 0
    primes = [2, 3]
    k_val = 1
    while True:
        while at_end_of_primes(counter, primes):
            next_primes = get_kval_primes(k_val, list(primes))
            k_val += 1
            for prime in next_primes:
                primes.append(prime)
        yield primes[counter]
        counter += 1
<|endoftext|>


<commit_before># Copyright 2008-2010 ITA Software, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Subprocess Queries"""

import os
import re
import signal

from zope.interface import classProvides
from twisted.internet import reactor, defer, protocol, process
from twisted.internet import error as neterror

from nagcat import errors, log, query

class SubprocessProtocol(protocol.ProcessProtocol):
    """Handle input/output for subprocess queries"""

    timedout = False

    def connectionMade(self):
        self.result = ""
        if self.factory.conf['data']:
            self.transport.write(self.factory.conf['data'])
        self.transport.closeStdin()

    def outReceived(self, data):
        self.result += data

    def errReceived(self, data):
        key = "Process stderr"
        self.factory.saved[key] = self.factory.saved.get(key, "") + data

    def timeout(self):
        self.timedout = True
        self.transport.loseConnection()
        # Kill all processes in the child's process group
        if self.transport.pid:
            try:
                os.kill(-int(self.transport.pid), signal.SIGTERM)
            except OSError, ex:
                log.warn("Failed to send TERM to a subprocess: %s", ex)

    def processEnded(self, reason):
        if isinstance(reason.value, neterror.ProcessDone):
            result = self.result
        elif isinstance(reason.value, neterror.ProcessTerminated):
            if self.timedout:
                result = errors.Failure(errors.TestCritical(
                    "Timeout waiting for command to finish."),
                    result=self.result)
            elif reason.value.exitCode == 127:
                result = errors.Failure(errors.TestCritical(
                    "Command not found."))
            else:
                result = errors.Failure(reason.value, result=self.result)
        else:
            result = reason

        self.factory.result(result)

class SubprocessFactory(process.Process):
    """Execute a subprocess"""

    def __init__(self, query):
        self.conf = query.conf
        self.saved = query.saved
        self.deferred = defer.Deferred()
        self._startProcess(("/bin/sh", "-c", self.conf['command']))

    def _startProcess(self, command):
        command = [str(x) for x in command]
        log.debug("Running process: %s", command)

        proto = SubprocessProtocol()
        proto.factory = self

        # Setup timeout
        call_id = reactor.callLater(self.conf['timeout'], proto.timeout)
        self.deferred.addBoth(self._cancelTimeout, call_id)

        # Setup shutdown cleanup
        call_id = reactor.addSystemEventTrigger('after', 'shutdown',
                proto.timeout)
        self.deferred.addBoth(self._cancelCleanup, call_id)

        process.Process.__init__(self, reactor, command[0], command,
                self.conf['environment'], path=None, proto=proto)

    def result(self, result):
        self.deferred.callback(result)

    def _cancelTimeout(self, result, call_id):
        if call_id.active():
            call_id.cancel()
        return result

    def _cancelCleanup(self, result, call_id):
        reactor.removeSystemEventTrigger(call_id)
        return result

    def _setupChild(self, *args, **kwargs):
        # called in the child fork, set new process group
        os.setpgrp()
        process.Process._setupChild(self, *args, **kwargs)

class SubprocessQuery(query.Query):
    """Query that runs a command"""

    classProvides(query.IQuery)

    name = "subprocess"

    def __init__(self, nagcat, conf):
        super(SubprocessQuery, self).__init__(nagcat, conf)

        env = os.environ.copy()
        if 'environment' in conf:
            env.update(conf['environment'])

        self.conf['command'] = conf['command']
        self.conf['data'] = conf.get('data', "")
        self.conf['environment'] = env

    def _start(self):
        proc = SubprocessFactory(self)
        proc.deferred.addErrback(self._checkError)
        return proc.deferred

    def _checkError(self, reason):
        if isinstance(reason.value, neterror.ProcessTerminated):
            return errors.Failure(errors.TestCritical(
                    reason.value.args[0]), result=reason.result)
        else:
            return reason

class NagiosPluginQuery(SubprocessQuery):
    """Query that runs a command"""

    classProvides(query.IQuery)

    name = "nagios_plugin"

    # 'label'=value[UOM];[warn];[crit];[min];[max]
    # where UOM (unit of measure) will usually be:
    #   a. no unit specified - assume a number (int or float) of things
    #   b. s - seconds (also us, ms)
    #   c. % - percentage
    #   d. B - bytes (also KB, MB, TB)
    #   e. c - a continous counter
    # See http://nagiosplug.sourceforge.net/developer-guidelines.html
    perf_format = re.compile(r'''(?P<label>\S+)=
                                (?P<value>[^;]*)
                                (;(?P<warn>[^;]*)
                                (;(?P<crit>[^;]*)
                                (;(?P<min>[^;]*)
                                (;(?P<max>[^;]*))?)?)?)?''', re.X)

    def __init__(self, nagcat, conf):
        super(NagiosPluginQuery, self).__init__(nagcat, conf)

        # don't pass stuff in on stdin
        self.conf['data'] = ''

        # Should we grab perf data or text? (empty means text)
        self.conf['perfdata'] = str(conf.get('perfdata', ''))

    def _start(self):
        proc = SubprocessFactory(self)
        proc.deferred.addErrback(self._checkError)
        proc.deferred.addCallback(self._parseOutput)
        return proc.deferred

    @errors.callback
    def _parseOutput(self, result):
        if not isinstance(result, basestring):
            return result

        # grab performance data from the first line
        lines = result.split('\n', 1)
        split = lines[0].split('|', 1)

        if self.conf['perfdata']:
            if len(split) != 2:
                raise errors.Critical("No performance data found")

            for match in self.perf_format.finditer(split[1]):
                if match.group('label') == self.conf['perfdata']:
                    return match.group('value')

            raise errors.Critical("No performance data found for %s" %
                                    (self.conf['perfdata'],))
        else:
            # Return the text sans perf data
            return split[0] + '\n'.join(lines[1:])

    def _checkError(self, reason):
        if isinstance(reason.value, neterror.ProcessTerminated):
            parsed = self._parseOutput(reason.result)
            if isinstance(parsed, errors.Failure):
                return parsed

            if reason.value.exitCode == 1:
                return errors.Failure(errors.TestWarning(
                    reason.result.split('\n', 1)[0]), result=parsed)
            elif reason.value.exitCode == 2:
                return errors.Failure(errors.TestCritical(
                    reason.result.split('\n', 1)[0]), result=parsed)
            elif reason.value.exitCode == 3:
                return errors.Failure(errors.TestUnknown(
                    reason.result.split('\n', 1)[0]), result=parsed)
            else:
                return errors.Failure(errors.TestUnknown(
                    reason.value.args[0]), result=parsed)
        else:
            return reason
<commit_msg>Add support for proper nagios_plugin query combining.<commit_after># Copyright 2008-2010 ITA Software, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Subprocess Queries"""

import os
import re
import signal

from zope.interface import classProvides
from twisted.internet import reactor, defer, protocol, process
from twisted.internet import error as neterror

from nagcat import errors, log, query

class SubprocessError(errors.TestError):
    """Process returned non-zero exit status"""

    def __init__(self, ex):
        assert isinstance(ex, neterror.ProcessTerminated)
        self.exitCode = ex.exitCode
        self.signal = ex.signal
        self.status = ex.status
        super(SubprocessError, self).__init__(ex.args[0])

class SubprocessProtocol(protocol.ProcessProtocol):
    """Handle input/output for subprocess queries"""

    timedout = False

    def connectionMade(self):
        self.result = ""
        if self.factory.conf['data']:
            self.transport.write(self.factory.conf['data'])
        self.transport.closeStdin()

    def outReceived(self, data):
        self.result += data

    def errReceived(self, data):
        key = "Process stderr"
        self.factory.saved[key] = self.factory.saved.get(key, "") + data

    def timeout(self):
        self.timedout = True
        self.transport.loseConnection()
        # Kill all processes in the child's process group
        if self.transport.pid:
            try:
                os.kill(-int(self.transport.pid), signal.SIGTERM)
            except OSError, ex:
                log.warn("Failed to send TERM to a subprocess: %s", ex)

    def processEnded(self, reason):
        if isinstance(reason.value, neterror.ProcessDone):
            result = self.result
        elif isinstance(reason.value, neterror.ProcessTerminated):
            if self.timedout:
                result = errors.Failure(errors.TestCritical(
                    "Timeout waiting for command to finish."),
                    result=self.result)
            elif reason.value.exitCode == 127:
                result = errors.Failure(errors.TestCritical(
                    "Command not found."))
            else:
                result = errors.Failure(SubprocessError(reason.value),
                                        result=self.result)
        else:
            result = reason

        self.factory.result(result)

class SubprocessFactory(process.Process):
    """Execute a subprocess"""

    def __init__(self, query):
        self.conf = query.conf
        self.saved = query.saved
        self.deferred = defer.Deferred()
        self._startProcess(("/bin/sh", "-c", self.conf['command']))

    def _startProcess(self, command):
        command = [str(x) for x in command]
        log.debug("Running process: %s", command)

        proto = SubprocessProtocol()
        proto.factory = self

        # Setup timeout
        call_id = reactor.callLater(self.conf['timeout'], proto.timeout)
        self.deferred.addBoth(self._cancelTimeout, call_id)

        # Setup shutdown cleanup
        call_id = reactor.addSystemEventTrigger('after', 'shutdown',
                proto.timeout)
        self.deferred.addBoth(self._cancelCleanup, call_id)

        process.Process.__init__(self, reactor, command[0], command,
                self.conf['environment'], path=None, proto=proto)

    def result(self, result):
        self.deferred.callback(result)

    def _cancelTimeout(self, result, call_id):
        if call_id.active():
            call_id.cancel()
        return result

    def _cancelCleanup(self, result, call_id):
        reactor.removeSystemEventTrigger(call_id)
        return result

    def _setupChild(self, *args, **kwargs):
        # called in the child fork, set new process group
        os.setpgrp()
        process.Process._setupChild(self, *args, **kwargs)

class SubprocessBase(query.Query):
    """Query that runs a command"""

    name = "subprocess_base"

    def __init__(self, nagcat, conf):
        super(SubprocessBase, self).__init__(nagcat, conf)

        env = os.environ.copy()
        if 'environment' in conf:
            env.update(conf['environment'])

        self.conf['command'] = conf['command']
        self.conf['data'] = conf.get('data', "")
        self.conf['environment'] = env

    def _start(self):
        proc = SubprocessFactory(self)
        return proc.deferred

class SubprocessQuery(SubprocessBase):
    """Query that runs a command"""

    classProvides(query.IQuery)

    name = "subprocess"

    def _start(self):
        deferred = super(SubprocessQuery, self)._start()
        deferred.addErrback(self._checkError)
        return deferred

    def _checkError(self, reason):
        if isinstance(reason.value, SubprocessError):
            return errors.Failure(errors.TestCritical(
                    str(reason.value)), result=reason.result)
        else:
            return reason

class NagiosPluginQuery(query.Query):
    """Query that runs a command"""

    classProvides(query.IQuery)

    name = "nagios_plugin"

    # 'label'=value[UOM];[warn];[crit];[min];[max]
    # where UOM (unit of measure) will usually be:
    #   a. no unit specified - assume a number (int or float) of things
    #   b. s - seconds (also us, ms)
    #   c. % - percentage
    #   d. B - bytes (also KB, MB, TB)
    #   e. c - a continous counter
    # See http://nagiosplug.sourceforge.net/developer-guidelines.html
    perf_format = re.compile(r'''(?P<label>\S+)=
                                (?P<value>[^;]*)
                                (;(?P<warn>[^;]*)
                                (;(?P<crit>[^;]*)
                                (;(?P<min>[^;]*)
                                (;(?P<max>[^;]*))?)?)?)?''', re.X)

    def __init__(self, nagcat, conf):
        super(NagiosPluginQuery, self).__init__(nagcat, conf)

        self.subquery = nagcat.new_query(conf, qcls=SubprocessBase)
        self.addDependency(self.subquery)
        self.conf.update(self.subquery.conf)

        # Should we grab perf data or text? (empty means text)
        self.conf['perfdata'] = str(conf.get('perfdata', ''))

    def _start(self):
        deferred = defer.succeed(self.subquery.result)
        deferred.addCallbacks(self._parseOutput, self._checkError)
        return deferred

    @errors.callback
    def _parseOutput(self, result):
        if not isinstance(result, basestring):
            return result

        # grab performance data from the first line
        lines = result.split('\n', 1)
        split = lines[0].split('|', 1)

        if self.conf['perfdata']:
            if len(split) != 2:
                raise errors.Critical("No performance data found")

            for match in self.perf_format.finditer(split[1]):
                if match.group('label') == self.conf['perfdata']:
                    return match.group('value')

            raise errors.Critical("No performance data found for %s" %
                                    (self.conf['perfdata'],))
        else:
            # Return the text sans perf data
            return split[0] + '\n'.join(lines[1:]) + '\n'

    def _checkError(self, reason):
        if isinstance(reason.value, SubprocessError):
            parsed = self._parseOutput(reason.result)
            if isinstance(parsed, errors.Failure):
                return parsed

            if reason.value.exitCode == 1:
                return errors.Failure(errors.TestWarning(
                    parsed.split('\n', 1)[0]), result=parsed)
            elif reason.value.exitCode == 2:
                return errors.Failure(errors.TestCritical(
                    parsed.split('\n', 1)[0]), result=parsed)
            elif reason.value.exitCode == 3:
                return errors.Failure(errors.TestUnknown(
                    parsed.split('\n', 1)[0]), result=parsed)
            else:
                return errors.Failure(errors.TestUnknown(
                    str(reason.value)), result=parsed)
        else:
            return reason
<|endoftext|>


<commit_before># Copyright 2010-2011 OpenStack LLC.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Cloud Keep version
"""

__version__ = '0.1.4dev'
__version_info__ = tuple(__version__.split('.'))
<commit_msg>manual move to next dev version<commit_after># Copyright 2010-2011 OpenStack LLC.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Cloud Keep version
"""

__version__ = '0.1.5dev'
__version_info__ = tuple(__version__.split('.'))
<|endoftext|>


<commit_before>""" Centralized access to LMS courseware app """
from django.utils import timezone

from courseware import courses, module_render
from courseware.model_data import FieldDataCache
from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import CourseKey, UsageKey
from opaque_keys.edx.locations import SlashSeparatedCourseKey, Location
from xmodule.modulestore import InvalidLocationError
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.exceptions import ItemNotFoundError

def get_modulestore():
    return modulestore()

def get_course(request, user, course_id, depth=0, load_content=False):
    """
    Utility method to obtain course components
    """
    course_descriptor = None
    course_content = None
    course_key = get_course_key(course_id)
    if course_key:
        course_descriptor = get_course_descriptor(course_key, depth)
        if course_descriptor and load_content:
            course_content = get_course_content(request, user, course_key, course_descriptor)
    return course_descriptor, course_key, course_content

def get_course_child(request, user, course_key, content_id, load_content=False):
    """
    Return a course xmodule/xblock to the caller
    """
    child_descriptor = None
    child_content = None
    child_key = get_course_child_key(content_id)
    if child_key:
        child_descriptor = get_course_child_descriptor(child_key)
        if child_descriptor and load_content:
            child_content = get_course_child_content(request, user, course_key, child_descriptor)
    return child_descriptor, child_key, child_content

def get_course_total_score(course_summary):
    """
    Traverse course summary to calculate max possible score for a course
    """
    score = 0
    for chapter in course_summary:  # accumulate score of each chapter
        for section in chapter['sections']:
            if section['section_total']:
                score += section['section_total'][1]
    return score

def get_course_leaf_nodes(course_key, detached_categories):
    """
    Get count of the leaf nodes with ability to exclude some categories
    """
    nodes = []
    verticals = get_modulestore().get_items(course_key, category='vertical')
    for vertical in verticals:
        nodes.extend([unit for unit in vertical.children
                      if getattr(unit, 'category') not in detached_categories])
    return nodes

def get_course_key(course_id, slashseparated=False):
    try:
        course_key = CourseKey.from_string(course_id)
    except InvalidKeyError:
        try:
            course_key = SlashSeparatedCourseKey.from_deprecated_string(course_id)
        except InvalidKeyError:
            course_key = None
    if slashseparated:
        try:
            course_key = course_key.to_deprecated_string()
        except:
            course_key = course_id
    return course_key

def get_course_descriptor(course_key, depth):
    try:
        course_descriptor = courses.get_course(course_key, depth)
    except ValueError:
        course_descriptor = None
    return course_descriptor

def get_course_content(request, user, course_key, course_descriptor):
    field_data_cache = FieldDataCache([course_descriptor], course_key, user)
    course_content = module_render.get_module_for_descriptor(
        user,
        request,
        course_descriptor,
        field_data_cache,
        course_key)
    return course_content

def course_exists(request, user, course_id):
    course_key = get_course_key(course_id)
    if not course_key:
        return False
    if not get_modulestore().has_course(course_key):
        return False
    return True

def get_course_child_key(content_id):
    try:
        content_key = UsageKey.from_string(content_id)
    except InvalidKeyError:
        try:
            content_key = Location.from_deprecated_string(content_id)
        except (InvalidLocationError, InvalidKeyError):
            content_key = None
    return content_key

def get_course_child_descriptor(child_key):
    try:
        content_descriptor = get_modulestore().get_item(child_key)
    except ItemNotFoundError:
        content_descriptor = None
    return content_descriptor

def get_course_child_content(request, user, course_key, child_descriptor):
    field_data_cache = FieldDataCache([child_descriptor], course_key, user)
    child_content = module_render.get_module_for_descriptor(
        user,
        request,
        child_descriptor,
        field_data_cache,
        course_key)
    return child_content

def calculate_proforma_grade(grade_summary, grading_policy):
    """
    Calculates a projected (proforma) final grade based on the current state
    of grades using the provided grading policy.  Sections equate to grading policy
    'types' and have values such as 'Homework', 'Lab', 'MidtermExam', and 'FinalExam'

    We invert the concepts here and use the section weights as the possible scores by
    assuming that the section weights total 100 percent.  So, if a Homework section
    is worth 15 percent of your overall grade, and you have currently scored 70 percent
    for that section, the normalized score for the Homework section is 0.105.  Note that
    we do not take into account dropped assignments/scores, such as lowest-two homeworks.

    After all scored sections are processed we take the remaining weight at its full
    value as a projection of the user obtaining 100 percent of the section potential.

    Example:
        - Section: Homework,    Weight: 15%, Totaled Score: 70%,  Normalized Score: 0.105
        - Section: MidtermExam, Weight: 30%, Totaled Score: 80%,  Normalized Score: 0.240
        - Section: Final Exam,  Weight: 40%, Totaled Score: 95%,  Normalized Score: 0.380
        - Remaining Weight: 0.15 (unscored Lab section), assume 100%, of 15% =>     0.150
        - Proforma Grade = 0.105 + 0.240 + 0.380 + 0.150 = 0.875  (87.5%)
    """
    remaining_weight = 1.00
    proforma_grade = 0.00
    totaled_scores = grade_summary['totaled_scores']
    grade = 0.00
    for section in totaled_scores:
        points_earned = 0.00
        points_possible = 0.00
        # totaled_scores is a collection of currently-recored scores for a given section
        # we need to iterate through and combine the scores to create an overall score for the section
        # This loop does not take into account dropped assignments (eg, homeworks)
        for score in totaled_scores[section]:
            # Only count grades where points have been scored, or where the due date has passed
            if score.earned or (score.due and score.due < timezone.now()):
                points_earned = points_earned + score.earned
                points_possible = points_possible + score.possible
        if points_possible:
            grade = points_earned / points_possible
            section_policy = next((policy for policy in grading_policy['GRADER'] if policy['type'] == section), None)
            if section_policy is not None:
                section_weight = section_policy['weight']
                proforma_grade = proforma_grade + (section_weight * grade)
                remaining_weight = remaining_weight - section_weight
    proforma_grade = proforma_grade + remaining_weight
    return proforma_grade
<commit_msg>Ensure that API calls using anonymous user have the user object containing an attribute 'known', set to False<commit_after>""" Centralized access to LMS courseware app """
from django.contrib.auth.models import AnonymousUser
from django.utils import timezone

from courseware import courses, module_render
from courseware.model_data import FieldDataCache
from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import CourseKey, UsageKey
from opaque_keys.edx.locations import SlashSeparatedCourseKey, Location
from xmodule.modulestore import InvalidLocationError
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.exceptions import ItemNotFoundError

def _anonymous_known_flag(user):
    if isinstance(user, AnonymousUser):
        user.known = False

def get_modulestore():
    return modulestore()

def get_course(request, user, course_id, depth=0, load_content=False):
    """
    Utility method to obtain course components
    """
    _anonymous_known_flag(user)
    course_descriptor = None
    course_content = None
    course_key = get_course_key(course_id)
    if course_key:
        course_descriptor = get_course_descriptor(course_key, depth)
        if course_descriptor and load_content:
            course_content = get_course_content(request, user, course_key, course_descriptor)
    return course_descriptor, course_key, course_content

def get_course_child(request, user, course_key, content_id, load_content=False):
    """
    Return a course xmodule/xblock to the caller
    """
    _anonymous_known_flag(user)
    child_descriptor = None
    child_content = None
    child_key = get_course_child_key(content_id)
    if child_key:
        child_descriptor = get_course_child_descriptor(child_key)
        if child_descriptor and load_content:
            child_content = get_course_child_content(request, user, course_key, child_descriptor)
    return child_descriptor, child_key, child_content

def get_course_total_score(course_summary):
    """
    Traverse course summary to calculate max possible score for a course
    """
    score = 0
    for chapter in course_summary:  # accumulate score of each chapter
        for section in chapter['sections']:
            if section['section_total']:
                score += section['section_total'][1]
    return score

def get_course_leaf_nodes(course_key, detached_categories):
    """
    Get count of the leaf nodes with ability to exclude some categories
    """
    nodes = []
    verticals = get_modulestore().get_items(course_key, category='vertical')
    for vertical in verticals:
        nodes.extend([unit for unit in vertical.children
                      if getattr(unit, 'category') not in detached_categories])
    return nodes

def get_course_key(course_id, slashseparated=False):
    try:
        course_key = CourseKey.from_string(course_id)
    except InvalidKeyError:
        try:
            course_key = SlashSeparatedCourseKey.from_deprecated_string(course_id)
        except InvalidKeyError:
            course_key = None
    if slashseparated:
        try:
            course_key = course_key.to_deprecated_string()
        except:
            course_key = course_id
    return course_key

def get_course_descriptor(course_key, depth):
    try:
        course_descriptor = courses.get_course(course_key, depth)
    except ValueError:
        course_descriptor = None
    return course_descriptor

def get_course_content(request, user, course_key, course_descriptor):
    field_data_cache = FieldDataCache([course_descriptor], course_key, user)
    course_content = module_render.get_module_for_descriptor(
        user,
        request,
        course_descriptor,
        field_data_cache,
        course_key)
    return course_content

def course_exists(request, user, course_id):
    course_key = get_course_key(course_id)
    if not course_key:
        return False
    if not get_modulestore().has_course(course_key):
        return False
    return True

def get_course_child_key(content_id):
    try:
        content_key = UsageKey.from_string(content_id)
    except InvalidKeyError:
        try:
            content_key = Location.from_deprecated_string(content_id)
        except (InvalidLocationError, InvalidKeyError):
            content_key = None
    return content_key

def get_course_child_descriptor(child_key):
    try:
        content_descriptor = get_modulestore().get_item(child_key)
    except ItemNotFoundError:
        content_descriptor = None
    return content_descriptor

def get_course_child_content(request, user, course_key, child_descriptor):
    field_data_cache = FieldDataCache([child_descriptor], course_key, user)
    child_content = module_render.get_module_for_descriptor(
        user,
        request,
        child_descriptor,
        field_data_cache,
        course_key)
    return child_content

def calculate_proforma_grade(grade_summary, grading_policy):
    """
    Calculates a projected (proforma) final grade based on the current state
    of grades using the provided grading policy.  Sections equate to grading policy
    'types' and have values such as 'Homework', 'Lab', 'MidtermExam', and 'FinalExam'

    We invert the concepts here and use the section weights as the possible scores by
    assuming that the section weights total 100 percent.  So, if a Homework section
    is worth 15 percent of your overall grade, and you have currently scored 70 percent
    for that section, the normalized score for the Homework section is 0.105.  Note that
    we do not take into account dropped assignments/scores, such as lowest-two homeworks.

    After all scored sections are processed we take the remaining weight at its full
    value as a projection of the user obtaining 100 percent of the section potential.

    Example:
        - Section: Homework,    Weight: 15%, Totaled Score: 70%,  Normalized Score: 0.105
        - Section: MidtermExam, Weight: 30%, Totaled Score: 80%,  Normalized Score: 0.240
        - Section: Final Exam,  Weight: 40%, Totaled Score: 95%,  Normalized Score: 0.380
        - Remaining Weight: 0.15 (unscored Lab section), assume 100%, of 15% =>     0.150
        - Proforma Grade = 0.105 + 0.240 + 0.380 + 0.150 = 0.875  (87.5%)
    """
    remaining_weight = 1.00
    proforma_grade = 0.00
    totaled_scores = grade_summary['totaled_scores']
    grade = 0.00
    for section in totaled_scores:
        points_earned = 0.00
        points_possible = 0.00
        # totaled_scores is a collection of currently-recored scores for a given section
        # we need to iterate through and combine the scores to create an overall score for the section
        # This loop does not take into account dropped assignments (eg, homeworks)
        for score in totaled_scores[section]:
            # Only count grades where points have been scored, or where the due date has passed
            if score.earned or (score.due and score.due < timezone.now()):
                points_earned = points_earned + score.earned
                points_possible = points_possible + score.possible
        if points_possible:
            grade = points_earned / points_possible
            section_policy = next((policy for policy in grading_policy['GRADER'] if policy['type'] == section), None)
            if section_policy is not None:
                section_weight = section_policy['weight']
                proforma_grade = proforma_grade + (section_weight * grade)
                remaining_weight = remaining_weight - section_weight
    proforma_grade = proforma_grade + remaining_weight
    return proforma_grade
<|endoftext|>