<commit_before># Copyright 2012-2013 James McCauley
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
A collection of some useful IOWorkers

These were quickly adapted from another project.  The versions of the
server ones here haven't been tested.  The persistent ones at least
sort of have.  The backoff one is new.
"""

import errno
import socket
from pox.lib.addresses import IP_ANY, IPAddr
from pox.lib.ioworker import *
from pox.core import core

log = core.getLogger()

class LoggerBase (object):
  def _error (self, *args, **kw):
    log.error(type(self).__name__ + ": " + str(args[0]), *args[1:], **kw)
  def _warn (self, *args, **kw):
    log.warn(type(self).__name__ + ": " + str(args[0]), *args[1:], **kw)
  def _info (self, *args, **kw):
    log.info(type(self).__name__ + ": " + str(args[0]), *args[1:], **kw)
  def _debug (self, *args, **kw):
    log.debug(type(self).__name__ + ": " + str(args[0]), *args[1:], **kw)

class TCPServerWorkerBase (IOWorker, LoggerBase):
  def __init__ (self, ip = IP_ANY, port = None,
        backlog = 5, *args, **kw):
    """
    Listens on ip/port and fires _do_accept when there's a connection
    """
    #super(TCPServerWorkerBase,self).__init__(*args, **kw)
    IOWorker.__init__(self)

    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    self.socket = s
    #s.setblocking(0)
    if port is None: port = 0
    s.bind((str(IPAddr(ip)), port))
    s.listen(backlog)

  @property
  def local_ip (self):
    return IPAddr(s.getsockname()[0])
  @property
  def local_port (self):
    return s.getsockname()[1]

  def _do_accept (self, socket, loop):
    """
    Override me
    """
    pass

  def _do_recv (self, loop):
    s,addr = self.socket.accept()
    s.setblocking(0)

    self._do_accept(loop, s)

  def _handle_close (self):
    # Just here to kill log message
    pass

class TCPServerWorker (TCPServerWorkerBase):
  def __init__ (self, child_worker_type, ip = IP_ANY, port = None,
      child_args = {}, *args, **kw):
    """
    Listens on ip/port and creates a child_worker_type for each connnection
    """
    super(TCPServerWorker,self).__init__(ip=ip,port=port,*args, **kw)

    self.child_worker_type = child_worker_type
    self.child_args = child_args

  def _do_accept (self, socket, loop):
    self._debug("accepting %s:%i" % addr)
    out = loop.new_worker(socket = socket,
        _worker_type = self.child_worker_type,
        **self.child_args)
    return out

class PersistentIOWorker (RecocoIOWorker, LoggerBase):
  """
  An IOWorker which opens a duplicate of itself when it closes

  Subclasses can add keyword parameters for constructor
  """

  _default_retry_delay = 2

  def __repr__ (self):
    return object.__repr__(self)

  def __init__ (self, **kw):
    """
    Initialize

    See _make_connection for arg list.

    callbacks take a single arg -- the worker in question
    If the disconnect callback returns False, a new connection will NOT
    be opened.
    """
    IOWorker.__init__(self)

    self.kw = kw

    self._connecting = True

    self._make_connection(**kw)

  def _make_connection (self, loop, addr, port,
      reconnect_delay = _default_retry_delay,
      connect_callback = None, disconnect_callback = None, **kw):

    self.loop = loop
    self.addr = addr #IPAddr(addr)
    self.port = port
    self.reconnect_delay = reconnect_delay
    self.connect_callback = connect_callback
    self.disconnect_callback = disconnect_callback

    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    self.socket = s
    s.setblocking(0)
    self._debug("Attempting to connect to %s:%s", self.addr, self.port)
    r = s.connect_ex((str(self.addr), self.port))
    if r == 0 or r == errno.EINPROGRESS:
      # We either connected or connection is in progress
      pass
    else:
      #self._error("Couldn't connect to %s:%s", self.addr, self.port)
      #raise RuntimeError("Couldn't connect")
      core.callLater(self._handle_close)
      return

    self.loop.register_worker(self)

  @classmethod
  def begin (cls, **kw):
    #if len(args) >= 4:
    #  reconnect_delay = args[3]
    #else:
    reconnect_delay = kw.get('reconnect_delay',
        cls._default_retry_delay)

    try:
      w = cls(**kw)
      return w
    except:
      raise
      core.callDelayed(reconnect_delay, cls.begin, **kw)
      return None

  def open_later (self):
    core.callDelayed(self.reconnect_delay, self.begin, **self.kw)

  def _handle_close (self):
    self._debug("disconnected")
    super(PersistentIOWorker, self)._handle_close()
    if self.disconnect_callback:
      if self.disconnect_callback(self) is False:
        return
    self.open_later()

  def _handle_connect (self):
    super(PersistentIOWorker, self)._handle_connect()
    if self.connect_callback:
      self.connect_callback(self)

class BackoffWorker (PersistentIOWorker):
  def __init__ (self, **kw):
    kw.setdefault('reconnect_delay', 0.5)
    self.max_retry_delay = kw.get('max_retry_delay',16)
    super(BackoffWorker,self).__init__(**kw)

  def _handle_connect (self):
    self.reconnect_delay = 0.5
    super(BackoffWorker, self)._handle_connect()

  def open_later (self):
    self.reconnect_delay *= 2
    self.reconnect_delay = int(self.reconnect_delay)
    if self.reconnect_delay > self.max_retry_delay:
      self.reconnect_delay = self.max_retry_delay
    self.kw['reconnect_delay'] = self.reconnect_delay
    self._debug("Try again in %s seconds", self.reconnect_delay)
    from pox.core import core
    core.callDelayed(self.reconnect_delay, self.begin, **self.kw)
<commit_msg>ioworker: Fix server worker _do_accept param reversal<commit_after># Copyright 2012-2013 James McCauley
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
A collection of some useful IOWorkers

These were quickly adapted from another project.  The versions of the
server ones here haven't been tested.  The persistent ones at least
sort of have.  The backoff one is new.
"""

import errno
import socket
from pox.lib.addresses import IP_ANY, IPAddr
from pox.lib.ioworker import *
from pox.core import core

log = core.getLogger()

class LoggerBase (object):
  def _error (self, *args, **kw):
    log.error(type(self).__name__ + ": " + str(args[0]), *args[1:], **kw)
  def _warn (self, *args, **kw):
    log.warn(type(self).__name__ + ": " + str(args[0]), *args[1:], **kw)
  def _info (self, *args, **kw):
    log.info(type(self).__name__ + ": " + str(args[0]), *args[1:], **kw)
  def _debug (self, *args, **kw):
    log.debug(type(self).__name__ + ": " + str(args[0]), *args[1:], **kw)

class TCPServerWorkerBase (IOWorker, LoggerBase):
  def __init__ (self, ip = IP_ANY, port = None,
        backlog = 5, *args, **kw):
    """
    Listens on ip/port and fires _do_accept when there's a connection
    """
    #super(TCPServerWorkerBase,self).__init__(*args, **kw)
    IOWorker.__init__(self)

    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    self.socket = s
    #s.setblocking(0)
    if port is None: port = 0
    s.bind((str(IPAddr(ip)), port))
    s.listen(backlog)

  @property
  def local_ip (self):
    return IPAddr(s.getsockname()[0])
  @property
  def local_port (self):
    return s.getsockname()[1]

  def _do_accept (self, loop, socket):
    """
    Override me
    """
    pass

  def _do_recv (self, loop):
    s,addr = self.socket.accept()
    s.setblocking(0)

    self._do_accept(loop, s)

  def _handle_close (self):
    # Just here to kill log message
    pass

class TCPServerWorker (TCPServerWorkerBase):
  def __init__ (self, child_worker_type, ip = IP_ANY, port = None,
      child_args = {}, *args, **kw):
    """
    Listens on ip/port and creates a child_worker_type for each connnection
    """
    super(TCPServerWorker,self).__init__(ip=ip,port=port,*args, **kw)

    self.child_worker_type = child_worker_type
    self.child_args = child_args

  def _do_accept (self, loop, socket):
    self._debug("accepting %s:%i" % addr)
    out = loop.new_worker(socket = socket,
        _worker_type = self.child_worker_type,
        **self.child_args)
    return out

class PersistentIOWorker (RecocoIOWorker, LoggerBase):
  """
  An IOWorker which opens a duplicate of itself when it closes

  Subclasses can add keyword parameters for constructor
  """

  _default_retry_delay = 2

  def __repr__ (self):
    return object.__repr__(self)

  def __init__ (self, **kw):
    """
    Initialize

    See _make_connection for arg list.

    callbacks take a single arg -- the worker in question
    If the disconnect callback returns False, a new connection will NOT
    be opened.
    """
    IOWorker.__init__(self)

    self.kw = kw

    self._connecting = True

    self._make_connection(**kw)

  def _make_connection (self, loop, addr, port,
      reconnect_delay = _default_retry_delay,
      connect_callback = None, disconnect_callback = None, **kw):

    self.loop = loop
    self.addr = addr #IPAddr(addr)
    self.port = port
    self.reconnect_delay = reconnect_delay
    self.connect_callback = connect_callback
    self.disconnect_callback = disconnect_callback

    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    self.socket = s
    s.setblocking(0)
    self._debug("Attempting to connect to %s:%s", self.addr, self.port)
    r = s.connect_ex((str(self.addr), self.port))
    if r == 0 or r == errno.EINPROGRESS:
      # We either connected or connection is in progress
      pass
    else:
      #self._error("Couldn't connect to %s:%s", self.addr, self.port)
      #raise RuntimeError("Couldn't connect")
      core.callLater(self._handle_close)
      return

    self.loop.register_worker(self)

  @classmethod
  def begin (cls, **kw):
    #if len(args) >= 4:
    #  reconnect_delay = args[3]
    #else:
    reconnect_delay = kw.get('reconnect_delay',
        cls._default_retry_delay)

    try:
      w = cls(**kw)
      return w
    except:
      raise
      core.callDelayed(reconnect_delay, cls.begin, **kw)
      return None

  def open_later (self):
    core.callDelayed(self.reconnect_delay, self.begin, **self.kw)

  def _handle_close (self):
    self._debug("Disconnected")
    super(PersistentIOWorker, self)._handle_close()
    if self.disconnect_callback:
      if self.disconnect_callback(self) is False:
        return
    self.open_later()

  def _handle_connect (self):
    super(PersistentIOWorker, self)._handle_connect()
    if self.connect_callback:
      self.connect_callback(self)

class BackoffWorker (PersistentIOWorker):
  def __init__ (self, **kw):
    kw.setdefault('reconnect_delay', 0.5)
    self.max_retry_delay = kw.get('max_retry_delay',16)
    super(BackoffWorker,self).__init__(**kw)

  def _handle_connect (self):
    self.reconnect_delay = 0.5
    super(BackoffWorker, self)._handle_connect()

  def open_later (self):
    self.reconnect_delay *= 2
    self.reconnect_delay = int(self.reconnect_delay)
    if self.reconnect_delay > self.max_retry_delay:
      self.reconnect_delay = self.max_retry_delay
    self.kw['reconnect_delay'] = self.reconnect_delay
    self._debug("Try again in %s seconds", self.reconnect_delay)
    from pox.core import core
    core.callDelayed(self.reconnect_delay, self.begin, **self.kw)
<|endoftext|>


<commit_before>from .calchasTreeVisitor import CalchasTreeVisitor
from .calchasTreeVisitor import StdSympyFunction
from .calchasTree import FunctionCall as CalchasFunctionCall
from sympy import *

class SympyTreeBuilder(CalchasTreeVisitor):
    def __init__(self):
        super().__init__()

    def visitCalchasId(self, tree, debug=False):
        if not tree.getId() in self.variables.keys():
            if debug :
                print("New id: "+str(tree.getId()))
            self.variables[tree.getId()] = symbols(tree.getId())
        return self.variables[tree.getId()]

    def visitCalchasNumber(self, tree, debug=False):
        if tree.getType() == int:
            return Integer(tree.getNumber())
        return QQ(tree.getNumber())

    def visitCalchasFunctionCall(self, tree, debug=False):
        function = tree.getFunction()
        if isinstance(function, CalchasFunctionCall):
            function = self.visitCalchasTree(function, debug=debug)
            args = tuple(self.visitCalchasTree(e, debug=debug) for e in tree.getArgs())
            return function(*args)
        if not function in self.functions.keys():
            if debug:
                print("New function: "+str(function))
            self.functions[function] = StdSympyFunction(symbols(function, cls=Function), len(tree.getArgs()))
        if not (self.functions[function]).isArity(tree.getArity()):
            if debug:
                print("Syntax error")
            raise SyntaxError
        f = self.functions[function]
        args = tuple(self.visitCalchasTree(e, debug=debug) for e in tree.getArgs())
        if debug:
            print(tuple(type(e) for e in args))
            print(args)
        a = f.callFunctionWithUnrearrangedArgs(args, debug=debug)
        if debug:
            print("visitCalchasFunctionCall: ", end="")
            print(type(a))
            print("                          ", end="")
            print(a)
        return a

    def toSympyTree(self, tree, debug=False):
        if debug:
            print("toSympyTree > tree: ", end="")
            print(tree)
        c = simplify(self.visitCalchasTree(tree, debug=debug))
        return c
<commit_msg>It's better to use a function which exists<commit_after>from .calchasTreeVisitor import CalchasTreeVisitor
from .calchasTreeVisitor import StdSympyFunction
from .calchasTree import FunctionCall as CalchasFunctionCall
from sympy import *

class SympyTreeBuilder(CalchasTreeVisitor):
    def __init__(self):
        super().__init__()

    def visitCalchasId(self, tree, debug=False):
        if not tree.getId() in self.variables.keys():
            if debug :
                print("New id: "+str(tree.getId()))
            self.variables[tree.getId()] = symbols(tree.getId())
        return self.variables[tree.getId()]

    def visitCalchasNumber(self, tree, debug=False):
        if tree.getType() == int:
            return Integer(tree.getNumber())
        return Rational(str(tree.getNumber()))

    def visitCalchasFunctionCall(self, tree, debug=False):
        function = tree.getFunction()
        if isinstance(function, CalchasFunctionCall):
            function = self.visitCalchasTree(function, debug=debug)
            args = tuple(self.visitCalchasTree(e, debug=debug) for e in tree.getArgs())
            return function(*args)
        if not function in self.functions.keys():
            if debug:
                print("New function: "+str(function))
            self.functions[function] = StdSympyFunction(symbols(function, cls=Function), len(tree.getArgs()))
        if not (self.functions[function]).isArity(tree.getArity()):
            if debug:
                print("Syntax error")
            raise SyntaxError
        f = self.functions[function]
        args = tuple(self.visitCalchasTree(e, debug=debug) for e in tree.getArgs())
        if debug:
            print(tuple(type(e) for e in args))
            print(args)
        a = f.callFunctionWithUnrearrangedArgs(args, debug=debug)
        if debug:
            print("visitCalchasFunctionCall: ", end="")
            print(type(a))
            print("                          ", end="")
            print(a)
        return a

    def toSympyTree(self, tree, debug=False):
        if debug:
            print("toSympyTree > tree: ", end="")
            print(tree)
        c = simplify(self.visitCalchasTree(tree, debug=debug))
        return c
<|endoftext|>


<commit_before>"""
Log messages and related utilities.
"""

from __future__ import unicode_literals

import time
from uuid import uuid4

from six import text_type as unicode

from pyrsistent import PClass, pmap, thaw, pmap_field

MESSAGE_TYPE_FIELD = 'message_type'
TASK_UUID_FIELD = 'task_uuid'
TASK_LEVEL_FIELD = 'task_level'
TIMESTAMP_FIELD = 'timestamp'

EXCEPTION_FIELD = 'exception'
REASON_FIELD = 'reason'

class Message(object):
    """
    A log message.

    Messages are basically dictionaries, mapping "fields" to "values". Field
    names should not start with C{'_'}, as those are reserved for system use
    (e.g. C{"_id"} is used by Elasticsearch for unique message identifiers and
    may be auto-populated by logstash).
    """
    # Overrideable for testing purposes:
    _time = time.time

    @classmethod
    def new(_class, _serializer=None, **fields):
        """
        Create a new L{Message}.

        The keyword arguments will become the initial contents of the L{Message}.

        @param _serializer: A positional argument, either C{None} or a
            L{eliot._validation._MessageSerializer} with which a
            L{eliot.ILogger} may choose to serialize the message. If you're
            using L{eliot.MessageType} this will be populated for you.

        @return: The new L{Message}
        """
        return _class(fields, _serializer)

    @classmethod
    def log(_class, **fields):
        """
        Write a new L{Message} to the default L{Logger}.

        The keyword arguments will become contents of the L{Message}.
        """
        _class.new(**fields).write()

    def __init__(self, contents, serializer=None):
        """
        You can also use L{Message.new} to create L{Message} objects.

        @param contents: The contents of this L{Message}, a C{dict} whose keys
           must be C{unicode}, or text that has been UTF-8 encoded to
           C{bytes}.

        @param serializer: Either C{None}, or
            L{eliot._validation._MessageSerializer} with which a
            L{eliot.Logger} may choose to serialize the message. If you're
            using L{eliot.MessageType} this will be populated for you.
        """
        self._contents = pmap(contents)
        self._serializer = serializer

    def bind(self, **fields):
        """
        Return a new L{Message} with this message's contents plus the
        additional given bindings.
        """
        return Message(self._contents.update(fields), self._serializer)

    def contents(self):
        """
        Return a copy of L{Message} contents.
        """
        return dict(self._contents)

    def _timestamp(self):
        """
        Return the current time.
        """
        return self._time()

    def _freeze(self, action=None):
        """
        Freeze this message for logging, registering it with C{action}.

        @param action: The L{Action} which is the context for this message. If
            C{None}, the L{Action} will be deduced from the current call
            stack.

        @return: A L{PMap} with added C{timestamp}, C{task_uuid}, and
            C{task_level} entries.
        """
        if action is None:
            action = currentAction()
        if action is None:
            task_uuid = unicode(uuid4())
            task_level = [1]
        else:
            task_uuid = action._identification[TASK_UUID_FIELD]
            task_level = thaw(action._nextTaskLevel().level)
        timestamp = self._timestamp()
        return self._contents.update({
            TIMESTAMP_FIELD: timestamp,
            TASK_UUID_FIELD: task_uuid,
            TASK_LEVEL_FIELD: task_level,
        })

    def write(self, logger=None, action=None):
        """
        Write the message to the given logger.

        This will additionally include a timestamp, the action context if any,
        and any other fields.

        Byte field names will be converted to Unicode.

        @type logger: L{eliot.ILogger} or C{None} indicating the default one.

        @param action: The L{Action} which is the context for this message. If
            C{None}, the L{Action} will be deduced from the current call
            stack.
        """
        if logger is None:
            logger = _output._DEFAULT_LOGGER
        logged_dict = self._freeze(action=action)
        logger.write(dict(logged_dict), self._serializer)

class WrittenMessage(PClass):
    """
    A L{Message} that has been logged.

    @ivar _logged_dict: The originally logged dictionary.
    """

    _logged_dict = pmap_field(
        (str, unicode),
        (str, unicode, int, float, list, dict, tuple, type(None)))

    @property
    def timestamp(self):
        """
        The Unix timestamp of when the message was logged.
        """
        return self._logged_dict[TIMESTAMP_FIELD]

    @property
    def task_uuid(self):
        """
        The UUID of the task in which the message was logged.
        """
        return self._logged_dict[TASK_UUID_FIELD]

    @property
    def task_level(self):
        """
        The L{TaskLevel} of this message appears within the task.
        """
        return TaskLevel(level=self._logged_dict[TASK_LEVEL_FIELD])

    @property
    def contents(self):
        """
        A C{PMap}, the message contents without Eliot metadata.
        """
        return self._logged_dict.discard(
            TIMESTAMP_FIELD).discard(TASK_UUID_FIELD).discard(TASK_LEVEL_FIELD)

    @classmethod
    def from_dict(cls, logged_dictionary):
        """
        Reconstruct a L{WrittenMessage} from a logged dictionary.

        @param logged_dictionary: A C{PMap} representing a parsed log entry.
        @return: A L{WrittenMessage} for that dictionary.
        """
        return cls(_logged_dict=logged_dictionary)

    def as_dict(self):
        """
        Return the dictionary that was used to write this message.

        @return: A C{dict}, as might be logged by Eliot.
        """
        return self._logged_dict

# Import at end to deal with circular imports:
from ._action import currentAction, TaskLevel
from . import _output

<commit_msg>Realistically this may be full of random other stuff so give up on enforcement.<commit_after>"""
Log messages and related utilities.
"""

from __future__ import unicode_literals

import time
from uuid import uuid4

from six import text_type as unicode

from pyrsistent import PClass, pmap, thaw, pmap_field

MESSAGE_TYPE_FIELD = 'message_type'
TASK_UUID_FIELD = 'task_uuid'
TASK_LEVEL_FIELD = 'task_level'
TIMESTAMP_FIELD = 'timestamp'

EXCEPTION_FIELD = 'exception'
REASON_FIELD = 'reason'

class Message(object):
    """
    A log message.

    Messages are basically dictionaries, mapping "fields" to "values". Field
    names should not start with C{'_'}, as those are reserved for system use
    (e.g. C{"_id"} is used by Elasticsearch for unique message identifiers and
    may be auto-populated by logstash).
    """
    # Overrideable for testing purposes:
    _time = time.time

    @classmethod
    def new(_class, _serializer=None, **fields):
        """
        Create a new L{Message}.

        The keyword arguments will become the initial contents of the L{Message}.

        @param _serializer: A positional argument, either C{None} or a
            L{eliot._validation._MessageSerializer} with which a
            L{eliot.ILogger} may choose to serialize the message. If you're
            using L{eliot.MessageType} this will be populated for you.

        @return: The new L{Message}
        """
        return _class(fields, _serializer)

    @classmethod
    def log(_class, **fields):
        """
        Write a new L{Message} to the default L{Logger}.

        The keyword arguments will become contents of the L{Message}.
        """
        _class.new(**fields).write()

    def __init__(self, contents, serializer=None):
        """
        You can also use L{Message.new} to create L{Message} objects.

        @param contents: The contents of this L{Message}, a C{dict} whose keys
           must be C{unicode}, or text that has been UTF-8 encoded to
           C{bytes}.

        @param serializer: Either C{None}, or
            L{eliot._validation._MessageSerializer} with which a
            L{eliot.Logger} may choose to serialize the message. If you're
            using L{eliot.MessageType} this will be populated for you.
        """
        self._contents = pmap(contents)
        self._serializer = serializer

    def bind(self, **fields):
        """
        Return a new L{Message} with this message's contents plus the
        additional given bindings.
        """
        return Message(self._contents.update(fields), self._serializer)

    def contents(self):
        """
        Return a copy of L{Message} contents.
        """
        return dict(self._contents)

    def _timestamp(self):
        """
        Return the current time.
        """
        return self._time()

    def _freeze(self, action=None):
        """
        Freeze this message for logging, registering it with C{action}.

        @param action: The L{Action} which is the context for this message. If
            C{None}, the L{Action} will be deduced from the current call
            stack.

        @return: A L{PMap} with added C{timestamp}, C{task_uuid}, and
            C{task_level} entries.
        """
        if action is None:
            action = currentAction()
        if action is None:
            task_uuid = unicode(uuid4())
            task_level = [1]
        else:
            task_uuid = action._identification[TASK_UUID_FIELD]
            task_level = thaw(action._nextTaskLevel().level)
        timestamp = self._timestamp()
        return self._contents.update({
            TIMESTAMP_FIELD: timestamp,
            TASK_UUID_FIELD: task_uuid,
            TASK_LEVEL_FIELD: task_level,
        })

    def write(self, logger=None, action=None):
        """
        Write the message to the given logger.

        This will additionally include a timestamp, the action context if any,
        and any other fields.

        Byte field names will be converted to Unicode.

        @type logger: L{eliot.ILogger} or C{None} indicating the default one.

        @param action: The L{Action} which is the context for this message. If
            C{None}, the L{Action} will be deduced from the current call
            stack.
        """
        if logger is None:
            logger = _output._DEFAULT_LOGGER
        logged_dict = self._freeze(action=action)
        logger.write(dict(logged_dict), self._serializer)

class WrittenMessage(PClass):
    """
    A L{Message} that has been logged.

    @ivar _logged_dict: The originally logged dictionary.
    """

    _logged_dict = pmap_field((str, unicode), object)

    @property
    def timestamp(self):
        """
        The Unix timestamp of when the message was logged.
        """
        return self._logged_dict[TIMESTAMP_FIELD]

    @property
    def task_uuid(self):
        """
        The UUID of the task in which the message was logged.
        """
        return self._logged_dict[TASK_UUID_FIELD]

    @property
    def task_level(self):
        """
        The L{TaskLevel} of this message appears within the task.
        """
        return TaskLevel(level=self._logged_dict[TASK_LEVEL_FIELD])

    @property
    def contents(self):
        """
        A C{PMap}, the message contents without Eliot metadata.
        """
        return self._logged_dict.discard(
            TIMESTAMP_FIELD).discard(TASK_UUID_FIELD).discard(TASK_LEVEL_FIELD)

    @classmethod
    def from_dict(cls, logged_dictionary):
        """
        Reconstruct a L{WrittenMessage} from a logged dictionary.

        @param logged_dictionary: A C{PMap} representing a parsed log entry.
        @return: A L{WrittenMessage} for that dictionary.
        """
        return cls(_logged_dict=logged_dictionary)

    def as_dict(self):
        """
        Return the dictionary that was used to write this message.

        @return: A C{dict}, as might be logged by Eliot.
        """
        return self._logged_dict

# Import at end to deal with circular imports:
from ._action import currentAction, TaskLevel
from . import _output

<|endoftext|>


<commit_before>import os
import itertools
import threading
import subprocess
from ply import lex, yacc
from nltk.corpus import wordnet
from collections import namedtuple

from ppp_datamodel import Resource, Triple, Missing

class ParserException(Exception):
    pass

FORMS_ETRE = frozenset(filter(bool, '''
        suis es est sommes êtes sont étais était
        étions êtiez étaient
        '''.split(' ')))
FORMS_AVOIR = frozenset(filter(bool, '''
        ai as a avons avez ont avais avait
        avions aviez avaient
        '''.split(' ')))

def is_etre(v):
    if v.lower() in FORMS_ETRE:
        return True
    else:
        return False
def is_avoir(v):
    if v.lower() in FORMS_AVOIR:
        return True
    else:
        return False

# from https://web.archive.org/web/20140724023431/http://www.computing.dcu.ie/~acahill/tagset.html
"""
tags = {
        'CC': 'COORDINATING_CONJUNCTION',
        'CD': 'CARDINAL_NUMBER',
        'DT': 'DETERMINER',
        'EX': 'EXISTENTIAL_THERE',
        'FW': 'FOREIGN_WORD',
        'IN': 'P_OR_S_CONJUNCTION', # PREPOSITION_OR_SUBORDINATING_CONJUNCTION
        'JJ': 'ADJECTIVE',
        'JJR': 'COMPARATIVE_ADJECTIVE',
        'JJS': 'SUPERLATIVE_ADJECTIVE',
        'LS': 'LIST_ITEM_MARKER',
        'MD': 'MODAL',
        'NN': 'PLURAL_NOUN',
        'NNP': 'PROPER_SINGULAR_NOUN',
        'NNPS': 'PROPER_PLURAL_NOUN',
        'NNS': 'PROPER_NOUN',
        'PDT': 'PREDETERMINER',
        'POS': 'POSSESSIVE_ENDING',
        'PRP': 'PERSONAL_PRONOUN',
        'PRP$': 'POSSIVE_PRONOUN',
        'RB': 'ADVERBE',
        'RBR': 'COMPARATIVE_ADVERB',
        'RP': 'PARTICLE',
        'SYM': 'SYMBOL',
        'TO': 'TO',
        'UH': 'INTERJECTION',
        'VB': 'BASE_VERB',
        'VBD': 'PAST_VERB',
        'VBG': 'GERUND_VERB',
        'VBN': 'PAST_PARTICIPLE_VERB',
        'VBP': 'SINGULAR_VERB',
        'VBZ': 'SINGULAR_VERB',
        'WDT': 'WH_DETERMINER',
        'WP': 'WH_PRONOUN',
        'WP$': 'POSSESSIVE_WH_PRONOUN',
        'WRB': 'WH_ADVERB',
        }
"""

"""
tags = {
        'ADJWH': 'MOT_INTERROGATIF',
        'DET': 'DETERMINANT',
        'NC': 'NOM_COMMUN',
        'V': 'VERBE',
        #'PUNC': 'PONCTUATION',
       }
"""

class Nom(str):
    pass
class Determinant(str):
    pass
class IntroCompl(str):
    pass
class Verbe(str):
    pass
class VerbeSujet(namedtuple('_VS', 'verbe sujet')):
    pass
class MotInterrogatif(str):
    pass
class Hole:
    pass

tokens = ('MOT_INTERROGATIF', 'DETERMINANT', 'NOM', 'VERBE',
        'VERBE_SUJET', 'INTRO_COMPL', 'GROUPE_NOMINAL',
        )

t_ignore = ' \n'

def t_error(t):
    raise ParserException('Illegal string `%r`' % t.value)

def t_PONCTUATION(t):
    '''[^ "]*_PUNC '''
    return None

def t_MOT_INTERROGATIF(t):
    '''[^ ]*_(ADJWH|PROWH) '''
    t.value = MotInterrogatif(t.value.rsplit('_', 1)[0])
    return t
def t_DETERMINANT(t):
    '''[^ ]*_DET '''
    t.value = Determinant(t.value.rsplit('_', 1)[0])
    return t
def t_NOM(t):
    '''[^ ]*_(NC|NPP)'''
    t.value = Nom(t.value.rsplit('_', 1)[0])
    return t
def t_quotes(t):
    '''"_PUNC (?P<content>[^"]*) "_PUNC'''
    t.type = 'NOM'
    c = lexer.lexmatch.group('content')
    t.value = ' '.join(x.rsplit('_', 1)[0] for x in c.split(' ')).strip()
    return t
def t_VERBE(t):
    '''[^ -]*_(V|VPP)[ ]'''
    t.value = Verbe(t.value.rsplit('_', 1)[0])
    return t
def t_VERBE_SUJET(t):
    '''[^ ]*-[^ ]*_VPP '''
    t.value = t.value.rsplit('_', 1)[0]
    (verb, noun) = t.value.split('-', 1)
    t.value = VerbeSujet(Verbe(verb), Nom(noun))
    return t
def t_INTRO_COMPL(t):
    '''[^ ]*_P '''
    t.value = IntroCompl(t.value.rsplit('_', 1)[0])
    return t
def t_GROUPE_NOMINAL(t):
    '''[^ ]*['’][^ ]*_VINF '''
    t.value = t.value.rsplit('_', 1)[0]
    (det, noun) = t.value.replace('’', '\'').split('\'', 1)
    t.value = GroupeNominal(Determinant(det), [], Nom(noun))
    return t

lexer = lex.lex()

precedence = (
        ('right', 'INTRO_COMPL'),
        )

class GroupeNominal(namedtuple('_GN', 'determinant qualificateurs nom')):
    pass

def det_to_subject(det):
    det = det.lower()
    if det in ('mon', 'ma', 'mes', 'me', 'je', 'moi'):
        return Resource('je')
    elif det in ('ton', 'ta', 'tes', 'te', 'tu', 'toi'):
        return Resource('toi')
    elif det in ('son', 'sa', 'ses', 'lui', 'elle', 'il', 'iel'):
        return Resource('iel')
    else:
        return None
def gn_to_subject(gn):
    if gn.determinant:
        return det_to_subject(gn.determinant)
    else:
        return None
def gn_to_triple(gn):
    if gn.qualificateurs:
        # TODO
        return Triple(
                gn_to_triple(gn.qualificateurs[0]),
                Resource(gn.nom),
                Missing())
    elif gn_to_subject(gn):
        return Triple(
                gn_to_subject(gn),
                Resource(gn.nom),
                Missing())
    else:
        return Resource(gn.nom)

def noun_to_predicate(noun):
    l = wordnet.synsets(noun, pos='n', lang='fra')
    fr_nouns = itertools.chain.from_iterable(
            x.lemma_names('fra') for x in l)
    fr_nouns = list(fr_nouns)
    if fr_nouns:
        return Resource(fr_nouns[0]) # TODO multiple
    else:
        return Resource(noun)
def verb_to_predicate(verb):
    l = wordnet.synsets(verb, lang='fra')
    # XXX maybe add pos='v'? (note: wouldn't work for others than infinitive)
    lemmas = itertools.chain.from_iterable(
            x.lemmas() for x in l if x.pos() == 'v' or True)
    drf = itertools.chain.from_iterable(
            x.derivationally_related_forms() for x in lemmas)
    nouns = (
            x for x in drf
            if x.synset().pos() == 'n')
    fr_nouns = itertools.chain.from_iterable(
            x.synset().lemma_names('fra') for x in nouns)
    fr_nouns = list(fr_nouns)
    if fr_nouns:
        return Resource(fr_nouns[0]) # TODO multiple
    else:
        return Resource(verb)

def p_verbe_simple(t):
    '''verbe : VERBE'''
    t[0] = t[1]
def p_verbe_compose(t):
    '''verbe : VERBE VERBE'''
    if is_etre(t[1]) or is_avoir(t[1]):
        t[0] = Verbe(t[2])
    else:
        assert False

def p_groupe_nominal_nom(t):
    '''groupe_nominal : NOM'''
    t[0] = GroupeNominal(None, [], t[1])
def p_groupe_nominal_gn(t):
    '''groupe_nominal : GROUPE_NOMINAL'''
    t[0] = t[1]
def p_groupe_nominal_simple(t):
    '''groupe_nominal_simple : DETERMINANT NOM'''
    t[0] = GroupeNominal(t[1], [], t[2])
def p_groupe_nominal_base(t):
    '''groupe_nominal : groupe_nominal_simple'''
    t[0] = t[1]
def p_groupe_nominal_det_nom_compl(t):
    '''groupe_nominal : groupe_nominal INTRO_COMPL groupe_nominal'''
    t[0] = GroupeNominal(t[1].determinant, [t[3]], t[1].nom)

def p_question_verb_first(t):
    '''question : MOT_INTERROGATIF verbe groupe_nominal'''
    word = t[1].lower()
    if word in ('quel', 'quelle', 'qui'):
        if is_etre(t[2]):
            t[0] = gn_to_triple(t[3])
        else:
            t[0] = Triple(
                    gn_to_triple(t[3]),
                    verb_to_predicate(t[2]),
                    Missing())
    else:
        assert False, word

def p_question_noun_first(t):
    '''question : MOT_INTERROGATIF NOM VERBE_SUJET'''
    word = t[1].lower()
    if word in ('quel', 'quelle', 'qui'):
        if is_avoir(t[3].verbe) or is_etre(t[3].verbe):
            t[0] = Triple(det_to_subject(t[3].sujet),
                    noun_to_predicate(t[2]),
                    Missing())
        else:
            assert False, t[3]
    else:
        assert False, word

def p_error(t):
    if t is None:
        raise ParserException('Unknown PLY error.')
    else:
        raise ParserException("Syntax error at '%s' (%s)" %
                (t.value, t.type))

parser = yacc.yacc(start='question', write_tables=0)

interpreters = [
        '/usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java',
        '/usr/local/bin/java',
        '/usr/bin/java',
        ]

tagger_options = [
        '-mx300m',
        '-classpath', 'stanford-postagger-full-2014-10-26/stanford-postagger.jar',
        'edu.stanford.nlp.tagger.maxent.MaxentTagger', '-model',
        'stanford-postagger-full-2014-10-26/models/french.tagger',
        ]
class Tagger:
    """Runs an instance of a POS tagger and provides it through the 'tag'
    method.
    Thread-safe."""
    def __init__(self):
        self.lock = threading.Lock()
        for interpreter in interpreters:
            if os.path.isfile(interpreter):
                self.start(interpreter)
                break
        else:
            self.start('/usr/bin/env java')
    def start(self, interpreter):
        self.process = subprocess.Popen(
                [interpreter] + tagger_options,
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                stderr=None,
                universal_newlines=True)

    def tag(self, s):
        with self.lock:
            self.process.stdin.write(s + '\n')
            self.process.stdin.flush()
            return self.process.stdout.readline()

tagger = Tagger()

def to_datamodel(t):
    return t

def parse(s):
    s = tagger.tag(s) + ' '
    """
    lexer.input(s)
    while True:
        tok = lexer.token()
        if not tok:
            break
        else:
            print(tok)"""
    return to_datamodel(parser.parse(s, lexer=lexer))
<commit_msg>Add Oracle's JVM.<commit_after>import os
import itertools
import threading
import subprocess
from ply import lex, yacc
from nltk.corpus import wordnet
from collections import namedtuple

from ppp_datamodel import Resource, Triple, Missing

class ParserException(Exception):
    pass

FORMS_ETRE = frozenset(filter(bool, '''
        suis es est sommes êtes sont étais était
        étions êtiez étaient
        '''.split(' ')))
FORMS_AVOIR = frozenset(filter(bool, '''
        ai as a avons avez ont avais avait
        avions aviez avaient
        '''.split(' ')))

def is_etre(v):
    if v.lower() in FORMS_ETRE:
        return True
    else:
        return False
def is_avoir(v):
    if v.lower() in FORMS_AVOIR:
        return True
    else:
        return False

# from https://web.archive.org/web/20140724023431/http://www.computing.dcu.ie/~acahill/tagset.html
"""
tags = {
        'CC': 'COORDINATING_CONJUNCTION',
        'CD': 'CARDINAL_NUMBER',
        'DT': 'DETERMINER',
        'EX': 'EXISTENTIAL_THERE',
        'FW': 'FOREIGN_WORD',
        'IN': 'P_OR_S_CONJUNCTION', # PREPOSITION_OR_SUBORDINATING_CONJUNCTION
        'JJ': 'ADJECTIVE',
        'JJR': 'COMPARATIVE_ADJECTIVE',
        'JJS': 'SUPERLATIVE_ADJECTIVE',
        'LS': 'LIST_ITEM_MARKER',
        'MD': 'MODAL',
        'NN': 'PLURAL_NOUN',
        'NNP': 'PROPER_SINGULAR_NOUN',
        'NNPS': 'PROPER_PLURAL_NOUN',
        'NNS': 'PROPER_NOUN',
        'PDT': 'PREDETERMINER',
        'POS': 'POSSESSIVE_ENDING',
        'PRP': 'PERSONAL_PRONOUN',
        'PRP$': 'POSSIVE_PRONOUN',
        'RB': 'ADVERBE',
        'RBR': 'COMPARATIVE_ADVERB',
        'RP': 'PARTICLE',
        'SYM': 'SYMBOL',
        'TO': 'TO',
        'UH': 'INTERJECTION',
        'VB': 'BASE_VERB',
        'VBD': 'PAST_VERB',
        'VBG': 'GERUND_VERB',
        'VBN': 'PAST_PARTICIPLE_VERB',
        'VBP': 'SINGULAR_VERB',
        'VBZ': 'SINGULAR_VERB',
        'WDT': 'WH_DETERMINER',
        'WP': 'WH_PRONOUN',
        'WP$': 'POSSESSIVE_WH_PRONOUN',
        'WRB': 'WH_ADVERB',
        }
"""

"""
tags = {
        'ADJWH': 'MOT_INTERROGATIF',
        'DET': 'DETERMINANT',
        'NC': 'NOM_COMMUN',
        'V': 'VERBE',
        #'PUNC': 'PONCTUATION',
       }
"""

class Nom(str):
    pass
class Determinant(str):
    pass
class IntroCompl(str):
    pass
class Verbe(str):
    pass
class VerbeSujet(namedtuple('_VS', 'verbe sujet')):
    pass
class MotInterrogatif(str):
    pass
class Hole:
    pass

tokens = ('MOT_INTERROGATIF', 'DETERMINANT', 'NOM', 'VERBE',
        'VERBE_SUJET', 'INTRO_COMPL', 'GROUPE_NOMINAL',
        )

t_ignore = ' \n'

def t_error(t):
    raise ParserException('Illegal string `%r`' % t.value)

def t_PONCTUATION(t):
    '''[^ "]*_PUNC '''
    return None

def t_MOT_INTERROGATIF(t):
    '''[^ ]*_(ADJWH|PROWH) '''
    t.value = MotInterrogatif(t.value.rsplit('_', 1)[0])
    return t
def t_DETERMINANT(t):
    '''[^ ]*_DET '''
    t.value = Determinant(t.value.rsplit('_', 1)[0])
    return t
def t_NOM(t):
    '''[^ ]*_(NC|NPP)'''
    t.value = Nom(t.value.rsplit('_', 1)[0])
    return t
def t_quotes(t):
    '''"_PUNC (?P<content>[^"]*) "_PUNC'''
    t.type = 'NOM'
    c = lexer.lexmatch.group('content')
    t.value = ' '.join(x.rsplit('_', 1)[0] for x in c.split(' ')).strip()
    return t
def t_VERBE(t):
    '''[^ -]*_(V|VPP)[ ]'''
    t.value = Verbe(t.value.rsplit('_', 1)[0])
    return t
def t_VERBE_SUJET(t):
    '''[^ ]*-[^ ]*_VPP '''
    t.value = t.value.rsplit('_', 1)[0]
    (verb, noun) = t.value.split('-', 1)
    t.value = VerbeSujet(Verbe(verb), Nom(noun))
    return t
def t_INTRO_COMPL(t):
    '''[^ ]*_P '''
    t.value = IntroCompl(t.value.rsplit('_', 1)[0])
    return t
def t_GROUPE_NOMINAL(t):
    '''[^ ]*['’][^ ]*_VINF '''
    t.value = t.value.rsplit('_', 1)[0]
    (det, noun) = t.value.replace('’', '\'').split('\'', 1)
    t.value = GroupeNominal(Determinant(det), [], Nom(noun))
    return t

lexer = lex.lex()

precedence = (
        ('right', 'INTRO_COMPL'),
        )

class GroupeNominal(namedtuple('_GN', 'determinant qualificateurs nom')):
    pass

def det_to_subject(det):
    det = det.lower()
    if det in ('mon', 'ma', 'mes', 'me', 'je', 'moi'):
        return Resource('je')
    elif det in ('ton', 'ta', 'tes', 'te', 'tu', 'toi'):
        return Resource('toi')
    elif det in ('son', 'sa', 'ses', 'lui', 'elle', 'il', 'iel'):
        return Resource('iel')
    else:
        return None
def gn_to_subject(gn):
    if gn.determinant:
        return det_to_subject(gn.determinant)
    else:
        return None
def gn_to_triple(gn):
    if gn.qualificateurs:
        # TODO
        return Triple(
                gn_to_triple(gn.qualificateurs[0]),
                Resource(gn.nom),
                Missing())
    elif gn_to_subject(gn):
        return Triple(
                gn_to_subject(gn),
                Resource(gn.nom),
                Missing())
    else:
        return Resource(gn.nom)

def noun_to_predicate(noun):
    l = wordnet.synsets(noun, pos='n', lang='fra')
    fr_nouns = itertools.chain.from_iterable(
            x.lemma_names('fra') for x in l)
    fr_nouns = list(fr_nouns)
    if fr_nouns:
        return Resource(fr_nouns[0]) # TODO multiple
    else:
        return Resource(noun)
def verb_to_predicate(verb):
    l = wordnet.synsets(verb, lang='fra')
    # XXX maybe add pos='v'? (note: wouldn't work for others than infinitive)
    lemmas = itertools.chain.from_iterable(
            x.lemmas() for x in l if x.pos() == 'v' or True)
    drf = itertools.chain.from_iterable(
            x.derivationally_related_forms() for x in lemmas)
    nouns = (
            x for x in drf
            if x.synset().pos() == 'n')
    fr_nouns = itertools.chain.from_iterable(
            x.synset().lemma_names('fra') for x in nouns)
    fr_nouns = list(fr_nouns)
    if fr_nouns:
        return Resource(fr_nouns[0]) # TODO multiple
    else:
        return Resource(verb)

def p_verbe_simple(t):
    '''verbe : VERBE'''
    t[0] = t[1]
def p_verbe_compose(t):
    '''verbe : VERBE VERBE'''
    if is_etre(t[1]) or is_avoir(t[1]):
        t[0] = Verbe(t[2])
    else:
        assert False

def p_groupe_nominal_nom(t):
    '''groupe_nominal : NOM'''
    t[0] = GroupeNominal(None, [], t[1])
def p_groupe_nominal_gn(t):
    '''groupe_nominal : GROUPE_NOMINAL'''
    t[0] = t[1]
def p_groupe_nominal_simple(t):
    '''groupe_nominal_simple : DETERMINANT NOM'''
    t[0] = GroupeNominal(t[1], [], t[2])
def p_groupe_nominal_base(t):
    '''groupe_nominal : groupe_nominal_simple'''
    t[0] = t[1]
def p_groupe_nominal_det_nom_compl(t):
    '''groupe_nominal : groupe_nominal INTRO_COMPL groupe_nominal'''
    t[0] = GroupeNominal(t[1].determinant, [t[3]], t[1].nom)

def p_question_verb_first(t):
    '''question : MOT_INTERROGATIF verbe groupe_nominal'''
    word = t[1].lower()
    if word in ('quel', 'quelle', 'qui'):
        if is_etre(t[2]):
            t[0] = gn_to_triple(t[3])
        else:
            t[0] = Triple(
                    gn_to_triple(t[3]),
                    verb_to_predicate(t[2]),
                    Missing())
    else:
        assert False, word

def p_question_noun_first(t):
    '''question : MOT_INTERROGATIF NOM VERBE_SUJET'''
    word = t[1].lower()
    if word in ('quel', 'quelle', 'qui'):
        if is_avoir(t[3].verbe) or is_etre(t[3].verbe):
            t[0] = Triple(det_to_subject(t[3].sujet),
                    noun_to_predicate(t[2]),
                    Missing())
        else:
            assert False, t[3]
    else:
        assert False, word

def p_error(t):
    if t is None:
        raise ParserException('Unknown PLY error.')
    else:
        raise ParserException("Syntax error at '%s' (%s)" %
                (t.value, t.type))

parser = yacc.yacc(start='question', write_tables=0)

interpreters = [
        '/usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java',
        '/usr/lib/jvm/java-8-oracle/bin/java',
        '/usr/local/bin/java',
        '/usr/bin/java',
        ]

tagger_options = [
        '-mx300m',
        '-classpath', 'stanford-postagger-full-2014-10-26/stanford-postagger.jar',
        'edu.stanford.nlp.tagger.maxent.MaxentTagger', '-model',
        'stanford-postagger-full-2014-10-26/models/french.tagger',
        ]
class Tagger:
    """Runs an instance of a POS tagger and provides it through the 'tag'
    method.
    Thread-safe."""
    def __init__(self):
        self.lock = threading.Lock()
        for interpreter in interpreters:
            if os.path.isfile(interpreter):
                self.start(interpreter)
                break
        else:
            self.start('/usr/bin/env java')
    def start(self, interpreter):
        print('Using interpreter: %s' % interpreter)
        self.process = subprocess.Popen(
                [interpreter] + tagger_options,
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                stderr=None,
                universal_newlines=True)

    def tag(self, s):
        with self.lock:
            self.process.stdin.write(s + '\n')
            self.process.stdin.flush()
            return self.process.stdout.readline()

tagger = Tagger()

def to_datamodel(t):
    return t

def parse(s):
    s = tagger.tag(s) + ' '
    """
    lexer.input(s)
    while True:
        tok = lexer.token()
        if not tok:
            break
        else:
            print(tok)"""
    return to_datamodel(parser.parse(s, lexer=lexer))
<|endoftext|>


<commit_before>import dropq
import os
from .helpers import package_up_vars
import json
import requests
from requests.exceptions import Timeout, RequestException
from .helpers import arrange_totals_by_row
import requests_mock
requests_mock.Mocker.TEST_PREFIX = 'dropq'

dqversion_info = dropq._version.get_versions()
dropq_version = ".".join([dqversion_info['version'], dqversion_info['full'][:6]])
NUM_BUDGET_YEARS = int(os.environ.get('NUM_BUDGET_YEARS', 10))
START_YEAR = int(os.environ.get('START_YEAR', 2016))
#Hard fail on lack of dropq workers
dropq_workers = os.environ.get('DROPQ_WORKERS', '')
DROPQ_WORKERS = dropq_workers.split(",")
ENFORCE_REMOTE_VERSION_CHECK = os.environ.get('ENFORCE_VERSION', 'False') == 'True'
TIMEOUT_IN_SECONDS = 1.0
MAX_ATTEMPTS_SUBMIT_JOB = 20
TAXCALC_RESULTS_TOTAL_ROW_KEYS = dropq.dropq.total_row_names
ELASTIC_RESULTS_TOTAL_ROW_KEYS = ["gdp_elasticity"]

class JobFailError(Exception):
    '''An Exception to raise when a remote jobs has failed'''
    pass

class DropqCompute(object):

    def __init__(self):
        pass

    def remote_submit_job(self, theurl, data, timeout=TIMEOUT_IN_SECONDS):
        response = requests.post(theurl, data=data, timeout=timeout)
        return response

    def remote_results_ready(self, theurl, params):
        job_response = requests.get(theurl, params=params)
        return job_response

    def remote_retrieve_results(self, theurl, params):
        job_response = requests.get(theurl, params=params)
        return job_response

    def submit_dropq_calculation(self, mods, first_budget_year):
        url_template = "http://{hn}/dropq_start_job"
        return self.submit_calculation(mods, first_budget_year, url_template)

    def submit_elastic_calculation(self, mods, first_budget_year):
        url_template = "http://{hn}/elastic_gdp_start_job"
        return self.submit_calculation(mods, first_budget_year, url_template,
                                       start_budget_year=1)

    def submit_calculation(self, mods, first_budget_year, url_template,
                           start_budget_year=0):
        print "mods is ", mods
        user_mods = package_up_vars(mods, first_budget_year)
        if not bool(user_mods):
            return False
        print "user_mods is ", user_mods
        print "submit work"
        user_mods={first_budget_year:user_mods}
        years = list(range(start_budget_year,NUM_BUDGET_YEARS))

        hostnames = DROPQ_WORKERS
        num_hosts = len(hostnames)
        data = {}
        data['user_mods'] = json.dumps(user_mods)
        job_ids = []
        hostname_idx = 0
        max_queue_length = 0
        for y in years:
            year_submitted = False
            attempts = 0
            while not year_submitted:
                data['year'] = str(y)
                theurl = url_template.format(hn=hostnames[hostname_idx])
                try:
                    response = self.remote_submit_job(theurl, data=data, timeout=TIMEOUT_IN_SECONDS)
                    response_d = response.json()
                    if response.status_code == 200:
                        year_submitted = True
                        job_ids.append((response_d['job_id'], hostnames[hostname_idx]))
                        hostname_idx = (hostname_idx + 1) % num_hosts
                        if response_d['qlength'] > max_queue_length:
                            max_queue_length = response_d['qlength']
                    else:
                        print "FAILED: ", str(y), hostnames[hostname_idx]
                        hostname_idx = (hostname_idx + 1) % num_hosts
                        attempts += 1
                except Timeout:
                    print "Couldn't submit to: ", hostnames[hostname_idx]
                    hostname_idx = (hostname_idx + 1) % num_hosts
                    attempts += 1
                except RequestException as re:
                    print "Something unexpected happened: ", re
                    hostname_idx = (hostname_idx + 1) % num_hosts
                    attempts += 1
                if attempts > MAX_ATTEMPTS_SUBMIT_JOB:
                    print "Exceeded max attempts. Bailing out."
                    raise IOError()

        return job_ids, max_queue_length

    def dropq_results_ready(self, job_ids):
        jobs_done = [False] * len(job_ids)
        for idx, id_hostname in enumerate(job_ids):
            id_, hostname = id_hostname
            result_url = "http://{hn}/dropq_query_result".format(hn=hostname)
            job_response = self.remote_results_ready(result_url, params={'job_id':id_})
            if job_response.status_code == 200: # Valid response
                rep = job_response.text
                if rep == 'YES':
                    jobs_done[idx] = True
                    print "got one!: ", id_
                elif rep == 'FAIL':
                    msg = '{0} failed on host: {1}'.format(id_, hostname)
                    raise JobFailError(msg)

        return jobs_done

    def dropq_get_results(self, job_ids):
        ans = []
        for idx, id_hostname in enumerate(job_ids):
            id_, hostname = id_hostname
            result_url = "http://{hn}/dropq_get_result".format(hn=hostname)
            job_response = self.remote_retrieve_results(result_url, params={'job_id':id_})
            if job_response.status_code == 200: # Valid response
                ans.append(job_response.json())

        mY_dec = {}
        mX_dec = {}
        df_dec = {}
        pdf_dec = {}
        cdf_dec = {}
        mY_bin = {}
        mX_bin = {}
        df_bin = {}
        pdf_bin = {}
        cdf_bin = {}
        fiscal_tots = {}
        for result in ans:
            mY_dec.update(result['mY_dec'])
            mX_dec.update(result['mX_dec'])
            df_dec.update(result['df_dec'])
            pdf_dec.update(result['pdf_dec'])
            cdf_dec.update(result['cdf_dec'])
            mY_bin.update(result['mY_bin'])
            mX_bin.update(result['mX_bin'])
            df_bin.update(result['df_bin'])
            pdf_bin.update(result['pdf_bin'])
            cdf_bin.update(result['cdf_bin'])
            fiscal_tots.update(result['fiscal_tots'])

        if ENFORCE_REMOTE_VERSION_CHECK:
            versions = [r.get('taxcalc_version', None) for r in ans]
            if not all([ver==taxcalc_version for ver in versions]):
                msg ="Got different taxcalc versions from workers. Bailing out"
                print msg
                raise IOError(msg)
            versions = [r.get('dropq_version', None) for r in ans]
            if not all([same_version(ver, dropq_version) for ver in versions]):
                msg ="Got different dropq versions from workers. Bailing out"
                print msg
                raise IOError(msg)

        fiscal_tots = arrange_totals_by_row(fiscal_tots,
                                            TAXCALC_RESULTS_TOTAL_ROW_KEYS)

        results = {'mY_dec': mY_dec, 'mX_dec': mX_dec, 'df_dec': df_dec,
                'pdf_dec': pdf_dec, 'cdf_dec': cdf_dec, 'mY_bin': mY_bin,
                'mX_bin': mX_bin, 'df_bin': df_bin, 'pdf_bin': pdf_bin,
                'cdf_bin': cdf_bin, 'fiscal_tots': fiscal_tots}

        return results

    def elastic_get_results(self, job_ids):
        ans = []
        for idx, id_hostname in enumerate(job_ids):
            id_, hostname = id_hostname
            result_url = "http://{hn}/dropq_get_result".format(hn=hostname)
            job_response = self.remote_retrieve_results(result_url, params={'job_id':id_})
            if job_response.status_code == 200: # Valid response
                ans.append(job_response.json())

        elasticity_gdp = {}
        for result in ans:
            elasticity_gdp.update(result['elasticity_gdp'])

        if ENFORCE_REMOTE_VERSION_CHECK:
            versions = [r.get('taxcalc_version', None) for r in ans]
            if not all([ver==taxcalc_version for ver in versions]):
                msg ="Got different taxcalc versions from workers. Bailing out"
                print msg
                raise IOError(msg)
            versions = [r.get('dropq_version', None) for r in ans]
            if not all([same_version(ver, dropq_version) for ver in versions]):
                msg ="Got different dropq versions from workers. Bailing out"
                print msg
                raise IOError(msg)

        elasticity_gdp[u'gdp_elasticity_0'] = u'NA'
        elasticity_gdp = arrange_totals_by_row(elasticity_gdp,
                                            ELASTIC_RESULTS_TOTAL_ROW_KEYS)

        results = {'elasticity_gdp': elasticity_gdp}

        return results

class MockCompute(DropqCompute):

    __slots__ = ('count', 'num_times_to_wait')

    def __init__(self, num_times_to_wait=0):
        self.count = 0
        # Number of times to respond 'No' before
        # replying that a job is ready
        self.num_times_to_wait = num_times_to_wait

    def remote_submit_job(self, theurl, data, timeout):
        with requests_mock.Mocker() as mock:
            resp = {'job_id': '424242', 'qlength':2}
            resp = json.dumps(resp)
            mock.register_uri('POST', '/dropq_start_job', text=resp)
            mock.register_uri('POST', '/elastic_gdp_start_job', text=resp)
            return DropqCompute.remote_submit_job(self, theurl, data, timeout)

    def remote_results_ready(self, theurl, params):
        with requests_mock.Mocker() as mock:
            if self.num_times_to_wait > 0:
                mock.register_uri('GET', '/dropq_query_result', text='NO')
                self.num_times_to_wait -= 1
            else:
                mock.register_uri('GET', '/dropq_query_result', text='YES')
            return DropqCompute.remote_results_ready(self, theurl, params)

    def remote_retrieve_results(self, theurl, params):
        mock_path = os.path.join(os.path.split(__file__)[0], "tests",
                                 "response_year_{0}.json")
        with open(mock_path.format(self.count), 'r') as f:
            text = f.read()
        self.count += 1
        with requests_mock.Mocker() as mock:
            mock.register_uri('GET', '/dropq_get_result', text=text)
            return DropqCompute.remote_retrieve_results(self, theurl, params)

class ElasticMockCompute(MockCompute):

    def remote_retrieve_results(self, theurl, params):
        self.count += 1
        text = (u'{"elasticity_gdp": {"gdp_elasticity_1": "0.00310"}, '
                '"dropq_version": "0.6.a96303", "taxcalc_version": '
                '"0.6.10d462"}')
        with requests_mock.Mocker() as mock:
            mock.register_uri('GET', '/dropq_get_result', text=text)
            return DropqCompute.remote_retrieve_results(self, theurl, params)

class MockFailedCompute(MockCompute):

    def remote_results_ready(self, theurl, params):
        with requests_mock.Mocker() as mock:
            mock.register_uri('GET', '/dropq_query_result', text='FAIL')
            return DropqCompute.remote_results_ready(self, theurl, params)
<commit_msg>Add print of job submit back in<commit_after>import dropq
import os
from .helpers import package_up_vars
import json
import requests
from requests.exceptions import Timeout, RequestException
from .helpers import arrange_totals_by_row
import requests_mock
requests_mock.Mocker.TEST_PREFIX = 'dropq'

dqversion_info = dropq._version.get_versions()
dropq_version = ".".join([dqversion_info['version'], dqversion_info['full'][:6]])
NUM_BUDGET_YEARS = int(os.environ.get('NUM_BUDGET_YEARS', 10))
START_YEAR = int(os.environ.get('START_YEAR', 2016))
#Hard fail on lack of dropq workers
dropq_workers = os.environ.get('DROPQ_WORKERS', '')
DROPQ_WORKERS = dropq_workers.split(",")
ENFORCE_REMOTE_VERSION_CHECK = os.environ.get('ENFORCE_VERSION', 'False') == 'True'
TIMEOUT_IN_SECONDS = 1.0
MAX_ATTEMPTS_SUBMIT_JOB = 20
TAXCALC_RESULTS_TOTAL_ROW_KEYS = dropq.dropq.total_row_names
ELASTIC_RESULTS_TOTAL_ROW_KEYS = ["gdp_elasticity"]

class JobFailError(Exception):
    '''An Exception to raise when a remote jobs has failed'''
    pass

class DropqCompute(object):

    def __init__(self):
        pass

    def remote_submit_job(self, theurl, data, timeout=TIMEOUT_IN_SECONDS):
        response = requests.post(theurl, data=data, timeout=timeout)
        return response

    def remote_results_ready(self, theurl, params):
        job_response = requests.get(theurl, params=params)
        return job_response

    def remote_retrieve_results(self, theurl, params):
        job_response = requests.get(theurl, params=params)
        return job_response

    def submit_dropq_calculation(self, mods, first_budget_year):
        url_template = "http://{hn}/dropq_start_job"
        return self.submit_calculation(mods, first_budget_year, url_template)

    def submit_elastic_calculation(self, mods, first_budget_year):
        url_template = "http://{hn}/elastic_gdp_start_job"
        return self.submit_calculation(mods, first_budget_year, url_template,
                                       start_budget_year=1)

    def submit_calculation(self, mods, first_budget_year, url_template,
                           start_budget_year=0):
        print "mods is ", mods
        user_mods = package_up_vars(mods, first_budget_year)
        if not bool(user_mods):
            return False
        print "user_mods is ", user_mods
        print "submit work"
        user_mods={first_budget_year:user_mods}
        years = list(range(start_budget_year,NUM_BUDGET_YEARS))

        hostnames = DROPQ_WORKERS
        num_hosts = len(hostnames)
        data = {}
        data['user_mods'] = json.dumps(user_mods)
        job_ids = []
        hostname_idx = 0
        max_queue_length = 0
        for y in years:
            year_submitted = False
            attempts = 0
            while not year_submitted:
                data['year'] = str(y)
                theurl = url_template.format(hn=hostnames[hostname_idx])
                try:
                    response = self.remote_submit_job(theurl, data=data, timeout=TIMEOUT_IN_SECONDS)
                    response_d = response.json()
                    if response.status_code == 200:
                        print "submitted: ", hostnames[hostname_idx]
                        year_submitted = True
                        job_ids.append((response_d['job_id'], hostnames[hostname_idx]))
                        hostname_idx = (hostname_idx + 1) % num_hosts
                        if response_d['qlength'] > max_queue_length:
                            max_queue_length = response_d['qlength']
                    else:
                        print "FAILED: ", str(y), hostnames[hostname_idx]
                        hostname_idx = (hostname_idx + 1) % num_hosts
                        attempts += 1
                except Timeout:
                    print "Couldn't submit to: ", hostnames[hostname_idx]
                    hostname_idx = (hostname_idx + 1) % num_hosts
                    attempts += 1
                except RequestException as re:
                    print "Something unexpected happened: ", re
                    hostname_idx = (hostname_idx + 1) % num_hosts
                    attempts += 1
                if attempts > MAX_ATTEMPTS_SUBMIT_JOB:
                    print "Exceeded max attempts. Bailing out."
                    raise IOError()

        return job_ids, max_queue_length

    def dropq_results_ready(self, job_ids):
        jobs_done = [False] * len(job_ids)
        for idx, id_hostname in enumerate(job_ids):
            id_, hostname = id_hostname
            result_url = "http://{hn}/dropq_query_result".format(hn=hostname)
            job_response = self.remote_results_ready(result_url, params={'job_id':id_})
            if job_response.status_code == 200: # Valid response
                rep = job_response.text
                if rep == 'YES':
                    jobs_done[idx] = True
                    print "got one!: ", id_
                elif rep == 'FAIL':
                    msg = '{0} failed on host: {1}'.format(id_, hostname)
                    raise JobFailError(msg)

        return jobs_done

    def dropq_get_results(self, job_ids):
        ans = []
        for idx, id_hostname in enumerate(job_ids):
            id_, hostname = id_hostname
            result_url = "http://{hn}/dropq_get_result".format(hn=hostname)
            job_response = self.remote_retrieve_results(result_url, params={'job_id':id_})
            if job_response.status_code == 200: # Valid response
                ans.append(job_response.json())

        mY_dec = {}
        mX_dec = {}
        df_dec = {}
        pdf_dec = {}
        cdf_dec = {}
        mY_bin = {}
        mX_bin = {}
        df_bin = {}
        pdf_bin = {}
        cdf_bin = {}
        fiscal_tots = {}
        for result in ans:
            mY_dec.update(result['mY_dec'])
            mX_dec.update(result['mX_dec'])
            df_dec.update(result['df_dec'])
            pdf_dec.update(result['pdf_dec'])
            cdf_dec.update(result['cdf_dec'])
            mY_bin.update(result['mY_bin'])
            mX_bin.update(result['mX_bin'])
            df_bin.update(result['df_bin'])
            pdf_bin.update(result['pdf_bin'])
            cdf_bin.update(result['cdf_bin'])
            fiscal_tots.update(result['fiscal_tots'])

        if ENFORCE_REMOTE_VERSION_CHECK:
            versions = [r.get('taxcalc_version', None) for r in ans]
            if not all([ver==taxcalc_version for ver in versions]):
                msg ="Got different taxcalc versions from workers. Bailing out"
                print msg
                raise IOError(msg)
            versions = [r.get('dropq_version', None) for r in ans]
            if not all([same_version(ver, dropq_version) for ver in versions]):
                msg ="Got different dropq versions from workers. Bailing out"
                print msg
                raise IOError(msg)

        fiscal_tots = arrange_totals_by_row(fiscal_tots,
                                            TAXCALC_RESULTS_TOTAL_ROW_KEYS)

        results = {'mY_dec': mY_dec, 'mX_dec': mX_dec, 'df_dec': df_dec,
                'pdf_dec': pdf_dec, 'cdf_dec': cdf_dec, 'mY_bin': mY_bin,
                'mX_bin': mX_bin, 'df_bin': df_bin, 'pdf_bin': pdf_bin,
                'cdf_bin': cdf_bin, 'fiscal_tots': fiscal_tots}

        return results

    def elastic_get_results(self, job_ids):
        ans = []
        for idx, id_hostname in enumerate(job_ids):
            id_, hostname = id_hostname
            result_url = "http://{hn}/dropq_get_result".format(hn=hostname)
            job_response = self.remote_retrieve_results(result_url, params={'job_id':id_})
            if job_response.status_code == 200: # Valid response
                ans.append(job_response.json())

        elasticity_gdp = {}
        for result in ans:
            elasticity_gdp.update(result['elasticity_gdp'])

        if ENFORCE_REMOTE_VERSION_CHECK:
            versions = [r.get('taxcalc_version', None) for r in ans]
            if not all([ver==taxcalc_version for ver in versions]):
                msg ="Got different taxcalc versions from workers. Bailing out"
                print msg
                raise IOError(msg)
            versions = [r.get('dropq_version', None) for r in ans]
            if not all([same_version(ver, dropq_version) for ver in versions]):
                msg ="Got different dropq versions from workers. Bailing out"
                print msg
                raise IOError(msg)

        elasticity_gdp[u'gdp_elasticity_0'] = u'NA'
        elasticity_gdp = arrange_totals_by_row(elasticity_gdp,
                                            ELASTIC_RESULTS_TOTAL_ROW_KEYS)

        results = {'elasticity_gdp': elasticity_gdp}

        return results

class MockCompute(DropqCompute):

    __slots__ = ('count', 'num_times_to_wait')

    def __init__(self, num_times_to_wait=0):
        self.count = 0
        # Number of times to respond 'No' before
        # replying that a job is ready
        self.num_times_to_wait = num_times_to_wait

    def remote_submit_job(self, theurl, data, timeout):
        with requests_mock.Mocker() as mock:
            resp = {'job_id': '424242', 'qlength':2}
            resp = json.dumps(resp)
            mock.register_uri('POST', '/dropq_start_job', text=resp)
            mock.register_uri('POST', '/elastic_gdp_start_job', text=resp)
            return DropqCompute.remote_submit_job(self, theurl, data, timeout)

    def remote_results_ready(self, theurl, params):
        with requests_mock.Mocker() as mock:
            if self.num_times_to_wait > 0:
                mock.register_uri('GET', '/dropq_query_result', text='NO')
                self.num_times_to_wait -= 1
            else:
                mock.register_uri('GET', '/dropq_query_result', text='YES')
            return DropqCompute.remote_results_ready(self, theurl, params)

    def remote_retrieve_results(self, theurl, params):
        mock_path = os.path.join(os.path.split(__file__)[0], "tests",
                                 "response_year_{0}.json")
        with open(mock_path.format(self.count), 'r') as f:
            text = f.read()
        self.count += 1
        with requests_mock.Mocker() as mock:
            mock.register_uri('GET', '/dropq_get_result', text=text)
            return DropqCompute.remote_retrieve_results(self, theurl, params)

class ElasticMockCompute(MockCompute):

    def remote_retrieve_results(self, theurl, params):
        self.count += 1
        text = (u'{"elasticity_gdp": {"gdp_elasticity_1": "0.00310"}, '
                '"dropq_version": "0.6.a96303", "taxcalc_version": '
                '"0.6.10d462"}')
        with requests_mock.Mocker() as mock:
            mock.register_uri('GET', '/dropq_get_result', text=text)
            return DropqCompute.remote_retrieve_results(self, theurl, params)

class MockFailedCompute(MockCompute):

    def remote_results_ready(self, theurl, params):
        with requests_mock.Mocker() as mock:
            mock.register_uri('GET', '/dropq_query_result', text='FAIL')
            return DropqCompute.remote_results_ready(self, theurl, params)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import sys
import pandas as pd

<commit_msg>Went horribly wrong, will improve later<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import sys
import pandas as pd

from sklearn.linear_model import LinearRegression
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier

def main():
	if len(sys.argv) < 3:
		print('Usage: ./test_logistic.py features labels')
		exit(1)

	features = pd.DataFrame.from_csv(sys.argv[1], index_col=None, header=0)
	labels = pd.Series.from_csv(sys.argv[2], index_col=None, header=None)

	features_train = features.loc[:1300]
	labels_train = labels.loc[:1300]
	features_test = features.loc[1300:]
	labels_test = labels.loc[1300:]

	lr = LinearRegression(copy_X=True, normalize=False)
	lr.fit(features_train, labels_train)

	# predictions = lr.predict(features_test)
	score = lr.score(features_test, labels_test)
	print(score)

if __name__ == '__main__':
	main()<|endoftext|>


<commit_before>from django.conf import settings
from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
from django.contrib.postgres.fields import JSONField
from django.db import models

class ActivityLog(models.Model):
    """The ActivityLog model represents one recorded event."""

    event_type = models.CharField(max_length=128)
    actor = models.ForeignKey(
        settings.AUTH_USER_MODEL,
        on_delete=models.CASCADE,
        related_name='+')
    context = JSONField(
        blank=True,
        null=True,
        help_text='Extra context information.')
    created_at = models.DateTimeField(auto_now_add=True)

    content_type = models.ForeignKey(ContentType, on_delete=models.SET_NULL)
    object_id = models.PositiveIntegerField()
    content_object = GenericForeignKey('content_type', 'object_id')

    class Meta:
        verbose_name = 'activity log'
        verbose_name_plural = 'activities logs'

    def __str__(self):
        return '{} - {}'.format(self.event_type, self.created_at)
<commit_msg>Update activity log model<commit_after>from django.conf import settings
from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
from django.contrib.postgres.fields import JSONField
from django.db import models

class ActivityLog(models.Model):
    """The ActivityLog model represents one recorded event."""

    event_type = models.CharField(max_length=128)
    actor = models.ForeignKey(
        settings.AUTH_USER_MODEL,
        on_delete=models.CASCADE,
        related_name='+')
    context = JSONField(help_text='Extra context information.')
    created_at = models.DateTimeField()

    content_type = models.ForeignKey(ContentType, on_delete=models.SET_NULL)
    object_id = models.PositiveIntegerField()
    content_object = GenericForeignKey('content_type', 'object_id')

    class Meta:
        verbose_name = 'activity log'
        verbose_name_plural = 'activities logs'

    def __str__(self):
        return '{} - {}'.format(self.event_type, self.created_at)
<|endoftext|>


<commit_before>import datetime
import time

from django.http import HttpResponse
from django.shortcuts import render_to_response, get_object_or_404
from django.utils.timezone import localtime, now
from graphite.util import json
from graphite.events import models
from graphite.render.attime import parseATTime
from django.core.urlresolvers import get_script_prefix

def to_timestamp(dt):
    return time.mktime(dt.timetuple())

class EventEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, datetime.datetime):
            return to_timestamp(obj)
        return json.JSONEncoder.default(self, obj)

def view_events(request):
    if request.method == "GET":
        context = { 'events' : fetch(request),
            'slash' : get_script_prefix()
        }
        return render_to_response("events.html", context)
    else:
        return post_event(request)

def detail(request, event_id):
    e = get_object_or_404(models.Event, pk=event_id)
    context = { 'event' : e,
       'slash' : get_script_prefix()
    }
    return render_to_response("event.html", context)

def post_event(request):
    if request.method == 'POST':
        event = json.loads(request.raw_post_data)
        assert isinstance(event, dict)

        values = {}
        values["what"] = event["what"]
        values["tags"] = event.get("tags", None)
        values["when"] = datetime.datetime.fromtimestamp(
            event.get("when", time.time()))
        if "data" in event:
            values["data"] = event["data"]

        e = models.Event(**values)
        e.save()

        return HttpResponse(status=200)
    else:
        return HttpResponse(status=405)

def get_data(request):
    if 'jsonp' in request.REQUEST:
        response = HttpResponse(
          "%s(%s)" % (request.REQUEST.get('jsonp'),
              json.dumps(fetch(request), cls=EventEncoder)),
          mimetype='text/javascript')
    else:
        response = HttpResponse(
            json.dumps(fetch(request), cls=EventEncoder),
            mimetype="application/json")
    return response

def fetch(request):
    #XXX we need to move to USE_TZ=True to get rid of localtime() conversions
    if request.GET.get("from", None) is not None:
        time_from = localtime(parseATTime(request.GET["from"])).replace(tzinfo=None)
    else:
        time_from = datetime.datetime.fromtimestamp(0)

    if request.GET.get("until", None) is not None:
        time_until = localtime(parseATTime(request.GET["until"])).replace(tzinfo=None)
    else:
        time_until = now()

    tags = request.GET.get("tags", None)
    if tags is not None:
        tags = request.GET.get("tags").split(" ")

    return [x.as_dict() for x in
            models.Event.find_events(time_from, time_until, tags=tags)]
<commit_msg>Remove use of django 1.4 timezone helpers<commit_after>import datetime
import time

from django.conf import settings
from django.core.urlresolvers import get_script_prefix
from django.http import HttpResponse
from django.shortcuts import render_to_response, get_object_or_404
from pytz import timezone

from graphite.util import json
from graphite.events import models
from graphite.render.attime import parseATTime

def to_timestamp(dt):
    return time.mktime(dt.timetuple())

class EventEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, datetime.datetime):
            return to_timestamp(obj)
        return json.JSONEncoder.default(self, obj)

def view_events(request):
    if request.method == "GET":
        context = { 'events' : fetch(request),
            'slash' : get_script_prefix()
        }
        return render_to_response("events.html", context)
    else:
        return post_event(request)

def detail(request, event_id):
    e = get_object_or_404(models.Event, pk=event_id)
    context = { 'event' : e,
       'slash' : get_script_prefix()
    }
    return render_to_response("event.html", context)

def post_event(request):
    if request.method == 'POST':
        event = json.loads(request.raw_post_data)
        assert isinstance(event, dict)

        values = {}
        values["what"] = event["what"]
        values["tags"] = event.get("tags", None)
        values["when"] = datetime.datetime.fromtimestamp(
            event.get("when", time.time()))
        if "data" in event:
            values["data"] = event["data"]

        e = models.Event(**values)
        e.save()

        return HttpResponse(status=200)
    else:
        return HttpResponse(status=405)

def get_data(request):
    if 'jsonp' in request.REQUEST:
        response = HttpResponse(
          "%s(%s)" % (request.REQUEST.get('jsonp'),
              json.dumps(fetch(request), cls=EventEncoder)),
          mimetype='text/javascript')
    else:
        response = HttpResponse(
            json.dumps(fetch(request), cls=EventEncoder),
            mimetype="application/json")
    return response

def fetch(request):
    #XXX we need to move to USE_TZ=True to get rid of naive-time conversions
    def make_naive(dt):
      tz = timezone(request.GET.get('tz', settings.TIME_ZONE))
      local_dt = dt.astimezone(tz)
      if hasattr(local_dt, 'normalize'):
        local_dt = local_dt.normalize()
      return local_dt.replace(tzinfo=None)

    if request.GET.get("from", None) is not None:
        time_from = make_naive(parseATTime(request.GET["from"]))
    else:
        time_from = datetime.datetime.fromtimestamp(0)

    if request.GET.get("until", None) is not None:
        time_until = make_naive(parseATTime(request.GET["until"]))
    else:
        time_until = datetime.datetime.now()

    tags = request.GET.get("tags", None)
    if tags is not None:
        tags = request.GET.get("tags").split(" ")

    return [x.as_dict() for x in
            models.Event.find_events(time_from, time_until, tags=tags)]
<|endoftext|>


<commit_before>from __future__ import print_function
from builtins import str
from .context import phaseflow

import fenics

def test_1d_output():

    T_h = 1.

    T_c = -1.

    w = phaseflow.run(
        output_dir = "output/test_1D_output/",
        prandtl_number = 1.,
        stefan_number = 1.,
        gravity = [0.],
        mesh = fenics.UnitIntervalMesh(1000),
        initial_values_expression = (
            "0.",
            "0.",
            "(" + str(T_h) + " - " + str(T_c) + ")*near(x[0],  0.) " + str(T_c)),
        boundary_conditions = [
            {"subspace": 0, "value_expression": [0.], "degree": 3, "location_expression": "near(x[0],  0.) | near(x[0],  1.)", "method": "topological"},
            {"subspace": 2, "value_expression": T_h, "degree": 2, "location_expression": "near(x[0],  0.)", "method": "topological"},
            {"subspace": 2, "value_expression": T_c, "degree": 2, "location_expression": "near(x[0],  1.)", "method": "topological"}],
        temperature_of_fusion = 0.01,
        regularization_smoothing_factor = 0.005,
        end_time = 0.001,
        time_step_size = 0.001)

def test_1d_velocity():

    mesh = fenics.UnitIntervalMesh(fenics.dolfin.mpi_comm_world(), 5)

    V = fenics.VectorFunctionSpace(mesh, "P", 1)

    u = fenics.Function(V)

    bc = fenics.DirichletBC(V, [10.0], "x[0] < 0.5")

    print(bc.get_boundary_values())

if __name__=="__main__":

    test_1d_output()

    test_1d_velocity()
<commit_msg>Simplified 1D unit test<commit_after>from __future__ import print_function
from builtins import str
from .context import phaseflow

import fenics

def test_1d_output():

    mesh = fenics.UnitIntervalMesh(10)

    mixed_element = phaseflow.make_mixed_fe(mesh.ufl_cell())

    W = fenics.FunctionSpace(mesh, mixed_element)

    boundaries = "near(x[0],  0.) | near(x[0],  1.)"

    w, time = phaseflow.run(
        output_dir = "output/test_1D_output/",
        initial_values = fenics.interpolate(
            fenics.Expression(("0.", "0.", "0."), element = mixed_element), W),
        boundary_conditions = [
            fenics.DirichletBC(W.sub(0), [0.], boundaries),
            fenics.DirichletBC(W.sub(2), 0., boundaries)],
        gravity = [0.],
        end_time = 0.)

def test_1d_velocity():

    mesh = fenics.UnitIntervalMesh(fenics.dolfin.mpi_comm_world(), 5)

    V = fenics.VectorFunctionSpace(mesh, "P", 1)

    u = fenics.Function(V)

    bc = fenics.DirichletBC(V, [10.0], "x[0] < 0.5")

    print(bc.get_boundary_values())

if __name__=="__main__":

    test_1d_output()

    test_1d_velocity()
<|endoftext|>


<commit_before>import angr
import nose

import os
test_location = os.path.join(os.path.dirname(os.path.realpath(str(__file__))), '../../binaries/tests/')

arch_data = { # (steps, [hit addrs], finished)
    'x86_64': (498, (0x400440, 0x400570, 0x1021c20, 0x1021980), False),  # AAAAAAAAAAAAAAAAAAAAAAA
    'i386':   (366, (0x80482f8, 0x8048440, 0x90198e0, 0x90195c0, 0x9019630), False),  # blocked on syscalls on my libc, bogged down in initializers here....
    'ppc':    (207, (0x100002e8, 0x11022f50, 0x11022eb0, 0x10000340), False),
    'ppc64':  (14, (0x10000368, 0x11047490), False),     # TODO: Improve
    'mips':   (175, (0x400470, 0x1016f20, 0x400500), False),
    'armel':  (163, (0x8348, 0x84b0, 0x10154b8, 0x1108244, 0x83a8), False)     # blocked on __kuser_cmpxchg
}

def emulate(arch):
    steps, hit_addrs, finished = arch_data[arch]
    filepath = test_location + arch + '/test_arrays'
    p = angr.Project(filepath, use_sim_procedures=False)
    pg = p.path_group([p.path_generator.full_init(args=['./test_arrays'])])
    pg2 = pg.step(until=lambda lpg: len(lpg.active) != 1,
                  step_func=lambda lpg: lpg if len(lpg.active) == 1 else lpg.prune()
                  )

    is_finished = False
    if len(pg2.active) > 0:
        path = pg2.active[0]
    elif len(pg2.deadended) > 0:
        path = pg2.deadended[0]
        is_finished = True
    elif len(pg2.errored) > 0:
        path = pg2.errored[0]
    else:
        raise ValueError("This pathgroup does not contain a path we can use for this test?")

    nose.tools.assert_greater_equal(path.length, steps)
    for addr in hit_addrs:
        nose.tools.assert_in(addr, path.addr_backtrace)

    if finished:
        nose.tools.assert_true(is_finished)

def test_emulation():
    for arch in arch_data:
        yield emulate, arch

if __name__ == '__main__':
    print 'x86_64'
    emulate('x86_64')
    print 'i386'
    emulate('i386')
    print 'ppc'
    emulate('ppc')
    print 'ppc64'
    emulate('ppc64')
    print 'mips'
    emulate('mips')
    print 'armel'
    emulate('armel')
<commit_msg>Added test_accuracy entries for aarch64/mips64<commit_after>import angr
import nose

import os
test_location = os.path.join(os.path.dirname(os.path.realpath(str(__file__))), '../../binaries/tests/')

arch_data = { # (steps, [hit addrs], finished)
    'x86_64':  (498, (0x400440, 0x400570, 0x1021c20, 0x1021980), False),  # AAAAAAAAAAAAAAAAAAAAAAA
    'i386':    (366, (0x80482f8, 0x8048440, 0x90198e0, 0x90195c0, 0x9019630), False),  # blocked on syscalls on my libc, bogged down in initializers here....
    'ppc':     (207, (0x100002e8, 0x11022f50, 0x11022eb0, 0x10000340), False),
    'ppc64':   (14, (0x10000368, 0x11047490), False),     # Needs more relocations
    'mips':    (175, (0x400470, 0x1016f20, 0x400500), False),
    'armel':   (163, (0x8348, 0x84b0, 0x10154b8, 0x1108244, 0x83a8), False),     # blocked on __kuser_cmpxchg
    'aarch64': (208, (0x4003b8, 0x400538, 0x1020b04, 0x400430), False),
    'mips64':  (207, (0x1200007e0, 0x12103b828, 0x120000870), False)
}

def emulate(arch):
    steps, hit_addrs, finished = arch_data[arch]
    filepath = test_location + arch + '/test_arrays'
    p = angr.Project(filepath, use_sim_procedures=False)
    pg = p.path_group([p.path_generator.full_init(args=['./test_arrays'])])
    pg2 = pg.step(until=lambda lpg: len(lpg.active) != 1,
                  step_func=lambda lpg: lpg if len(lpg.active) == 1 else lpg.prune()
                  )

    is_finished = False
    if len(pg2.active) > 0:
        path = pg2.active[0]
    elif len(pg2.deadended) > 0:
        path = pg2.deadended[0]
        is_finished = True
    elif len(pg2.errored) > 0:
        path = pg2.errored[0]
    else:
        raise ValueError("This pathgroup does not contain a path we can use for this test?")

    nose.tools.assert_greater_equal(path.length, steps)
    for addr in hit_addrs:
        nose.tools.assert_in(addr, path.addr_backtrace)

    if finished:
        nose.tools.assert_true(is_finished)

def test_emulation():
    for arch in arch_data:
        yield emulate, arch

if __name__ == '__main__':
    print 'aarch64'
    emulate('aarch64')
    print 'mips64'
    emulate('mips64')
    print 'x86_64'
    emulate('x86_64')
    print 'i386'
    emulate('i386')
    print 'ppc'
    emulate('ppc')
    print 'ppc64'
    emulate('ppc64')
    print 'mips'
    emulate('mips')
    print 'armel'
    emulate('armel')
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from ExecuteAll import ExecuteAll

print "Sentiment Analisys di tweet da tradurre sia positivi che negativi"
ExecuteAll('db/TrPosNegTweets.txt','results/TrPosNegtweet.xml',False)

print "Sentiment Analisys di tweet in inglese sia positivi che negativi"
ExecuteAll('db/PosNegTweets.txt','results/PosNegtweet.xml',False)

print "Sentiment Analisys di tweet da tradurre solo positivi"
ExecuteAll('db/TrPosTweets.txt','results/TrPostweet.xml',False)

print "Sentiment Analisys di tweet da tradurre solo negativi"
ExecuteAll('db/TrNegTweets.txt','results/TrNegtweet.xml',False)

print "Sentiment Analisys di tweet in inglese solo positivi"
ExecuteAll('db/PosTweets.txt','results/Postweet.xml',False)

print "Sentiment Analisys di tweet in inglese solo negativi"
ExecuteAll('db/NegTweets.txt','results/Negtweet.xml',False)

print "Sentiment Analisys dei tweet di una singola persona : Papa Francesco"
ExecuteAll('db/PopeTweets100.txt','results/Popetweet.xml',True)
<commit_msg>esempio caricamento UI<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from ExecuteAll import ExecuteAll
from gi.repository import Gtk

"""
builder = Gtk.Builder()
builder.add_from_file("GUI.glade")

window = builder.get_object("window1")
window.show_all()

Gtk.main()
"""

print "Sentiment Analisys di tweet da tradurre sia positivi che negativi"
ExecuteAll('db/TrPosNegTweets.txt','results/TrPosNegtweet.xml',False)

print "Sentiment Analisys di tweet in inglese sia positivi che negativi"
ExecuteAll('db/PosNegTweets.txt','results/PosNegtweet.xml',False)

print "Sentiment Analisys di tweet da tradurre solo positivi"
ExecuteAll('db/TrPosTweets.txt','results/TrPostweet.xml',False)

print "Sentiment Analisys di tweet da tradurre solo negativi"
ExecuteAll('db/TrNegTweets.txt','results/TrNegtweet.xml',False)

print "Sentiment Analisys di tweet in inglese solo positivi"
ExecuteAll('db/PosTweets.txt','results/Postweet.xml',False)

print "Sentiment Analisys di tweet in inglese solo negativi"
ExecuteAll('db/NegTweets.txt','results/Negtweet.xml',False)

print "Sentiment Analisys dei tweet di una singola persona : Papa Francesco"
ExecuteAll('db/PopeTweets100.txt','results/Popetweet.xml',True)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import
import unittest
from odin import adapters
from odin.utils import field_iter_items
from .resources import *

class ResourceOptionsAdapterTestCase(unittest.TestCase):
    def test_repr(self):
        options = adapters.ResourceAdapter(Book())._meta

        self.assertEqual('<Options Adapter for library.Book>', repr(options))

    def test_attribute_fields(self):
        options = adapters.ResourceAdapter(Book())._meta

        self.assertListEqual(['fiction'], [f.name for f in options.attribute_fields])

    def test_elements_fields(self):
        options = adapters.ResourceAdapter(Book())._meta

        self.assertListEqual(
            ['title', 'isbn', 'num_pages', 'rrp', 'genre', 'published', 'authors', 'publisher'],
            [f.name for f in options.element_fields]
        )

    def test_attribute_proxying(self):
        options = adapters.ResourceAdapter(Book())._meta

        self.assertEqual("Book", options.verbose_name)

class ResourceAdapterTestCase(unittest.TestCase):
    def test_field_proxy(self):
        target = adapters.ResourceAdapter(Book(title="Foo"))

        self.assertEqual('Foo', target.title,)

    def test_include_fields(self):
        book = Book(title="Foo", rrp=123.45, num_pages=10, fiction=True)

        target = adapters.ResourceAdapter(book, include=('title', 'rrp'))

        self.assertListEqual(['rrp', 'title'], sorted(target._meta.field_map.keys()))

    def test_exclude_fields(self):
        book = Book(title="Foo", rrp=123.45, num_pages=10, fiction=True)

        target = adapters.ResourceAdapter(book, exclude=('title', 'rrp'))

        self.assertListEqual(
            ['authors', 'fiction', 'genre', 'isbn', 'num_pages', 'published', 'publisher'],
            sorted(target._meta.field_map.keys()))

    def test_repr(self):
        target = adapters.ResourceAdapter(Book(title="Foo"))

        self.assertEqual('<ResourceAdapter: library.Book resource adapter>', repr(target))

    def test_iter(self):
        book = Book(title="Foo", isbn='abc123', rrp=123.45, num_pages=10, fiction=True)
        target = adapters.ResourceAdapter(book)

        actual = list((f.name, str(v)) for f, v in field_iter_items(target))
        self.assertListEqual([
            ('title', 'Foo'),
            ('isbn', 'abc123'),
            ('num_pages', '10'),
            ('rrp', '123.45'),
            ('fiction', 'True'),
            ('genre', 'None'),
            ('published', '[]'),
            ('authors', '[]'),
            ('publisher', 'None')], actual)

    def test_to_dict(self):
        publisher = Publisher()
        book = Book(title="Foo", isbn='abc123', rrp=123.45, num_pages=10, fiction=True, publisher=publisher)
        target = adapters.ResourceAdapter(book)

        self.assertDictEqual({
            'authors': [],
            'fiction': True,
            'genre': None,
            'num_pages': 10,
            'published': [],
            'publisher': publisher,
            'rrp': 123.45,
            'isbn': 'abc123',
            'title': 'Foo'
        }, target.to_dict())

    def test_apply_to(self):
        sources = [
            Library(name="Foo"),
            Publisher(name='Bar'),
            Library(name="Eek")
        ]
        result = adapters.ResourceAdapter.apply_to(sources, include=['name'])
        actuals = list(result)

        self.assertListEqual([
            {'name': 'Foo'},
            {'name': 'Bar'},
            {'name': 'Eek'}
        ], [a.to_dict() for a in actuals])
<commit_msg>Added a test for CurriedAdapter<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import
import unittest
from odin import adapters
from odin.utils import field_iter_items
from .resources import *

class ResourceOptionsAdapterTestCase(unittest.TestCase):
    def test_repr(self):
        options = adapters.ResourceAdapter(Book())._meta

        self.assertEqual('<Options Adapter for library.Book>', repr(options))

    def test_attribute_fields(self):
        options = adapters.ResourceAdapter(Book())._meta

        self.assertListEqual(['fiction'], [f.name for f in options.attribute_fields])

    def test_elements_fields(self):
        options = adapters.ResourceAdapter(Book())._meta

        self.assertListEqual(
            ['title', 'isbn', 'num_pages', 'rrp', 'genre', 'published', 'authors', 'publisher'],
            [f.name for f in options.element_fields]
        )

    def test_attribute_proxying(self):
        options = adapters.ResourceAdapter(Book())._meta

        self.assertEqual("Book", options.verbose_name)

class ResourceAdapterTestCase(unittest.TestCase):
    def test_field_proxy(self):
        target = adapters.ResourceAdapter(Book(title="Foo"))

        self.assertEqual('Foo', target.title,)

    def test_include_fields(self):
        book = Book(title="Foo", rrp=123.45, num_pages=10, fiction=True)

        target = adapters.ResourceAdapter(book, include=('title', 'rrp'))

        self.assertListEqual(['rrp', 'title'], sorted(target._meta.field_map.keys()))

    def test_exclude_fields(self):
        book = Book(title="Foo", rrp=123.45, num_pages=10, fiction=True)

        target = adapters.ResourceAdapter(book, exclude=('title', 'rrp'))

        self.assertListEqual(
            ['authors', 'fiction', 'genre', 'isbn', 'num_pages', 'published', 'publisher'],
            sorted(target._meta.field_map.keys()))

    def test_repr(self):
        target = adapters.ResourceAdapter(Book(title="Foo"))

        self.assertEqual('<ResourceAdapter: library.Book resource adapter>', repr(target))

    def test_iter(self):
        book = Book(title="Foo", isbn='abc123', rrp=123.45, num_pages=10, fiction=True)
        target = adapters.ResourceAdapter(book)

        actual = list((f.name, str(v)) for f, v in field_iter_items(target))
        self.assertListEqual([
            ('title', 'Foo'),
            ('isbn', 'abc123'),
            ('num_pages', '10'),
            ('rrp', '123.45'),
            ('fiction', 'True'),
            ('genre', 'None'),
            ('published', '[]'),
            ('authors', '[]'),
            ('publisher', 'None')], actual)

    def test_to_dict(self):
        publisher = Publisher()
        book = Book(title="Foo", isbn='abc123', rrp=123.45, num_pages=10, fiction=True, publisher=publisher)
        target = adapters.ResourceAdapter(book)

        self.assertDictEqual({
            'authors': [],
            'fiction': True,
            'genre': None,
            'num_pages': 10,
            'published': [],
            'publisher': publisher,
            'rrp': 123.45,
            'isbn': 'abc123',
            'title': 'Foo'
        }, target.to_dict())

    def test_apply_to(self):
        sources = [
            Library(name="Foo"),
            Publisher(name='Bar'),
            Library(name="Eek")
        ]
        result = adapters.ResourceAdapter.apply_to(sources, include=['name'])
        actuals = list(result)

        self.assertListEqual([
            {'name': 'Foo'},
            {'name': 'Bar'},
            {'name': 'Eek'}
        ], [a.to_dict() for a in actuals])

    def test_curried(self):
        book = Book(title="Foo", isbn='abc123', rrp=123.45, num_pages=10, fiction=True, publisher=Publisher())
        SummaryAdapter = adapters.ResourceAdapter.curry(include=('title', 'rrp', 'fiction'))
        target = SummaryAdapter(book)

        self.assertDictEqual({
            'fiction': True,
            'rrp': 123.45,
            'title': 'Foo'
        }, target.to_dict())
<|endoftext|>