<commit_before># plugin for nose tests
#
# provides the option --with-networkx-doctest
# which sets up the environment with
# import networkx as nx
# for all docstring tests in modules

from nose.plugins.doctests import Doctest, DocTestCase
import os
from nose.util import src
import networkx
import logging
log = logging.getLogger(__name__)

class NetworkXDoctest(Doctest):
    name = 'networkx-doctest'

    def options(self, parser, env=os.environ):
        """New plugin API: override to just set options. Implement
        this method instead of addOptions or add_options for normal
        options behavior with protection from OptionConflictErrors.
        """
        env_opt = 'NOSE_WITH_%s' % self.name.upper()
        env_opt = env_opt.replace('-', '_')
        parser.add_option("--with-%s" % self.name,
                          action="store_true",
                          dest=self.enableOpt,
                          default=env.get(env_opt),
                          help="Enable plugin %s: %s [%s]" %
                          (self.__class__.__name__, self.help(), env_opt))

    def loadTestsFromModule(self, module):
        log.debug("loading from %s", module)
        if not self.matches(module.__name__):
            log.debug("Doctest doesn't want module %s", module)
            return
        try:
            tests = self.finder.find(module)
        except AttributeError:
            log.exception("Attribute error loading from %s", module)
            # nose allows module.__test__ = False; doctest does not and throws
            # AttributeError
            return
        if not tests:
            log.debug("No tests found in %s", module)
            return
        tests.sort()
        module_file = src(module.__file__)
        # FIXME this breaks the id plugin somehow (tests probably don't
        # get wrapped in result proxy or something)
        cases = []
        for test in tests:
            if not test.examples:
                continue
            if not test.filename:
                test.filename = module_file
            test.globs = {'__builtins__':__builtins__,
                          '__file__':'__main__',
                          '__name__':'__main__',
                          'nx':networkx}
            cases.append(DocTestCase(test, result_var=self.doctest_result_var))
        if cases:
            yield self.suiteClass(cases, context=module, can_split=False)

<commit_msg>auto "import networkx" for doctests<commit_after># plugin for nose tests
#
# provides the option --with-networkx-doctest
# which sets up the environment with
# import networkx as nx
# for all docstring tests in modules

from nose.plugins.doctests import Doctest, DocTestCase
import os
from nose.util import src
import networkx
import logging
log = logging.getLogger(__name__)

class NetworkXDoctest(Doctest):
    name = 'networkx-doctest'

    def options(self, parser, env=os.environ):
        """New plugin API: override to just set options. Implement
        this method instead of addOptions or add_options for normal
        options behavior with protection from OptionConflictErrors.
        """
        env_opt = 'NOSE_WITH_%s' % self.name.upper()
        env_opt = env_opt.replace('-', '_')
        parser.add_option("--with-%s" % self.name,
                          action="store_true",
                          dest=self.enableOpt,
                          default=env.get(env_opt),
                          help="Enable plugin %s: %s [%s]" %
                          (self.__class__.__name__, self.help(), env_opt))

    def loadTestsFromModule(self, module):
        log.debug("loading from %s", module)
        if not self.matches(module.__name__):
            log.debug("Doctest doesn't want module %s", module)
            return
        try:
            tests = self.finder.find(module)
        except AttributeError:
            log.exception("Attribute error loading from %s", module)
            # nose allows module.__test__ = False; doctest does not and throws
            # AttributeError
            return
        if not tests:
            log.debug("No tests found in %s", module)
            return
        tests.sort()
        module_file = src(module.__file__)
        # FIXME this breaks the id plugin somehow (tests probably don't
        # get wrapped in result proxy or something)
        cases = []
        for test in tests:
            if not test.examples:
                continue
            if not test.filename:
                test.filename = module_file
            test.globs = {'__builtins__':__builtins__,
                          '__file__':'__main__',
                          '__name__':'__main__',
                          'nx':networkx,
                          'networkx':networkx}
            cases.append(DocTestCase(test, result_var=self.doctest_result_var))
        if cases:
            yield self.suiteClass(cases, context=module, can_split=False)

<|endoftext|>


<commit_before>"""Drudge for wick-style algebraic systems.

In this module, we have a abstract base class for Wick-style algebraic systems,
as well as function helpful for its subclasses.
"""

import abc
import collections

from pyspark import RDD

from .drudge import Drudge
from .term import Term
from .utils import sympy_key

class WickDrudge(Drudge, abc.ABC):
    """Drudge for Wick-style algebras.

    A Wick-style algebra is an algebraic system where the commutator between any
    generators of the algebra is a simple scalar value.  This drudge will
    attempt to put the vectors into normal order based on the given comparator
    and contractor by Wick theorem.

    """

    @abc.abstractproperty
    def contractor(self):
        """Get the contractor for the algebraic system.

        The contractor is going to be called with two vectors to return the
        value of their contraction.

        """
        pass

    @abc.abstractproperty
    def phase(self):
        """Get the phase for the commutation rule.

        The phase should be a constant defining the phase of the commutation
        rule.
        """
        pass

    @abc.abstractproperty
    def comparator(self):
        """Get the comparator for the canonicalized vectors.

        The normal ordering operation will be performed according to this
        comparator.  It will be called with two **canonicalized vectors** for a
        boolean value.  True should be returned if the first given vector is
        less than the second vector.  The two vectors will be attempted to be
        transposed when False is returned.

        """
        pass

    def normal_order(self, terms: RDD):
        """Normal order the terms according to generalized Wick theorem.

        The actual expansion is based on the information given in the subclasses
        by the abstract properties.

        """
        comparator = self.comparator
        contractor = self.contractor
        phase = self.phase
        symms = self.symms
        return terms.flatMap(lambda term: wick_expand(
            term, comparator=comparator, contractor=contractor, phase=phase,
            symms=symms.value
        ))

#
# Utility functions.
#

def wick_expand(term: Term, comparator, contractor, phase, symms=None):
    """Expand a Term by wick theorem.

    When the comparator is None, it is assumed that only terms with all the
    vectors contracted will be kept, as in the case in vacuum expectation value
    evaluation in many-body physics.

    """

    symms = {} if symms is None else symms
    contr_all = comparator is None
    vecs = term.vecs
    n_vecs = len(vecs)

    if n_vecs == 0:
        return [term]
    elif n_vecs == 1:
        return [] if contr_all else [term]

    # First the vectors and contractions need to be formed as required by the
    # Wick expander.

    if contr_all:
        contrs = _get_all_contrs(vecs, contractor)
        vec_order = list(range(n_vecs))
    else:
        term = _preproc_term(term, symms)
        vecs = term.vecs
        vec_order, contrs = _sort_vecs(vecs, comparator, contractor)

    expander = _WickExpander(vecs, vec_order, contrs, phase, contr_all)
    expanded = expander.expand(term.amp)

    return [Term(term.sums, i[0], i[1]) for i in expanded]

def _sort_vecs(vecs, comparator, contractor):
    """Sort the vectors and get the contraction values.

    Here insertion sort is used to sort the vectors into the normal order
    required by the comparator.
    """

    n_vecs = len(vecs)
    contrs = [{} for _ in range(n_vecs)]

    vec_order = list(range(0, n_vecs))

    front = 2
    pivot = 1
    while pivot < n_vecs:

        pivot_i = vec_order[pivot]
        pivot_vec = vecs[pivot_i]
        prev = pivot - 1

        if pivot == 0 or comparator(vecs[vec_order[prev]], pivot_vec):
            pivot, front = front, front + 1
        else:

            prev_i = vec_order[prev]
            prev_vec = vecs[prev_i]
            vec_order[prev], vec_order[pivot] = pivot_i, prev_i

            contr_val = contractor(prev_vec, pivot_vec)
            if contr_val != 0:
                contrs[prev_i][pivot_i] = contr_val
            pivot -= 1

        continue

    return vec_order, contrs

def _preproc_term(term, symms):
    """Prepare the term for Wick expansion.

    This is the preparation task for normal ordering.  The term will be
    canonicalized with all the vectors considered the same.
    """

    # Make the dummies factory to canonicalize the vectors.
    dumms = collections.defaultdict(list)
    for i, v in term.sums:
        dumms[v].append(i)
    for i in dumms.values():
        i.sort(key=sympy_key)

    canon_term = (
        term.canon(symms=symms, vec_colour=lambda idx, vec: 0)
            .reset_dumms(dumms)[0]
    )

    return canon_term

def _get_all_contrs(vecs, contractor):
    """Generate all possible contractions.

    This function is going to be called when we do not actually need to normal
    order the vectors and only need the results where all the vectors are
    contracted.
    """
    n_vecs = len(vecs)
    contrs = []

    for i in range(n_vecs):
        curr_contrs = {}
        for j in range(i, n_vecs):
            vec_prev = vecs[i]
            vec_lat = vecs[j]
            contr_val = contractor(vec_prev, vec_lat)
            if contr_val != 0:
                curr_contrs[j] = contr_val
            continue
        contrs.append(curr_contrs)
        continue

    return contrs

class _WickExpander:
    """Expander of vectors based on Wick theorem.

    Here, the problem should be specified by a sequence ``vecs``, where each
    entry should be a pair of a unique integral identifier for the vector and
    the actual vector.  The given contractions will be queried with the
    identifier of the first vector for a dictionary of non-vanishing
    contractions with vectors **later** in the sequence.
    """

    def __init__(self, vecs, vec_order, contrs, phase, contr_all):
        """Initialize the expander."""

        self.vecs = vecs
        self.vec_order = vec_order
        self.contrs = contrs
        self.phase = phase
        self.contr_all = contr_all
        self.n_vecs = len(self.vecs)

    def expand(self, base_amp):
        """Make the Wick expansion."""

        expanded = []
        avail = [True for _ in range(self.n_vecs)]
        self._add_terms(expanded, base_amp, avail, 0)
        return expanded

    def _add_terms(self, expanded, amp, avail, pivot):
        """Add terms recursively."""

        vecs = self.vecs
        vec_order = self.vec_order
        contrs = self.contrs
        contr_all = self.contr_all
        phase = self.phase

        if pivot == self.n_vecs - 1:
            # Add the current term.
            if not contr_all or all(not i for i in avail):
                rem_idxes = [i for i in vec_order if avail[i]]
                final_phase = _get_perm_phase(rem_idxes, phase)
                expanded.append((
                    final_phase * amp, [vecs[i] for i in rem_idxes]
                ))
            return

        pivot_contrs = contrs[pivot]
        if contr_all and len(pivot_contrs) == 0:
            return

        if not contr_all:
            self._add_terms(expanded, amp, avail, pivot + 1)

        avail[pivot] = False
        n_vecs_between = 0
        for vec_idx in range(pivot + 1, self.n_vecs):
            if avail[vec_idx]:
                if vec_idx in pivot_contrs:
                    avail[vec_idx] = False
                    contr_val = pivot_contrs[vec_idx]
                    contr_phase = phase ** n_vecs_between
                    self._add_terms(
                        expanded, amp * contr_val * contr_phase,
                        avail, pivot + 1
                    )
                    avail[vec_idx] = True
                n_vecs_between += 1
            continue

        avail[pivot] = True
        return

def _get_perm_phase(order, phase):
    """Get the phase of the given permutation of points."""
    n_points = len(order)
    return phase ** sum(
        1 for i in range(n_points) for j in range(i + 1, n_points)
        if order[i] > order[j]
    )
<commit_msg>Fix Wick expansion core<commit_after>"""Drudge for wick-style algebraic systems.

In this module, we have a abstract base class for Wick-style algebraic systems,
as well as function helpful for its subclasses.
"""

import abc
import collections

from pyspark import RDD

from .drudge import Drudge
from .term import Term
from .utils import sympy_key

class WickDrudge(Drudge, abc.ABC):
    """Drudge for Wick-style algebras.

    A Wick-style algebra is an algebraic system where the commutator between any
    generators of the algebra is a simple scalar value.  This drudge will
    attempt to put the vectors into normal order based on the given comparator
    and contractor by Wick theorem.

    """

    @abc.abstractproperty
    def contractor(self):
        """Get the contractor for the algebraic system.

        The contractor is going to be called with two vectors to return the
        value of their contraction.

        """
        pass

    @abc.abstractproperty
    def phase(self):
        """Get the phase for the commutation rule.

        The phase should be a constant defining the phase of the commutation
        rule.
        """
        pass

    @abc.abstractproperty
    def comparator(self):
        """Get the comparator for the canonicalized vectors.

        The normal ordering operation will be performed according to this
        comparator.  It will be called with two **canonicalized vectors** for a
        boolean value.  True should be returned if the first given vector is
        less than the second vector.  The two vectors will be attempted to be
        transposed when False is returned.

        """
        pass

    def normal_order(self, terms: RDD):
        """Normal order the terms according to generalized Wick theorem.

        The actual expansion is based on the information given in the subclasses
        by the abstract properties.

        """
        comparator = self.comparator
        contractor = self.contractor
        phase = self.phase
        symms = self.symms
        return terms.flatMap(lambda term: wick_expand(
            term, comparator=comparator, contractor=contractor, phase=phase,
            symms=symms.value
        ))

#
# Utility functions.
#

def wick_expand(term: Term, comparator, contractor, phase, symms=None):
    """Expand a Term by wick theorem.

    When the comparator is None, it is assumed that only terms with all the
    vectors contracted will be kept, as in the case in vacuum expectation value
    evaluation in many-body physics.

    """

    symms = {} if symms is None else symms
    contr_all = comparator is None
    vecs = term.vecs
    n_vecs = len(vecs)

    if n_vecs == 0:
        return [term]
    elif n_vecs == 1:
        return [] if contr_all else [term]

    # First the vectors and contractions need to be formed as required by the
    # Wick expander.

    if contr_all:
        contrs = _get_all_contrs(vecs, contractor)
        vec_order = list(range(n_vecs))
    else:
        term = _preproc_term(term, symms)
        vecs = term.vecs
        vec_order, contrs = _sort_vecs(vecs, comparator, contractor)

    expander = _WickExpander(vecs, vec_order, contrs, phase, contr_all)
    expanded = expander.expand(term.amp)

    return [Term(term.sums, i[0], i[1]) for i in expanded]

def _sort_vecs(vecs, comparator, contractor):
    """Sort the vectors and get the contraction values.

    Here insertion sort is used to sort the vectors into the normal order
    required by the comparator.
    """

    n_vecs = len(vecs)
    contrs = [{} for _ in range(n_vecs)]

    vec_order = list(range(0, n_vecs))

    front = 2
    pivot = 1
    while pivot < n_vecs:

        pivot_i = vec_order[pivot]
        pivot_vec = vecs[pivot_i]
        prev = pivot - 1

        if pivot == 0 or comparator(vecs[vec_order[prev]], pivot_vec):
            pivot, front = front, front + 1
        else:

            prev_i = vec_order[prev]
            prev_vec = vecs[prev_i]
            vec_order[prev], vec_order[pivot] = pivot_i, prev_i

            contr_val = contractor(prev_vec, pivot_vec)
            if contr_val != 0:
                contrs[prev_i][pivot_i] = contr_val
            pivot -= 1

        continue

    return vec_order, contrs

def _preproc_term(term, symms):
    """Prepare the term for Wick expansion.

    This is the preparation task for normal ordering.  The term will be
    canonicalized with all the vectors considered the same.
    """

    # Make the dummies factory to canonicalize the vectors.
    dumms = collections.defaultdict(list)
    for i, v in term.sums:
        dumms[v].append(i)
    for i in dumms.values():
        i.sort(key=sympy_key)

    canon_term = (
        term.canon(symms=symms, vec_colour=lambda idx, vec: 0)
            .reset_dumms(dumms)[0]
    )

    return canon_term

def _get_all_contrs(vecs, contractor):
    """Generate all possible contractions.

    This function is going to be called when we do not actually need to normal
    order the vectors and only need the results where all the vectors are
    contracted.
    """
    n_vecs = len(vecs)
    contrs = []

    for i in range(n_vecs):
        curr_contrs = {}
        for j in range(i, n_vecs):
            vec_prev = vecs[i]
            vec_lat = vecs[j]
            contr_val = contractor(vec_prev, vec_lat)
            if contr_val != 0:
                curr_contrs[j] = contr_val
            continue
        contrs.append(curr_contrs)
        continue

    return contrs

class _WickExpander:
    """Expander of vectors based on Wick theorem.

    Here, the problem should be specified by a sequence ``vecs``, where each
    entry should be a pair of a unique integral identifier for the vector and
    the actual vector.  The given contractions will be queried with the
    identifier of the first vector for a dictionary of non-vanishing
    contractions with vectors **later** in the sequence.
    """

    def __init__(self, vecs, vec_order, contrs, phase, contr_all):
        """Initialize the expander."""

        self.vecs = vecs
        self.vec_order = vec_order
        self.contrs = contrs
        self.phase = phase
        self.contr_all = contr_all
        self.n_vecs = len(self.vecs)

    def expand(self, base_amp):
        """Make the Wick expansion."""

        expanded = []
        avail = [True for _ in range(self.n_vecs)]
        self._add_terms(expanded, base_amp, avail, 0)
        return expanded

    def _add_terms(self, expanded, amp, avail, pivot):
        """Add terms recursively."""

        vecs = self.vecs
        n_vecs = len(vecs)
        vec_order = self.vec_order
        contrs = self.contrs
        contr_all = self.contr_all
        phase = self.phase

        # Find the actual pivot, which has to be available.
        try:
            pivot = next(i for i in range(pivot, n_vecs - 1) if avail[i])
        except StopIteration:
            # When everything is already decided, add the current term.
            if not contr_all or all(not i for i in avail):
                rem_idxes = [i for i in vec_order if avail[i]]
                final_phase = _get_perm_phase(rem_idxes, phase)
                expanded.append((
                    final_phase * amp, [vecs[i] for i in rem_idxes]
                ))
            return

        pivot_contrs = contrs[pivot]
        if contr_all and len(pivot_contrs) == 0:
            return

        if not contr_all:
            self._add_terms(expanded, amp, avail, pivot + 1)

        avail[pivot] = False
        n_vecs_between = 0
        for vec_idx in range(pivot + 1, self.n_vecs):
            if avail[vec_idx]:
                if vec_idx in pivot_contrs:
                    avail[vec_idx] = False
                    contr_val = pivot_contrs[vec_idx]
                    contr_phase = phase ** n_vecs_between
                    self._add_terms(
                        expanded, amp * contr_val * contr_phase,
                        avail, pivot + 1
                    )
                    avail[vec_idx] = True
                n_vecs_between += 1
            continue

        avail[pivot] = True
        return

def _get_perm_phase(order, phase):
    """Get the phase of the given permutation of points."""
    n_points = len(order)
    return phase ** sum(
        1 for i in range(n_points) for j in range(i + 1, n_points)
        if order[i] > order[j]
    )
<|endoftext|>


<commit_before>#!/user/bin/env python

import time
import datetime
import threading
import sys
import os
import signal
from traceback import print_exception

import logbook
import beanstalkc
import anyjson

import tweetsclient

def beanstalk(host='localhost', port=11300, watch=None, use=None):
    beanstalk = beanstalkc.Connection(host=host, port=port)
    if use:
        beanstalk.use(use)
    if watch:
        beanstalk.watch(watch)
    return beanstalk

def configure_log_handler(application_name, loglevel, output):
    if isinstance(loglevel, (str, unicode)):
        loglevel = getattr(logbook, loglevel.upper())

    if not isinstance(loglevel, int):
        raise TypeError("configure_log_handler expects loglevel to be either an integer or a string corresponding to an integer attribute of the logbook module.")

    if output == 'syslog':
        log_handler = logbook.SyslogHandler(
            application_name=application_name,
            facility='user',
            bubble=False,
            level=loglevel)
    elif output == '-' or not output:
        log_handler = logbook.StderrHandler(
            level=loglevel,
            bubble=False)
    else:
        log_handler = logbook.FileHandler(
            filename=output,
            encoding='utf-8',
            level=loglevel,
            bubble=False)

    return log_handler

def run_with_restart(fn, max_restart=0, args=(), kwargs={}):
    restartCounter = 0
    while True:
        try:
            return apply(fn, args, kwargs)
        except AssertionError:
            raise
        except Exception as e:
            logbook.error("Unhandled exception of type {exctype}: {exception}",
                          exctype=type(e),
                          exception=str(e))

            restartCounter += 1
            if max_restart and restartCounter > max_restart:
                logbook.critical("Alreadying restarted {nth} times. Exiting.",
                                 nth=restartCounter)
            else:
                # Sleep longer each time we restart, but no more than 5 minutes
                delay = min(restartCounter * 15, 300)
                logbook.error("Restarting for {nth} time in {sec} seconds.",
                              nth=restartCounter,
                              sec=delay)
                time.sleep(delay)

def restart_process(signum, frame):
    """
    Replaces the current process with a new process invoked
    using the same command line.
    """
    os.execl(sys.executable, sys.executable, *sys.argv)

def start_heartbeat_thread(heart):
    """
    Triggers a regular heartbeat from a background thread
    for scripts that making blocking calls and thus can't
    heartbeat from their main loop.
    """
    def _heartbeat():
        while True:
            heart.sleep()
            heart.beat()

    heartbeat = threading.Thread(target=_heartbeat)
    # This causes the heartbeat thread to die with the main thread
    heartbeat.daemon = True
    heartbeat.start()

def start_watchdog_thread(heart):
    """
    Watch a heartbeat file and restart when the file mtime is either
    too old or too far in the future.
    """

    def _watchdog():
        while True:
            heart.sleep()
            try:
                stat = os.stat(heart.filepath)
                mtime = datetime.datetime.fromtimestamp(stat.st_mtime)
            except OSError as e:
                if e.errno == 2: # No such file or directory
                    logbook.warning("Heartbeat file disappeared, restarting via SIGHUP.")
                    os.kill(heart.pid, signal.SIGHUP)
                    return
                else:
                    raise

            now = datetime.datetime.now()
            if mtime >= now:
                logbook.warning("Heartbeat file mtime is in the future, restarting via SIGHUP.")
                os.kill(heart.pid, signal.SIGHUP)
                return

    watchdog = threading.Thread(target=_watchdog)
    # This causes the watchdog thread to die with the main thread
    watchdog.daemon = True
    watchdog.start()

class Heart(object):
    """
    Updates the access and modification timestamp of a
    file every `interval` seconds.
    """
    def __init__(self):
        self.last_beat = datetime.datetime.now()

        config = tweetsclient.Config().get()
        try:
            self.interval = datetime.timedelta(seconds=float(config.get('tweets-client', 'heartbeat_interval')))
        except:
            logbook.warning("No heartbeat_interval configuration parameter, skipping heartbeat.")
            raise StopIteration

        try:
            directory = config.get('tweets-client', 'heartbeats_directory')
        except:
            logbook.warning("No heartbeats_directory configuration parameter, skipping heartbeat.")
            raise StopIteration

        if not os.path.isdir(directory):
            logbook.warning("The heartbeats_directory parameter ({0}) is not a directory.",
                             directory)
            raise StopIteration

        scriptname = os.path.basename(sys.argv[0])
        self.filepath = os.path.join(directory, scriptname)

        start_time = datetime.datetime.now().isoformat()
        self.pid = os.getpid()
        with file(self.filepath, 'w') as fil:
            fil.write(anyjson.serialize({
                'pid': self.pid,
                'started': start_time
            }))

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        if (exc_type, exc_value, traceback) == (None, None, None):
            os.unlink(self.filepath)
        else:
            with file(self.filepath, 'w') as outf:
                print_exception(exc_type, exc_value, traceback, 1000, outf)

    def sleep(self):
        while True:
            now = datetime.datetime.now()
            since = now - self.last_beat
            if since >= self.interval:
                return
            else:
                time.sleep(self.interval.total_seconds() * 0.10)

    def beat(self):
        now = datetime.datetime.now()
        since = now - self.last_beat
        if since >= self.interval:
            os.utime(self.filepath, None)
            self.last_beat = now
            return True
        else:
            return False

<commit_msg>Made the watchdog more granular than the heartbeat thread.<commit_after>#!/user/bin/env python

import time
import datetime
import threading
import sys
import os
import signal
from traceback import print_exception

import logbook
import beanstalkc
import anyjson

import tweetsclient

def beanstalk(host='localhost', port=11300, watch=None, use=None):
    beanstalk = beanstalkc.Connection(host=host, port=port)
    if use:
        beanstalk.use(use)
    if watch:
        beanstalk.watch(watch)
    return beanstalk

def configure_log_handler(application_name, loglevel, output):
    if isinstance(loglevel, (str, unicode)):
        loglevel = getattr(logbook, loglevel.upper())

    if not isinstance(loglevel, int):
        raise TypeError("configure_log_handler expects loglevel to be either an integer or a string corresponding to an integer attribute of the logbook module.")

    if output == 'syslog':
        log_handler = logbook.SyslogHandler(
            application_name=application_name,
            facility='user',
            bubble=False,
            level=loglevel)
    elif output == '-' or not output:
        log_handler = logbook.StderrHandler(
            level=loglevel,
            bubble=False)
    else:
        log_handler = logbook.FileHandler(
            filename=output,
            encoding='utf-8',
            level=loglevel,
            bubble=False)

    return log_handler

def run_with_restart(fn, max_restart=0, args=(), kwargs={}):
    restartCounter = 0
    while True:
        try:
            return apply(fn, args, kwargs)
        except AssertionError:
            raise
        except Exception as e:
            logbook.error("Unhandled exception of type {exctype}: {exception}",
                          exctype=type(e),
                          exception=str(e))

            restartCounter += 1
            if max_restart and restartCounter > max_restart:
                logbook.critical("Alreadying restarted {nth} times. Exiting.",
                                 nth=restartCounter)
            else:
                # Sleep longer each time we restart, but no more than 5 minutes
                delay = min(restartCounter * 15, 300)
                logbook.error("Restarting for {nth} time in {sec} seconds.",
                              nth=restartCounter,
                              sec=delay)
                time.sleep(delay)

def restart_process(signum, frame):
    """
    Replaces the current process with a new process invoked
    using the same command line.
    """
    os.execl(sys.executable, sys.executable, *sys.argv)

def start_heartbeat_thread(heart):
    """
    Triggers a regular heartbeat from a background thread
    for scripts that making blocking calls and thus can't
    heartbeat from their main loop.
    """
    def _heartbeat():
        while True:
            heart.sleep()
            heart.beat()

    heartbeat = threading.Thread(target=_heartbeat)
    # This causes the heartbeat thread to die with the main thread
    heartbeat.daemon = True
    heartbeat.start()

def start_watchdog_thread(heart):
    """
    Watch a heartbeat file and restart when the file mtime is either
    too old or too far in the future.
    """

    def _watchdog():
        while True:
            time.sleep(heart.interval.total_seconds() * 0.10)
            try:
                stat = os.stat(heart.filepath)
                mtime = datetime.datetime.fromtimestamp(stat.st_mtime)
            except OSError as e:
                if e.errno == 2: # No such file or directory
                    logbook.warning("Heartbeat file disappeared, restarting via SIGHUP.")
                    os.kill(heart.pid, signal.SIGHUP)
                    return
                else:
                    raise

            now = datetime.datetime.now()
            if mtime >= now:
                logbook.warning("Heartbeat file mtime is in the future, restarting via SIGHUP.")
                os.kill(heart.pid, signal.SIGHUP)
                return

    watchdog = threading.Thread(target=_watchdog)
    # This causes the watchdog thread to die with the main thread
    watchdog.daemon = True
    watchdog.start()

class Heart(object):
    """
    Updates the access and modification timestamp of a
    file every `interval` seconds.
    """
    def __init__(self):
        self.last_beat = datetime.datetime.now()

        config = tweetsclient.Config().get()
        try:
            self.interval = datetime.timedelta(seconds=float(config.get('tweets-client', 'heartbeat_interval')))
        except:
            logbook.warning("No heartbeat_interval configuration parameter, skipping heartbeat.")
            raise StopIteration

        try:
            directory = config.get('tweets-client', 'heartbeats_directory')
        except:
            logbook.warning("No heartbeats_directory configuration parameter, skipping heartbeat.")
            raise StopIteration

        if not os.path.isdir(directory):
            logbook.warning("The heartbeats_directory parameter ({0}) is not a directory.",
                             directory)
            raise StopIteration

        scriptname = os.path.basename(sys.argv[0])
        self.filepath = os.path.join(directory, scriptname)

        start_time = datetime.datetime.now().isoformat()
        self.pid = os.getpid()
        with file(self.filepath, 'w') as fil:
            fil.write(anyjson.serialize({
                'pid': self.pid,
                'started': start_time
            }))

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        if (exc_type, exc_value, traceback) == (None, None, None):
            os.unlink(self.filepath)
        else:
            with file(self.filepath, 'w') as outf:
                print_exception(exc_type, exc_value, traceback, 1000, outf)

    def sleep(self):
        while True:
            now = datetime.datetime.now()
            since = now - self.last_beat
            if since >= self.interval:
                return
            else:
                time.sleep(self.interval.total_seconds() * 0.10)

    def beat(self):
        now = datetime.datetime.now()
        since = now - self.last_beat
        if since >= self.interval:
            os.utime(self.filepath, None)
            self.last_beat = now
            return True
        else:
            return False

<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

from sqlalchemy.schema import FetchedValue
from sqlalchemy.dialects import postgresql as pg
from sqlalchemy.ext.declarative import declared_attr
from sqlalchemy.orm import relationship
from sqlalchemy.sql.expression import text

from warehouse import db
from warehouse.database.mixins import UUIDPrimaryKeyMixin, TimeStampedMixin
from warehouse.database.schema import TableDDL
from warehouse.database.utils import table_args

class Project(UUIDPrimaryKeyMixin, TimeStampedMixin, db.Model):

    __tablename__ = "projects"
    __table_args__ = declared_attr(table_args((
        TableDDL("""
            CREATE OR REPLACE FUNCTION normalize_name()
            RETURNS trigger AS $$
            BEGIN
                NEW.normalized = lower(regexp_replace(new.name, '[^A-Za-z0-9.]+', '-'));
                return NEW;
            END;
            $$ LANGUAGE plpgsql;

            CREATE TRIGGER %(table)s_normalize_name
            BEFORE INSERT OR UPDATE
            ON %(table)s
            FOR EACH ROW
            EXECUTE PROCEDURE normalize_name();
        """),
    )))

    name = db.Column(db.UnicodeText, unique=True, nullable=False)
    normalized = db.Column(db.UnicodeText, unique=True, nullable=False,
                           server_default=FetchedValue(),
                           server_onupdate=FetchedValue())

    versions = relationship("Version", backref="project")

    def __init__(self, name):
        self.name = name

    def __repr__(self):
        return "<Project: {name}>".format(name=self.name)

class Version(UUIDPrimaryKeyMixin, TimeStampedMixin, db.Model):

    __tablename__ = "versions"
    __table_args__ = declared_attr(table_args((
        db.Index("idx_project_version", "project_id", "version", unique=True),
    )))

    project_id = db.Column(pg.UUID(as_uuid=True),
                           db.ForeignKey("projects.id", ondelete="RESTRICT"),
                           nullable=False)
    version = db.Column(db.UnicodeText, nullable=False)

    summary = db.Column(db.UnicodeText, nullable=False, server_default="")
    description = db.Column(db.UnicodeText, nullable=False, server_default="")

    author = db.Column(db.UnicodeText, nullable=False, server_default="")
    author_email = db.Column(db.UnicodeText, nullable=False, server_default="")

    maintainer = db.Column(db.UnicodeText, nullable=False, server_default="")
    maintainer_email = db.Column(db.UnicodeText, nullable=False, server_default="")

    license = db.Column(db.UnicodeText, nullable=False, server_default="")

    # URIs
    uris = db.Column(pg.HSTORE, nullable=False, server_default=text("''::hstore"))

    # Requirements
    requires_python = db.Column(db.UnicodeText, nullable=False, server_default="")
    requires_external = db.Column(pg.ARRAY(db.UnicodeText, dimensions=1), nullable=False, server_default=text("'{}'"))

    def __repr__(self):
        ctx = {"name": self.project.name, "version": self.version}
        return "<Version: {name} {version}>".format(**ctx)
<commit_msg>Apply PEP8 style guide<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

from sqlalchemy.schema import FetchedValue
from sqlalchemy.dialects import postgresql as pg
from sqlalchemy.ext.declarative import declared_attr
from sqlalchemy.orm import relationship
from sqlalchemy.sql.expression import text

from warehouse import db
from warehouse.database.mixins import UUIDPrimaryKeyMixin, TimeStampedMixin
from warehouse.database.schema import TableDDL
from warehouse.database.utils import table_args

class Project(UUIDPrimaryKeyMixin, TimeStampedMixin, db.Model):

    __tablename__ = "projects"
    __table_args__ = declared_attr(table_args((
        TableDDL("""
            CREATE OR REPLACE FUNCTION normalize_name()
            RETURNS trigger AS $$
            BEGIN
                NEW.normalized = lower(regexp_replace(new.name, '[^A-Za-z0-9.]+', '-'));
                return NEW;
            END;
            $$ LANGUAGE plpgsql;

            CREATE TRIGGER %(table)s_normalize_name
            BEFORE INSERT OR UPDATE
            ON %(table)s
            FOR EACH ROW
            EXECUTE PROCEDURE normalize_name();
        """),
    )))

    name = db.Column(db.UnicodeText, unique=True, nullable=False)
    normalized = db.Column(db.UnicodeText, unique=True, nullable=False,
                           server_default=FetchedValue(),
                           server_onupdate=FetchedValue())

    versions = relationship("Version", backref="project")

    def __init__(self, name):
        self.name = name

    def __repr__(self):
        return "<Project: {name}>".format(name=self.name)

class Version(UUIDPrimaryKeyMixin, TimeStampedMixin, db.Model):

    __tablename__ = "versions"
    __table_args__ = declared_attr(table_args((
        db.Index("idx_project_version", "project_id", "version", unique=True),
    )))

    project_id = db.Column(pg.UUID(as_uuid=True),
                           db.ForeignKey("projects.id", ondelete="RESTRICT"),
                           nullable=False)
    version = db.Column(db.UnicodeText, nullable=False)

    summary = db.Column(db.UnicodeText, nullable=False, server_default="")
    description = db.Column(db.UnicodeText, nullable=False, server_default="")

    author = db.Column(db.UnicodeText, nullable=False, server_default="")
    author_email = db.Column(db.UnicodeText, nullable=False, server_default="")

    maintainer = db.Column(db.UnicodeText, nullable=False, server_default="")
    maintainer_email = db.Column(db.UnicodeText, nullable=False,
                                 server_default="")

    license = db.Column(db.UnicodeText, nullable=False, server_default="")

    # URIs
    uris = db.Column(pg.HSTORE, nullable=False,
                     server_default=text("''::hstore"))

    # Requirements
    requires_python = db.Column(db.UnicodeText, nullable=False,
                                server_default="")
    requires_external = db.Column(pg.ARRAY(db.UnicodeText, dimensions=1),
                                  nullable=False, server_default=text("'{}'"))

    def __repr__(self):
        ctx = {"name": self.project.name, "version": self.version}
        return "<Version: {name} {version}>".format(**ctx)
<|endoftext|>


<commit_before>from django.conf import settings
from django.contrib.sessions.backends.base import SessionBase, CreateError
from django.core.exceptions import SuspiciousOperation
try:
    from django.utils.encoding import force_unicode
except ImportError:
    from django.utils.encoding import force_text as force_unicode

from mongoengine.document import Document
from mongoengine import fields
from mongoengine.queryset import OperationError
from mongoengine.connection import DEFAULT_CONNECTION_NAME

from .utils import datetime_now

MONGOENGINE_SESSION_DB_ALIAS = getattr(
    settings, 'MONGOENGINE_SESSION_DB_ALIAS',
    DEFAULT_CONNECTION_NAME)

# a setting for the name of the collection used to store sessions
MONGOENGINE_SESSION_COLLECTION = getattr(
    settings, 'MONGOENGINE_SESSION_COLLECTION',
    'django_session')

# a setting for whether session data is stored encoded or not
MONGOENGINE_SESSION_DATA_ENCODE = getattr(
    settings, 'MONGOENGINE_SESSION_DATA_ENCODE',
    True)

class MongoSession(Document):
    session_key = fields.StringField(primary_key=True, max_length=40)
    session_data = fields.StringField() if MONGOENGINE_SESSION_DATA_ENCODE \
                                        else fields.DictField()
    expire_date = fields.DateTimeField()

    meta = {
        'collection': MONGOENGINE_SESSION_COLLECTION,
        'db_alias': MONGOENGINE_SESSION_DB_ALIAS,
        'allow_inheritance': False,
        'indexes': [
            {
                'fields': ['expire_date'],
                'expireAfterSeconds': 0
            }
        ]
    }

    def get_decoded(self):
        return SessionStore().decode(self.session_data)

class SessionStore(SessionBase):
    """A MongoEngine-based session store for Django.
    """

    def load(self):
        try:
            s = MongoSession.objects(session_key=self.session_key,
                                     expire_date__gt=datetime_now)[0]
            if MONGOENGINE_SESSION_DATA_ENCODE:
                return self.decode(force_unicode(s.session_data))
            else:
                return s.session_data
        except (IndexError, SuspiciousOperation):
            self.create()
            return {}

    def exists(self, session_key):
        return bool(MongoSession.objects(session_key=session_key).first())

    def create(self):
        while True:
            self._session_key = self._get_new_session_key()
            try:
                self.save(must_create=True)
            except CreateError:
                continue
            self.modified = True
            self._session_cache = {}
            return

    def save(self, must_create=False):
        if self.session_key is None:
            self._session_key = self._get_new_session_key()
        s = MongoSession(session_key=self.session_key)
        if MONGOENGINE_SESSION_DATA_ENCODE:
            s.session_data = self.encode(self._get_session(no_load=must_create))
        else:
            s.session_data = self._get_session(no_load=must_create)
        s.expire_date = self.get_expiry_date()
        try:
            s.save(force_insert=must_create)
        except OperationError:
            if must_create:
                raise CreateError
            raise

    def delete(self, session_key=None):
        if session_key is None:
            if self.session_key is None:
                return
            session_key = self.session_key
        MongoSession.objects(session_key=session_key).delete()
<commit_msg>fix(session): session serializer to json<commit_after>from django.conf import settings
from django.contrib.sessions.backends.base import SessionBase, CreateError
from django.core.exceptions import SuspiciousOperation
try:
    from django.utils.encoding import force_unicode
except ImportError:
    from django.utils.encoding import force_text as force_unicode

from mongoengine.document import Document
from mongoengine import fields
from mongoengine.queryset import OperationError
from mongoengine.connection import DEFAULT_CONNECTION_NAME

from .utils import datetime_now

MONGOENGINE_SESSION_DB_ALIAS = getattr(
    settings, 'MONGOENGINE_SESSION_DB_ALIAS',
    DEFAULT_CONNECTION_NAME)

# a setting for the name of the collection used to store sessions
MONGOENGINE_SESSION_COLLECTION = getattr(
    settings, 'MONGOENGINE_SESSION_COLLECTION',
    'django_session')

# a setting for whether session data is stored encoded or not
MONGOENGINE_SESSION_DATA_ENCODE = getattr(
    settings, 'MONGOENGINE_SESSION_DATA_ENCODE',
    True)

class MongoSession(Document):
    session_key = fields.StringField(primary_key=True, max_length=40)
    session_data = fields.StringField() if MONGOENGINE_SESSION_DATA_ENCODE \
                                        else fields.DictField()
    expire_date = fields.DateTimeField()

    meta = {
        'collection': MONGOENGINE_SESSION_COLLECTION,
        'db_alias': MONGOENGINE_SESSION_DB_ALIAS,
        'allow_inheritance': False,
        'indexes': [
            {
                'fields': ['expire_date'],
                'expireAfterSeconds': 0
            }
        ]
    }

    def get_decoded(self):
        return SessionStore().decode(self.session_data)

class SessionStore(SessionBase):
    """A MongoEngine-based session store for Django.
    """

    def _get_session(self, *args, **kwargs):
        sess = super(SessionStore, self)._get_session(*args, **kwargs)
        if sess.get('_auth_user_id', None):
            sess['_auth_user_id'] = str(sess.get('_auth_user_id'))
        return sess

    def load(self):
        try:
            s = MongoSession.objects(session_key=self.session_key,
                                     expire_date__gt=datetime_now)[0]
            if MONGOENGINE_SESSION_DATA_ENCODE:
                return self.decode(force_unicode(s.session_data))
            else:
                return s.session_data
        except (IndexError, SuspiciousOperation):
            self.create()
            return {}

    def exists(self, session_key):
        return bool(MongoSession.objects(session_key=session_key).first())

    def create(self):
        while True:
            self._session_key = self._get_new_session_key()
            try:
                self.save(must_create=True)
            except CreateError:
                continue
            self.modified = True
            self._session_cache = {}
            return

    def save(self, must_create=False):
        if self.session_key is None:
            self._session_key = self._get_new_session_key()
        s = MongoSession(session_key=self.session_key)
        if MONGOENGINE_SESSION_DATA_ENCODE:
            s.session_data = self.encode(self._get_session(no_load=must_create))
        else:
            s.session_data = self._get_session(no_load=must_create)
        s.expire_date = self.get_expiry_date()
        try:
            s.save(force_insert=must_create)
        except OperationError:
            if must_create:
                raise CreateError
            raise

    def delete(self, session_key=None):
        if session_key is None:
            if self.session_key is None:
                return
            session_key = self.session_key
        MongoSession.objects(session_key=session_key).delete()
<|endoftext|>


<commit_before>import json
import logging
import os
import socket
import struct
import threading
import time
import traceback
import zlib

from dmoj import sysinfo
from dmoj.judgeenv import get_supported_problems, get_runtime_versions

try:
    import ssl
except ImportError:
    ssl = None

log = logging.getLogger(__name__)
timer = time.clock if os.name == 'nt' else time.time

class JudgeAuthenticationFailed(Exception):
    pass

class PacketManager(object):
    SIZE_PACK = struct.Struct('!I')

    def __init__(self, host, port, judge, name, key, secure=False, no_cert_check=False, cert_store=None):
        self.host = host
        self.port = port
        self.judge = judge
        self.name = name
        self.key = key

        log.info('Preparing to connect to [%s]:%s as: %s', host, port, name)
        if secure and ssl:
            self.ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
            self.ssl_context.options |= ssl.OP_NO_SSLv2
            self.ssl_context.options |= ssl.OP_NO_SSLv3

            if not no_cert_check:
                self.ssl_context.verify_mode = ssl.CERT_REQUIRED
                self.ssl_context.check_hostname = True

            if cert_store is None:
                self.ssl_context.load_default_certs()
            else:
                self.ssl_context.load_verify_locations(cafile=cert_store)
            log.info('Configured to use TLS.')
        else:
            self.ssl_context = None
            log.info('TLS not enabled.')

        self.secure = secure
        self.no_cert_check = no_cert_check
        self.cert_store = cert_store

        self._lock = threading.RLock()
        self._batch = 0
        # Exponential backoff: starting at 4 seconds.
        # Certainly hope it won't stack overflow, since it will take days if not years.
        self.fallback = 4

        self.conn = None
        self._do_reconnect()

    def _connect(self):
        problems = get_supported_problems()
        versions = get_runtime_versions()

        log.info('Opening connection to: [%s]:%s', self.host, self.port)
        self.conn = socket.create_connection((self.host, self.port))
        self.conn.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)

        if self.ssl_context:
            log.info('Starting TLS on: [%s]:%s', self.host, self.port)
            self.conn = self.ssl_context.wrap_socket(self.conn, server_hostname=self.host)

        log.info('Starting handshake with: [%s]:%s', self.host, self.port)
        self.input = self.conn.makefile('r')
        self.output = self.conn.makefile('w', 0)
        self.handshake(problems, versions, self.name, self.key)

    def _reconnect(self):
        if self.fallback > 86400:
            # Return 0 to avoid supervisor restart.
            raise SystemExit(0)

        log.warning('Attempting reconnection in %.0fs: [%s]:%s', self.fallback, self.host, self.port)
        if self.conn is not None:
            log.info('Dropping old connection.')
            self.conn.close()
        time.sleep(self.fallback)
        self.fallback *= 1.5
        self._do_reconnect()

    def _do_reconnect(self):
        try:
            self._connect()
        except JudgeAuthenticationFailed:
            log.error('Authentication as "%s" failed on: [%s]:%s', self.name, self.host, self.port)
            self._reconnect()
        except socket.error:
            log.exception('Connection failed due to socket error: [%s]:%s', self.host, self.port)
            self._reconnect()

    def __del__(self):
        self.conn.shutdown(socket.SHUT_RDWR)

    def _read_async(self):
        try:
            while True:
                self._receive_packet(self._read_single())
        except KeyboardInterrupt:
            pass
        except Exception:  # connection reset by peer
            traceback.print_exc()
            raise SystemExit(1)

    def _read_single(self):
        try:
            data = self.input.read(PacketManager.SIZE_PACK.size)
        except socket.error:
            self._reconnect()
            return self._read_single()
        if not data:
            self._reconnect()
            return self._read_single()
        size = PacketManager.SIZE_PACK.unpack(data)[0]
        try:
            packet = self.input.read(size).decode('zlib')
        except zlib.error:
            self._reconnect()
            return self._read_single()
        else:
            return json.loads(packet)

    def run(self):
        self._read_async()

    def run_async(self):
        threading.Thread(target=self._read_async).start()

    def _send_packet(self, packet, rewrite=True):
        if rewrite and 'submission-id' in packet and self.judge.get_process_type() != 'submission':
            packet['%s-id' % self.judge.get_process_type()] = packet['submission-id']
            del packet['submission-id']

        for k, v in packet.items():
            if isinstance(v, str):
                # Make sure we don't have any garbage utf-8 from e.g. weird compilers
                # *cough* fpc *cough* that could cause this routine to crash
                packet[k] = v.decode('utf-8', 'replace')

        raw = json.dumps(packet).encode('zlib')
        with self._lock:
            self.output.writelines((PacketManager.SIZE_PACK.pack(len(raw)), raw))

    def _receive_packet(self, packet):
        name = packet['name']
        if name == 'ping':
            self.ping_packet(packet['when'])
        elif name == 'get-current-submission':
            self.current_submission_packet()
        elif name == 'submission-request':
            self.submission_acknowledged_packet(packet['submission-id'])
            self.judge.begin_grading(
                packet['submission-id'],
                packet['problem-id'],
                packet['language'],
                packet['source'],
                float(packet['time-limit']),
                int(packet['memory-limit']),
                packet['short-circuit'],
                packet['pretests-only']
            )
            self._batch = 0
            log.info('Accept submission: %d: executor: %s, code: %s',
                     packet['submission-id'], packet['language'], packet['problem-id'])
        elif name == 'invocation-request':
            self.invocation_acknowledged_packet(packet['invocation-id'])
            self.judge.custom_invocation(
                packet['invocation-id'],
                packet['language'],
                packet['source'],
                float(packet['time-limit']),
                int(packet['memory-limit']),
                packet['input-data']
            )
            log.info('Accept invocation: %d: executor: %s', packet['invocation-id'], packet['language'])
        elif name == 'terminate-submission':
            self.judge.terminate_grading()
        else:
            print 'ERROR: unknown packet %s, payload %s' % (name, packet)

    def handshake(self, problems, runtimes, id, key):
        self._send_packet({'name': 'handshake',
                           'problems': problems,
                           'executors': runtimes,
                           'id': id,
                           'key': key})
        log.info('Awaiting handshake response: [%s]:%s', self.host, self.port)
        try:
            data = self.input.read(PacketManager.SIZE_PACK.size)
            size = PacketManager.SIZE_PACK.unpack(data)[0]
            packet = self.input.read(size).decode('zlib')
            resp = json.loads(packet)
        except Exception:
            log.exception('Cannot understand handshake response: [%s]:%s', self.host, self.port)
            raise JudgeAuthenticationFailed()
        else:
            if resp['name'] != 'handshake-success':
                log.error('Handshake failed.')
                raise JudgeAuthenticationFailed()

    def invocation_begin_packet(self):
        log.info('Begin invoking: %d', self.judge.current_submission)
        self._send_packet({'name': 'invocation-begin',
                           'invocation-id': self.judge.current_submission})

    def invocation_end_packet(self, result):
        log.info('End invoking: %d', self.judge.current_submission)
        self.fallback = 4
        self._send_packet({'name': 'invocation-end',
                           'output': result.proc_output,
                           'status': result.status_flag,
                           'time': result.execution_time,
                           'memory': result.max_memory,
                           'feedback': result.feedback,
                           'invocation-id': self.judge.current_submission})

    def supported_problems_packet(self, problems):
        log.info('Update problems')
        self._send_packet({'name': 'supported-problems',
                           'problems': problems})

    def test_case_status_packet(self, position, result):
        self._send_packet({'name': 'test-case-status',
                           'submission-id': self.judge.current_submission,
                           'position': position,
                           'status': result.result_flag,
                           'time': result.execution_time,
                           'points': result.points,
                           'total-points': result.total_points,
                           'memory': result.max_memory,
                           'output': result.output,
                           'feedback': result.feedback})

    def compile_error_packet(self, log):
        log.info('Compile error: %d', self.judge.current_submission)
        self.fallback = 4
        self._send_packet({'name': 'compile-error',
                           'submission-id': self.judge.current_submission,
                           'log': log})

    def compile_message_packet(self, log):
        log.info('Compile message: %d', self.judge.current_submission)
        self._send_packet({'name': 'compile-message',
                           'submission-id': self.judge.current_submission,
                           'log': log})

    def internal_error_packet(self, message):
        log.info('Internal error: %d', self.judge.current_submission)
        self._send_packet({'name': 'internal-error',
                           'submission-id': self.judge.current_submission,
                           'message': message})

    def begin_grading_packet(self, is_pretested):
        log.info('Begin grading: %d', self.judge.current_submission)
        self._send_packet({'name': 'grading-begin',
                           'submission-id': self.judge.current_submission,
                           'pretested': is_pretested})

    def grading_end_packet(self):
        log.info('End grading: %d', self.judge.current_submission)
        self.fallback = 4
        self._send_packet({'name': 'grading-end',
                           'submission-id': self.judge.current_submission})

    def batch_begin_packet(self):
        self._batch += 1
        log.info('Enter batch number %d: %d', self._batch, self.judge.current_submission)
        self._send_packet({'name': 'batch-begin',
                           'submission-id': self.judge.current_submission})

    def batch_end_packet(self):
        log.info('Exit batch number %d: %d', self._batch, self.judge.current_submission)
        self._send_packet({'name': 'batch-end',
                           'submission-id': self.judge.current_submission})

    def current_submission_packet(self):
        log.info('Current submission query: %d', self.judge.current_submission)
        self._send_packet({'name': 'current-submission-id',
                           'submission-id': self.judge.current_submission})

    def submission_terminated_packet(self):
        log.info('Submission aborted: %d', self.judge.current_submission)
        self._send_packet({'name': 'submission-terminated',
                           'submission-id': self.judge.current_submission})

    def ping_packet(self, when):
        data = {'name': 'ping-response',
                'when': when,
                'time': time.time()}
        for fn in sysinfo.report_callbacks:
            key, value = fn()
            data[key] = value
        self._send_packet(data)

    def submission_acknowledged_packet(self, sub_id):
        self._send_packet({'name': 'submission-acknowledged',
                           'submission-id': sub_id}, rewrite=False)

    def invocation_acknowledged_packet(self, sub_id):
        self._send_packet({'name': 'submission-acknowledged',
                           'invocation-id': sub_id}, rewrite=False)
<commit_msg>Fix name collision.<commit_after>import json
import logging
import os
import socket
import struct
import threading
import time
import traceback
import zlib

from dmoj import sysinfo
from dmoj.judgeenv import get_supported_problems, get_runtime_versions

try:
    import ssl
except ImportError:
    ssl = None

log = logging.getLogger(__name__)
timer = time.clock if os.name == 'nt' else time.time

class JudgeAuthenticationFailed(Exception):
    pass

class PacketManager(object):
    SIZE_PACK = struct.Struct('!I')

    def __init__(self, host, port, judge, name, key, secure=False, no_cert_check=False, cert_store=None):
        self.host = host
        self.port = port
        self.judge = judge
        self.name = name
        self.key = key

        log.info('Preparing to connect to [%s]:%s as: %s', host, port, name)
        if secure and ssl:
            self.ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
            self.ssl_context.options |= ssl.OP_NO_SSLv2
            self.ssl_context.options |= ssl.OP_NO_SSLv3

            if not no_cert_check:
                self.ssl_context.verify_mode = ssl.CERT_REQUIRED
                self.ssl_context.check_hostname = True

            if cert_store is None:
                self.ssl_context.load_default_certs()
            else:
                self.ssl_context.load_verify_locations(cafile=cert_store)
            log.info('Configured to use TLS.')
        else:
            self.ssl_context = None
            log.info('TLS not enabled.')

        self.secure = secure
        self.no_cert_check = no_cert_check
        self.cert_store = cert_store

        self._lock = threading.RLock()
        self._batch = 0
        # Exponential backoff: starting at 4 seconds.
        # Certainly hope it won't stack overflow, since it will take days if not years.
        self.fallback = 4

        self.conn = None
        self._do_reconnect()

    def _connect(self):
        problems = get_supported_problems()
        versions = get_runtime_versions()

        log.info('Opening connection to: [%s]:%s', self.host, self.port)
        self.conn = socket.create_connection((self.host, self.port))
        self.conn.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)

        if self.ssl_context:
            log.info('Starting TLS on: [%s]:%s', self.host, self.port)
            self.conn = self.ssl_context.wrap_socket(self.conn, server_hostname=self.host)

        log.info('Starting handshake with: [%s]:%s', self.host, self.port)
        self.input = self.conn.makefile('r')
        self.output = self.conn.makefile('w', 0)
        self.handshake(problems, versions, self.name, self.key)
        log.info('Judge "%s" online: [%s]:%s', self.name, self.host, self.port)

    def _reconnect(self):
        if self.fallback > 86400:
            # Return 0 to avoid supervisor restart.
            raise SystemExit(0)

        log.warning('Attempting reconnection in %.0fs: [%s]:%s', self.fallback, self.host, self.port)
        if self.conn is not None:
            log.info('Dropping old connection.')
            self.conn.close()
        time.sleep(self.fallback)
        self.fallback *= 1.5
        self._do_reconnect()

    def _do_reconnect(self):
        try:
            self._connect()
        except JudgeAuthenticationFailed:
            log.error('Authentication as "%s" failed on: [%s]:%s', self.name, self.host, self.port)
            self._reconnect()
        except socket.error:
            log.exception('Connection failed due to socket error: [%s]:%s', self.host, self.port)
            self._reconnect()

    def __del__(self):
        self.conn.shutdown(socket.SHUT_RDWR)

    def _read_async(self):
        try:
            while True:
                self._receive_packet(self._read_single())
        except KeyboardInterrupt:
            pass
        except Exception:  # connection reset by peer
            traceback.print_exc()
            raise SystemExit(1)

    def _read_single(self):
        try:
            data = self.input.read(PacketManager.SIZE_PACK.size)
        except socket.error:
            self._reconnect()
            return self._read_single()
        if not data:
            self._reconnect()
            return self._read_single()
        size = PacketManager.SIZE_PACK.unpack(data)[0]
        try:
            packet = self.input.read(size).decode('zlib')
        except zlib.error:
            self._reconnect()
            return self._read_single()
        else:
            return json.loads(packet)

    def run(self):
        self._read_async()

    def run_async(self):
        threading.Thread(target=self._read_async).start()

    def _send_packet(self, packet, rewrite=True):
        if rewrite and 'submission-id' in packet and self.judge.get_process_type() != 'submission':
            packet['%s-id' % self.judge.get_process_type()] = packet['submission-id']
            del packet['submission-id']

        for k, v in packet.items():
            if isinstance(v, str):
                # Make sure we don't have any garbage utf-8 from e.g. weird compilers
                # *cough* fpc *cough* that could cause this routine to crash
                packet[k] = v.decode('utf-8', 'replace')

        raw = json.dumps(packet).encode('zlib')
        with self._lock:
            self.output.writelines((PacketManager.SIZE_PACK.pack(len(raw)), raw))

    def _receive_packet(self, packet):
        name = packet['name']
        if name == 'ping':
            self.ping_packet(packet['when'])
        elif name == 'get-current-submission':
            self.current_submission_packet()
        elif name == 'submission-request':
            self.submission_acknowledged_packet(packet['submission-id'])
            self.judge.begin_grading(
                packet['submission-id'],
                packet['problem-id'],
                packet['language'],
                packet['source'],
                float(packet['time-limit']),
                int(packet['memory-limit']),
                packet['short-circuit'],
                packet['pretests-only']
            )
            self._batch = 0
            log.info('Accept submission: %d: executor: %s, code: %s',
                     packet['submission-id'], packet['language'], packet['problem-id'])
        elif name == 'invocation-request':
            self.invocation_acknowledged_packet(packet['invocation-id'])
            self.judge.custom_invocation(
                packet['invocation-id'],
                packet['language'],
                packet['source'],
                float(packet['time-limit']),
                int(packet['memory-limit']),
                packet['input-data']
            )
            log.info('Accept invocation: %d: executor: %s', packet['invocation-id'], packet['language'])
        elif name == 'terminate-submission':
            self.judge.terminate_grading()
        else:
            print 'ERROR: unknown packet %s, payload %s' % (name, packet)

    def handshake(self, problems, runtimes, id, key):
        self._send_packet({'name': 'handshake',
                           'problems': problems,
                           'executors': runtimes,
                           'id': id,
                           'key': key})
        log.info('Awaiting handshake response: [%s]:%s', self.host, self.port)
        try:
            data = self.input.read(PacketManager.SIZE_PACK.size)
            size = PacketManager.SIZE_PACK.unpack(data)[0]
            packet = self.input.read(size).decode('zlib')
            resp = json.loads(packet)
        except Exception:
            log.exception('Cannot understand handshake response: [%s]:%s', self.host, self.port)
            raise JudgeAuthenticationFailed()
        else:
            if resp['name'] != 'handshake-success':
                log.error('Handshake failed.')
                raise JudgeAuthenticationFailed()

    def invocation_begin_packet(self):
        log.info('Begin invoking: %d', self.judge.current_submission)
        self._send_packet({'name': 'invocation-begin',
                           'invocation-id': self.judge.current_submission})

    def invocation_end_packet(self, result):
        log.info('End invoking: %d', self.judge.current_submission)
        self.fallback = 4
        self._send_packet({'name': 'invocation-end',
                           'output': result.proc_output,
                           'status': result.status_flag,
                           'time': result.execution_time,
                           'memory': result.max_memory,
                           'feedback': result.feedback,
                           'invocation-id': self.judge.current_submission})

    def supported_problems_packet(self, problems):
        log.info('Update problems')
        self._send_packet({'name': 'supported-problems',
                           'problems': problems})

    def test_case_status_packet(self, position, result):
        self._send_packet({'name': 'test-case-status',
                           'submission-id': self.judge.current_submission,
                           'position': position,
                           'status': result.result_flag,
                           'time': result.execution_time,
                           'points': result.points,
                           'total-points': result.total_points,
                           'memory': result.max_memory,
                           'output': result.output,
                           'feedback': result.feedback})

    def compile_error_packet(self, message):
        log.info('Compile error: %d', self.judge.current_submission)
        self.fallback = 4
        self._send_packet({'name': 'compile-error',
                           'submission-id': self.judge.current_submission,
                           'log': message})

    def compile_message_packet(self, message):
        log.info('Compile message: %d', self.judge.current_submission)
        self._send_packet({'name': 'compile-message',
                           'submission-id': self.judge.current_submission,
                           'log': message})

    def internal_error_packet(self, message):
        log.info('Internal error: %d', self.judge.current_submission)
        self._send_packet({'name': 'internal-error',
                           'submission-id': self.judge.current_submission,
                           'message': message})

    def begin_grading_packet(self, is_pretested):
        log.info('Begin grading: %d', self.judge.current_submission)
        self._send_packet({'name': 'grading-begin',
                           'submission-id': self.judge.current_submission,
                           'pretested': is_pretested})

    def grading_end_packet(self):
        log.info('End grading: %d', self.judge.current_submission)
        self.fallback = 4
        self._send_packet({'name': 'grading-end',
                           'submission-id': self.judge.current_submission})

    def batch_begin_packet(self):
        self._batch += 1
        log.info('Enter batch number %d: %d', self._batch, self.judge.current_submission)
        self._send_packet({'name': 'batch-begin',
                           'submission-id': self.judge.current_submission})

    def batch_end_packet(self):
        log.info('Exit batch number %d: %d', self._batch, self.judge.current_submission)
        self._send_packet({'name': 'batch-end',
                           'submission-id': self.judge.current_submission})

    def current_submission_packet(self):
        log.info('Current submission query: %d', self.judge.current_submission)
        self._send_packet({'name': 'current-submission-id',
                           'submission-id': self.judge.current_submission})

    def submission_terminated_packet(self):
        log.info('Submission aborted: %d', self.judge.current_submission)
        self._send_packet({'name': 'submission-terminated',
                           'submission-id': self.judge.current_submission})

    def ping_packet(self, when):
        data = {'name': 'ping-response',
                'when': when,
                'time': time.time()}
        for fn in sysinfo.report_callbacks:
            key, value = fn()
            data[key] = value
        self._send_packet(data)

    def submission_acknowledged_packet(self, sub_id):
        self._send_packet({'name': 'submission-acknowledged',
                           'submission-id': sub_id}, rewrite=False)

    def invocation_acknowledged_packet(self, sub_id):
        self._send_packet({'name': 'submission-acknowledged',
                           'invocation-id': sub_id}, rewrite=False)
<|endoftext|>


<commit_before>#! /usr/bin/env python
'''
A script to create a simple, browser-based countdown timer.

usage: bokeh serve --show timer.py
'''
# Imports
from bokeh.layouts import column, row
from bokeh import plotting as bkplt
from bokeh.models import ColumnDataSource, Button, Slider
from bokeh.driving import count

# Functions
@count()
def countdown(step):
    '''Count down the time remaining'''
    global seconds_left
    global text_color
    if seconds_left > 0:
        seconds_left -= 1
        update_timer(seconds_left, start_time)
    # make the timer text flash if time has run out
    if seconds_left == 0:
        if 'ff0000' in text_color:
            text_color = ['%ffffff']
            source.data['text_color'] = text_color
            source.trigger('data', source.data, source.data)
        else:
            text_color = ['ff0000']
            source.data['text_color'] = text_color
            source.trigger('data', source.data, source.data)

def update_timer(seconds_remaining, start_time):
    '''Update the appearance of the timer'''
    global seconds_left
    seconds_left = seconds_remaining
    minutes_remaining = int(seconds_remaining/60)
    remainder = seconds_remaining%60
    time_string = ['%02d:%02d' % (minutes_remaining, remainder)]
    source.data['time_remaining'] = [time_string]
# change background colour if remaining time <10% of initial value
    if seconds_remaining <= start_time/10:
        fill_color = 'ff0000'
    else:
        fill_color = '78c400'
    source.data['text_color'] = ['%ffffff']
    source.data['fill_color'] = [fill_color]
    source.trigger('data', source.data, source.data)
    return True

def run_timer():
    '''(Re)Set the timer running;
    disable start button and sliders;
    enable stop button'''
    start_button.disabled = True
    stop_button.disabled = False
    minutes_slider.disabled = True
    minutes_slider.disabled = True
    # set timer running, by calling countdown() every 1000 millisecs
    bkplt.curdoc().add_periodic_callback(countdown, 1000)
    return True

def reset_timer():
    '''Reset the timer to the specified start time'''
    global seconds_left
    # check if the timer is running, and stop it if it is
    if start_button.disabled == True:
        start_button.disabled = False
        stop_button.disabled = True
        minutes_slider.disabled = False
        minutes_slider.disabled = False
        # stop calling countdown()
        bkplt.curdoc().remove_periodic_callback(countdown)
    update_timer(start_time, start_time)
    seconds_left = start_time

def stop_timer():
    '''Stop the timer'''
    stop_button.disabled = True
    start_button.disabled = False
    minutes_slider.disabled = False
    minutes_slider.disabled = False
    # stop calling countdown()
    bkplt.curdoc().remove_periodic_callback(countdown)

def set_start_time(attrname, old, new):
    '''Set the start time of the timer'''
    global start_time
    start_mins = minutes_slider.value
    start_secs = seconds_slider.value
    start_time = start_mins*60 + start_secs
    # only update timer text if the timer isn't running
    if not minutes_slider.disabled:
        update_timer(start_time, start_time)

# Set default/starting values
default_minutes = 5
default_seconds = 0
seconds_left = default_minutes*60 + default_seconds
# ColumnDataSource data values must be iterable,
# so the values below are placed inside single-element lists
color = ['78c400']
text_color = ['%ffffff']
start_time = [default_minutes*60 + default_seconds]
time_remaining = ['%02d:%02d' % (default_minutes, default_seconds)]

# Create data source for timer plot
source=ColumnDataSource(data=dict(start_time=start_time,
                                  time_remaining=time_remaining,
                                  fill_color=color,
                                  text_color=text_color))

# No tools required for this one
tools = ''

# Create plot: a color block, with text centered inside
p1 = bkplt.figure(x_range=(-8, 8), y_range=(-5, 5),
                  plot_width=900, plot_height=600,
                  title=None, tools=tools)
p1.rect(x=[0],
        y=[0],
        width=16,
        height=10,
        fill_color='fill_color',
        line_color=None,
        name='block',
        source=source)
p1.text(x=[0],
        y=[0],
        text='time_remaining',
        text_color='text_color',
        alpha=0.75,
        text_font_size='128pt',
        text_baseline='middle',
        text_align='center',
        name='timer',
        source=source)

# Remove axes, labels & tick lines
p1.ygrid.grid_line_color = None
p1.xgrid.grid_line_color = None
p1.axis.axis_line_color  = None
p1.axis.major_label_text_color = None
p1.axis.major_tick_line_color = None
p1.axis.minor_tick_line_color = None

# Sliders
minutes_slider = Slider(start=0, end=99,
                        value=default_minutes, step=1,
                        title="Minutes")
minutes_slider.on_change("value", set_start_time)
seconds_slider = Slider(start=0, end=59,
                        value=default_seconds, step=1,
                        title="Seconds")
seconds_slider.on_change("value", set_start_time)

# Buttons
start_button = Button(label="Start")
start_button.on_click(run_timer)
stop_button = Button(label="Stop")
stop_button.on_click(stop_timer)
reset_button = Button(label="Reset")
reset_button.on_click(reset_timer)

# Layout plot & widgets
bkplt.curdoc().add_root(column(row(minutes_slider, seconds_slider),
                               row(start_button, stop_button, reset_button),
                               p1))
<commit_msg>buggy version of JS callbacks approach.<commit_after>#! /usr/bin/env python
'''
A script to create a simple, browser-based countdown timer.

usage: bokeh serve --show timer.py
'''
# Imports
from bokeh.layouts import column, row
from bokeh import plotting as bkplt
from bokeh.models import ColumnDataSource, Button, Slider, CustomJS

# Set default/starting values
default_minutes = 0
default_seconds = 20
seconds_left = default_minutes*60 + default_seconds
# ColumnDataSource data values must be iterable,
# so the values below are placed inside single-element lists
color = ['78c400']
text_color = ['#ffffff']
start_time = [default_minutes*60 + default_seconds]
time_remaining = start_time.copy()
time_string = ['%02d:%02d' % (default_minutes, default_seconds)]
# Create data source for timer plot
source=ColumnDataSource(data=dict(start_time=start_time,
                                  start_mins=[default_minutes],
                                  start_secs=[default_seconds],
                                  time_remaining=time_remaining,
                                  time_string=time_string,
                                  fill_color=color,
                                  text_color=text_color,
                                  interval_id=[0]))

# No tools required for this one
tools = ''

# JS Callbacks
run_timer_JS = CustomJS(args=dict(source=source), code="""
    function disable_button(button) {
        button_id = button.get('id');
        button_element = document.querySelector('#modelid_' + button_id + '>button');
        button_element.setAttribute('disabled')
    }
    function enable_button(button) {
        button_id = button.get('id');
        button_element = document.querySelector('#modelid_' + button_id + '>button');
        button_element.removeAttribute('disabled')
    }
    var data = source.get('data');
    var start_button_disabled = start_button.get('disabled');
    interval_id = data['interval_id'];
    function countdown() {
        start_time = data['start_time'];
        time_remaining = data['time_remaining'];
        time_string = data['time_string'];
        text_color = data['text_color'];
        fill_color = data['fill_color'];
        if (time_remaining[0] == 0) {
            if (text_color[0] == '#ffffff') {
                text_color[0] = '#ff0000';
            }else {
                text_color[0] = '#ffffff';
            }
        }else{
            time_remaining[0]--;
            time_string[0] = ('0' + Math.floor(time_remaining[0] / 60)).slice(-2) + ':' + ('0' + Math.floor(time_remaining[0] % 60)).slice(-2);
            if (time_remaining[0] <= start_time[0]/10) {
                fill_color[0] = '#ff0000';
            }
        }
        source.trigger('change');
    }
    disable_button(start_button);
    enable_button(stop_button);
    enable_button(reset_button);
    interval_id[0] = setInterval(countdown, 1000);
    source.trigger('change');
""")

stop_timer_JS = CustomJS(args=dict(source=source), code="""
    function disable_button(button) {
        button_id = button.get('id');
        button_element = document.querySelector('#modelid_' + button_id + '>button');
        button_element.setAttribute('disabled')
    }
    function enable_button(button) {
        button_id = button.get('id');
        button_element = document.querySelector('#modelid_' + button_id + '>button');
        button_element.removeAttribute('disabled')
    }
    var data = source.get('data');
    interval_id = data['interval_id'];
    clearInterval(interval_id[0]);
    enable_button(start_button);
    disable_button(stop_button);
    enable_button(reset_button);
""")

reset_timer_JS = CustomJS(args=dict(source=source), code="""
    function disable_button(button) {
        button_id = button.get('id');
        button_element = document.querySelector('#modelid_' + button_id + '>button');
        button_element.setAttribute('disabled')
    }
    function enable_button(button) {
        button_id = button.get('id');
        button_element = document.querySelector('#modelid_' + button_id + '>button');
        button_element.removeAttribute('disabled')
    }
    var data = source.get('data');
    interval_id = data['interval_id'];
    start_time = data['start_time'];
    time_remaining = data['time_remaining'];
    time_string = data['time_string'];
    text_color = data['text_color'];
    fill_color = data['fill_color'];
    text_color[0] = '#ffffff';
    fill_color[0] = '78c400';
    if (interval_id[0] != 0) {
        clearInterval(interval_id[0])
    }
    time_remaining[0] = start_time[0];
    time_string[0] = ('0' + Math.floor(time_remaining[0] / 60)).slice(-2) + ':' + ('0' + Math.floor(time_remaining[0] % 60)).slice(-2);
    enable_button(start_button);
    disable_button(stop_button);
    disable_button(reset_button);
    source.trigger('change');
""")

set_start_time_JS = CustomJS(args=dict(source=source), code="""
    var data = source.get('data');
    var input_mins = mins_slider.get('value');
    var input_secs = secs_slider.get('value');
    time_string = data['time_string'];
    time_remaining = data['time_remaining'];
    start_button_id = start_button.get('id');
    start_button_element = document.querySelector('#modelid_' + start_button_id + '>button');
    start_time = data['start_time'];
    start_time[0] = (input_mins * 60) + (input_secs);
    if (start_button_element.hasAttribute('disabled')) {
    } else {
        time_string[0] = ('0' + Math.floor(time_remaining[0] / 60)).slice(-2) + ':' + ('0' + Math.floor(time_remaining[0] % 60)).slice(-2);
    }
    source.trigger('change');
""")

# Create plot: a color block, with text centered inside
p1 = bkplt.figure(x_range=(-8, 8), y_range=(-5, 5),
                  plot_width=900, plot_height=600,
                  title=None, tools=tools)
p1.rect(x=[0], y=[0],
        width=16, height=10,
        fill_color='fill_color',
        line_color=None,
        name='block',
        source=source)
p1.text(x=[0], y=[0],
        text='time_string',
        text_color='text_color',
        alpha=0.75,
        text_font_size='128pt',
        text_baseline='middle',
        text_align='center',
        name='timer',
        source=source)

# Remove axes, labels & tick lines
p1.ygrid.grid_line_color = None
p1.xgrid.grid_line_color = None
p1.axis.axis_line_color  = None
p1.axis.major_label_text_color = None
p1.axis.major_tick_line_color = None
p1.axis.minor_tick_line_color = None

# Sliders
minutes_slider = Slider(start=0, end=99,
                        value=default_minutes, step=1,
                        title="Minutes",
                        callback=set_start_time_JS)
seconds_slider = Slider(start=0, end=59,
                        value=default_seconds, step=1,
                        title="Seconds",
                        callback=set_start_time_JS)
set_start_time_JS.args['mins_slider'] = minutes_slider
set_start_time_JS.args['secs_slider'] = seconds_slider

# Buttons
start_button = Button(label="Start", callback=run_timer_JS)
run_timer_JS.args['start_button'] = stop_timer_JS.args['start_button'] = reset_timer_JS.args['start_button'] = set_start_time_JS.args['start_button'] = start_button
stop_button = Button(label="Stop", callback=stop_timer_JS)
run_timer_JS.args['stop_button'] = stop_timer_JS.args['stop_button'] = reset_timer_JS.args['stop_button'] = stop_button
reset_button = Button(label="Reset", callback=reset_timer_JS)
run_timer_JS.args['reset_button'] = stop_timer_JS.args['reset_button'] = reset_timer_JS.args['reset_button'] = reset_button

# Layout plot & widgets
layout = column(row(minutes_slider, seconds_slider),
                row(start_button, stop_button, reset_button),
                p1)

# Show figure
bkplt.output_file('browsertimer.html')
bkplt.show(layout)
<|endoftext|>


<commit_before>#!flask/bin/python
import re, json
from flask import jsonify

from persistent_helpers import get_recipe_info

def route_command(recipe_id, text):
    """Decide if command is:
    a. Generic:
        - what is step <X>?
        - how much of <X> is required?
    b. Hardware related
        - what's the temperature of the oven?
        - is the oven ready?
    Route command accordingly

    @type recipe_id: number
    @type text: Request as String
    @return: Respose as String
    """
    text = text.lower()

    tokens = map(lambda x: x.strip().trim(), x.split())
    tokens = set(tokens)

    hardware_kw = set(['temperature', 'oven', 'owen'])

    if len(tokens & hardware_kw) > 0:
        response = exec_hardware_command(text)
    else:
        response = exec_generic_command(recipe_id, text)

    if response is None:
        # TODO Handle elegantly
        return None

    return response

def exec_generic_command(recipe_id, text):
    """Execute generic command:
    A. what is step <X>?
    B. how much of <X> is required?

    @type recipe_id: number
    @type text: Request as String
    @return: Respose as String
    """
    # Keywords for A and B
    step_kw = ['step']
    ing_qty_kw = ['how much', 'required', 'needed', 'need', 'how many']

    # Check what kind of a question it is
    step_match = sum([kw in text for kw in step_kw])
    ing_quantity_match = sum([kw in text for kw in ing_qty_kw])

    if step_match == 0 and ing_quantity_match == 0:
        return None
    elif step_match > 0 :
        respond_to_step(text, recipe_id)
    elif ing_quantity_match > 0:
        respond_to_ing_qty(text, recipe_id)
    else:
        return None

def exec_hardware_command(text):
    """Execute hardware-related command
    A. what is the temperature of the oven?
    B. is the oven ready?

    @type text: Request as String
    @return: Respose as String
    """
    # Keywords for A and B
    temp_kw = ['temperature', ]
    ready_kw = ['ready', ]

    # Check what kind of a question it is
    temp_match = sum([kw in text for kw in temp_kw])
    ready_match = sum([kw in text for kw in ready_kw])

    if temp_match == 0 and ready_kw == 0:
        return None
    elif temp_match > 0 :
        respond_to_step(text)
    elif ing_quantity_match > 0:
        ready_match(text)
    else:
        return None

def respond_to_step(text, recipe_id):
    """Respond to:
    'what is step <X>?'
    """
    # Which step is required?
    match = re.search('step (?P<step_num>\d+)', text)
    step_num = int(match.group('step_num'))

    # What are the steps in the recipe?
    recipe_info_str = get_recipe_info(recipe_id)
    recipe_info = json.loads(recipe_info_str)
    instructions = recipe_info["Instructions"].split('. ')
    num_instructions = len(instructions)

    # Construct response
    if step_num > num_instructions:
        response = "Step %d does not exist. There are %d steps." % (step_num, num_instructions)
    else:
        response = num_instructions[step_num-1]

    return jsonify({'response' : response})

def respond_to_ing_qty(text, recipe_id):
    """Respond to:
    'how much of <X> is required?'
    """
    # What is the ingredient?
    # Detect by simple pattern matching
    match = re.search('of (?P<ing>\w+) is', text)
    ingredient = match.group('ing')

    # What are the ingredients in this recipe?
    recipe_info_str = get_recipe_info(recipe_id)
    recipe_info = json.loads(recipe_info_str)
    # List of 'ingredient' dicts
    ingredients = recipe_info["Ingredients"]
    # Create a dict {ingredient_name -> quantity}
    ing_tuples = [(ing['Name'].lower(), int(ing['Quantity']), ing['Unit'].lower())]
    ing_dct = {name : quantity for name, quantity, unit in ing_tuples}

    # Construct a response
    if ingredient not in ing_dct.keys():
        response = "%s is not necessary" % ingredient
    else:
        response = "%d required" % ing_dct[ingredient]

    return jsonify({'response' : response})

def respond_to_temp(text):
    # What is the temperature?

    # Construct response
    pass

def respond_to_ready(text):
    # What is the state?

    # Construct response
    pass
<commit_msg>Fixing bugs<commit_after>#!flask/bin/python
import re, json
from flask import jsonify

from persistent_helpers import get_recipe_info

def route_command(text, recipe_id):
    """Decide if command is:
    a. Generic:
        - what is step <X>?
        - how much of <X> is required?
    b. Hardware related
        - what's the temperature of the oven?
        - is the oven ready?
    Route command accordingly

    @type recipe_id: number
    @type text: Request as String
    @return: Respose as String
    """
    text = text.lower()

    tokens = map(lambda x: x.strip(), text.split())
    tokens = set(tokens)

    hardware_kw = set(['temperature', 'oven', 'owen'])

    if len(tokens & hardware_kw) > 0:
        response = exec_hardware_command(text)
    else:
        response = exec_generic_command(recipe_id, text)

    if response is None:
        # TODO Handle elegantly
        return None

    return response

def exec_generic_command(recipe_id, text):
    """Execute generic command:
    A. what is step <X>?
    B. how much of <X> is required?

    @type recipe_id: number
    @type text: Request as String
    @return: Respose as String
    """
    # Keywords for A and B
    step_kw = ['step']
    ing_qty_kw = ['how much', 'required', 'needed', 'need', 'how many']

    # Check what kind of a question it is
    step_match = sum([kw in text for kw in step_kw])
    ing_quantity_match = sum([kw in text for kw in ing_qty_kw])

    if step_match == 0 and ing_quantity_match == 0:
        return None
    elif step_match > 0 :
        return respond_to_step(text, recipe_id)
    elif ing_quantity_match > 0:
        return respond_to_ing_qty(text, recipe_id)
    else:
        return None

def exec_hardware_command(text):
    """Execute hardware-related command
    A. what is the temperature of the oven?
    B. is the oven ready?

    @type text: Request as String
    @return: Respose as String
    """
    # Keywords for A and B
    temp_kw = ['temperature', ]
    ready_kw = ['ready', ]

    # Check what kind of a question it is
    temp_match = sum([kw in text for kw in temp_kw])
    ready_match = sum([kw in text for kw in ready_kw])

    if temp_match == 0 and ready_kw == 0:
        return None
    elif temp_match > 0 :
        return respond_to_step(text)
    elif ing_quantity_match > 0:
        return ready_match(text)
    else:
        return None

def respond_to_step(text, recipe_id):
    """Respond to:
    'what is step <X>?'
    """
    # Which step is required?
    match = re.search('step (?P<step_num>\d+)', text)
    step_num = int(match.group('step_num'))

    # What are the steps in the recipe?
    recipe_info_str = get_recipe_info(recipe_id)
    recipe_info = json.loads(recipe_info_str)
    instructions = recipe_info["Instructions"].split('. ')
    num_instructions = len(instructions)

    # Construct response
    if step_num > num_instructions:
        response = "Step %d does not exist. There are %d steps." % (step_num, num_instructions)
    else:
        response = instructions[step_num-1]

    return json.dumps({'response' : response})

def respond_to_ing_qty(text, recipe_id):
    """Respond to:
    'how much of <X> is required?'
    """
    # What is the ingredient?
    # Detect by simple pattern matching
    match = re.search('of (?P<ing>\w+) is', text)
    ingredient = match.group('ing')

    # What are the ingredients in this recipe?
    recipe_info_str = get_recipe_info(recipe_id)
    recipe_info = json.loads(recipe_info_str)
    # List of 'ingredient' dicts
    ingredients = recipe_info["Ingredients"]
    # Create a dict {ingredient_name -> quantity}
    ing_tuples = [(ing['Name'].lower(), ing['Quantity'], ing['Unit'].lower()) for ing in ingredients]
    ing_dct = {name : (quantity, unit) for name, quantity, unit in ing_tuples}

    # Construct a response
    if ingredient not in ing_dct.keys():
        response = "%s is not necessary" % ingredient
    else:
        response = "%s %s required" % ing_dct[ingredient]

    return json.dumps({'response' : response})

def respond_to_temp(text):
    # What is the temperature?

    # Construct response
    pass

def respond_to_ready(text):
    # What is the state?

    # Construct response
    pass

def main():
    recipe_id = 1
    text = "how much of flour is required?"
    # print respond_to_ing_qty(text, recipe_id)
    # text = "what is step 2?"
    print route_command(text, recipe_id)

if __name__ == '__main__':
    main()
<|endoftext|>