<commit_before>import sys
import shlex
import argparse
import traceback

from synapse.eventbus import EventBus

class CmdArgErr(Exception):

    def __init__(self, pars, msg):
        self.msg = msg
        self.pars = pars
        Exception.__init__(self, msg)

# fucking argparse....
class CmdArgExit(Exception):pass

class ArgumentParser(argparse.ArgumentParser):

    def error(self, msg):
        # fuck argparse and it's thinking it should unilaterally exit()
        raise CmdArgErr(self,msg)

    def exit(self, *args, **kwargs):
        raise CmdArgExit()

class CliFini(Exception): pass

class Cli(EventBus):
    '''
    A modular / event-driven CLI object similar to cmd.Cmd (but not lame)

    Example:

        import synapse.lib.cli as s_cli

        class WootCli(s_cli.Cli):

            def cmd_woot(self, cli, line):
                """
                The woot command.
                """
                # using CLI rather than self allows cli "federation"
                cli.vprint('woot!')

        wootcli = WootCli()

        # can also dynamically register commands...

        def lulz(cli,line):
            """
            Lulz the live long day.
            """
            stuff()

        wootcli.addCmdFunc(lulz)

        # or methods from other objects...

        class Foo:
            def cmd_bar(self, cli, line):

        foo = Foo()
        cli.addCmdMeths(foo)

        # cli now has "bar" command

    '''
    def __init__(self, outp=None):
        #cmd.Cmd.__init__(self)
        EventBus.__init__(self)

        self.cmdfuncs = {}
        self.cmdprompt = 'cli> '

        self.addCmdMeths(self)
        self.addCmdFunc(cmd_quit, name='quit')
        self.addCmdFunc(cmd_help, name='help')

    def getCmdNames(self):
        '''
        Return a list of all the known command names for the CLI.
        '''
        return list(self.cmdfuncs.keys())

    def delCmdFunc(self, name):
        '''
        Remove a previously registered command function.
        '''
        return self.cmdfuncs.pop(name,None)

    def getCmdDoc(self, name):
        '''
        Return the doc string for the given function name.
        '''
        func = self.getCmdFunc(name)
        if func == None:
            return None

        return func.__doc__

    def getCmdBrief(self, name):
        '''
        Return the strip()d first line from the cmd doc string.
        '''
        doc = self.getCmdDoc(name)
        if doc == None:
            return None

        return doc.strip().split('\n',1)[0].strip()

    def getCmdFunc(self, name):
        '''
        Return the func callback for a given command.
        '''
        return self.cmdfuncs.get(name)

    def addStdPrint(self):
        '''
        Add a callback to print cli:print events to stdout.
        '''
        self.on('cli:print', _cliprint)

    def addCmdFunc(self, func, name=None):
        '''
        Add an additional command callback to the CLI object.

        def woot(cli,line):
            do_woot_stuff(line)

        cli.addCmdFunc('woot', woot)

        # cli now has "woot" command

        '''
        if name == None:
            name = func.__name__

        self.cmdfuncs[name] = func

    def _printCliExc(self, exc):
        exctxt = traceback.format_exc()
        self.vprint(exctxt)
        self.vprint('error: %s' % exc)

    def getArgParser(self, prog):
        '''
        Return a synapse ArgumentParser instance.

        Example:

            pars = cli.getArgParser()

        Notes:

            * our ArgumentParser subclass doesnt exit()

        '''
        return ArgumentParser(prog=prog)

    def vprint(self, msg, addnl=True):
        '''
        Print output for the CLI.
        '''
        self.fire('cli:print', msg=msg, addnl=addnl)

    def getLineArgv(self, line):
        '''
        Chop up an input line and return argv list.

        Example:

            argv = cli.getLineArgv(line)

        '''
        return shlex.split(line)[1:]

    def addCmdMeths(self, item):
        '''
        Add all cmd_<foo> methods from the given item as commands.

        Example:

            import synapse.lib.cli as s_cli

            class Lulz:

                def cmd_woot(self, cli, line):
                    cli.vprint('woot: %s' % (line,))

            lulz = Lulz()

            cli = s_cli.Cli()
            cli.addCmdMeths(lulz)

            # cli now has command named "woot"

        '''
        for name in dir(item):

            if not name.startswith('cmd_'):
                continue

            meth = getattr(item,name,None)
            if not meth:
                continue

            self.addCmdFunc(meth, name=name[4:])

    def runCmdLoop(self, stdin=sys.stdin):
        '''
        Run commands from stdin until close or fini().
        '''

        while not self.isfini:

            # if our stdin closes, return from runCmdLoop

            # FIXME raw_input
            # FIXME history / completion

            try:

                self.vprint( self.cmdprompt, addnl=False )

                line = input()
                #line = stdin.readline()
                if not line:
                    break

                line = line.strip()
                if not line:
                    self.vprint('FIXME EMPTY')
                    continue

                self.runCmdLine(line)

            except KeyboardInterrupt as e:
                self.vprint('<ctrl-c>')

            except Exception as e:
                traceback.print_exc()

    def runCmdLine(self, line):
        '''
        Run a single command line.

        Example:

            cli.runCmdLine('woot --help')

        '''
        ret = None
        name = line.split(None,1)[0]
        func = self.getCmdFunc(name)

        # FIXME partial match!
        if func == None:
            self.vprint('cmd not found: %s' % (name,))
            return

        self.fire('cli:cmd:run', line=line)

        try:

            ret = func(self,line)

        except CliFini as e:
            self.fini()

        except KeyboardInterrupt as e:
            self.vprint('<ctrl-c>')

        except CmdArgErr as e:
            self.vprint( e.pars.format_usage() )
            self.vprint( e.msg )

        except CmdArgExit as e:
            pass

        except Exception as e:
            self._printCliExc(e)

        self.fire('cli:cmd:ret', line=line, ret=ret)

        return ret

def _cliprint(mesg):
    '''
    Callback for printing cli:print events.
    '''
    msg = mesg[1].get('msg')

    addnl = mesg[1].get('addnl')
    if addnl:
        msg += '\n'

    sys.stdout.write(msg)
    sys.stdout.flush()

def cmd_quit(cli, line):
    '''
    Quit the CLI.
    '''
    cli.fini()
    raise CliFini()

def cmd_help(cli, line):
    '''
    Show command list and descriptions.
    '''
    argv = cli.getLineArgv(line)
    if argv:

        for name in argv:
            cli.vprint('=== %s' % (name,))

            doc = cli.getCmdDoc(name)
            if doc != None:
                cli.vprint(doc)

            cli.runCmdLine('%s --help' % name)

        return

    names = cli.getCmdNames()

    names.sort()
    padsize = max( [ len(n) for n in names ] )

    for name in names:
        brief = cli.getCmdBrief(name)
        padname = name.ljust(padsize)
        cli.vprint('%s - %s' % (padname,brief))

    cli.vprint('(%d cmds)' % (len(names),))

<commit_msg>go around again on blank line rather than breaking out<commit_after>import sys
import shlex
import argparse
import traceback

from synapse.eventbus import EventBus

class CmdArgErr(Exception):

    def __init__(self, pars, msg):
        self.msg = msg
        self.pars = pars
        Exception.__init__(self, msg)

# fucking argparse....
class CmdArgExit(Exception):pass

class ArgumentParser(argparse.ArgumentParser):

    def error(self, msg):
        # fuck argparse and it's thinking it should unilaterally exit()
        raise CmdArgErr(self,msg)

    def exit(self, *args, **kwargs):
        raise CmdArgExit()

class CliFini(Exception): pass

class Cli(EventBus):
    '''
    A modular / event-driven CLI object similar to cmd.Cmd (but not lame)

    Example:

        import synapse.lib.cli as s_cli

        class WootCli(s_cli.Cli):

            def cmd_woot(self, cli, line):
                """
                The woot command.
                """
                # using CLI rather than self allows cli "federation"
                cli.vprint('woot!')

        wootcli = WootCli()

        # can also dynamically register commands...

        def lulz(cli,line):
            """
            Lulz the live long day.
            """
            stuff()

        wootcli.addCmdFunc(lulz)

        # or methods from other objects...

        class Foo:
            def cmd_bar(self, cli, line):

        foo = Foo()
        cli.addCmdMeths(foo)

        # cli now has "bar" command

    '''
    def __init__(self, outp=None):
        #cmd.Cmd.__init__(self)
        EventBus.__init__(self)

        self.cmdfuncs = {}
        self.cmdprompt = 'cli> '

        self.addCmdMeths(self)
        self.addCmdFunc(cmd_quit, name='quit')
        self.addCmdFunc(cmd_help, name='help')

    def getCmdNames(self):
        '''
        Return a list of all the known command names for the CLI.
        '''
        return list(self.cmdfuncs.keys())

    def delCmdFunc(self, name):
        '''
        Remove a previously registered command function.
        '''
        return self.cmdfuncs.pop(name,None)

    def getCmdDoc(self, name):
        '''
        Return the doc string for the given function name.
        '''
        func = self.getCmdFunc(name)
        if func == None:
            return None

        return func.__doc__

    def getCmdBrief(self, name):
        '''
        Return the strip()d first line from the cmd doc string.
        '''
        doc = self.getCmdDoc(name)
        if doc == None:
            return None

        return doc.strip().split('\n',1)[0].strip()

    def getCmdFunc(self, name):
        '''
        Return the func callback for a given command.
        '''
        return self.cmdfuncs.get(name)

    def addStdPrint(self):
        '''
        Add a callback to print cli:print events to stdout.
        '''
        self.on('cli:print', _cliprint)

    def addCmdFunc(self, func, name=None):
        '''
        Add an additional command callback to the CLI object.

        def woot(cli,line):
            do_woot_stuff(line)

        cli.addCmdFunc('woot', woot)

        # cli now has "woot" command

        '''
        if name == None:
            name = func.__name__

        self.cmdfuncs[name] = func

    def _printCliExc(self, exc):
        exctxt = traceback.format_exc()
        self.vprint(exctxt)
        self.vprint('error: %s' % exc)

    def getArgParser(self, prog):
        '''
        Return a synapse ArgumentParser instance.

        Example:

            pars = cli.getArgParser()

        Notes:

            * our ArgumentParser subclass doesnt exit()

        '''
        return ArgumentParser(prog=prog)

    def vprint(self, msg, addnl=True):
        '''
        Print output for the CLI.
        '''
        self.fire('cli:print', msg=msg, addnl=addnl)

    def getLineArgv(self, line):
        '''
        Chop up an input line and return argv list.

        Example:

            argv = cli.getLineArgv(line)

        '''
        return shlex.split(line)[1:]

    def addCmdMeths(self, item):
        '''
        Add all cmd_<foo> methods from the given item as commands.

        Example:

            import synapse.lib.cli as s_cli

            class Lulz:

                def cmd_woot(self, cli, line):
                    cli.vprint('woot: %s' % (line,))

            lulz = Lulz()

            cli = s_cli.Cli()
            cli.addCmdMeths(lulz)

            # cli now has command named "woot"

        '''
        for name in dir(item):

            if not name.startswith('cmd_'):
                continue

            meth = getattr(item,name,None)
            if not meth:
                continue

            self.addCmdFunc(meth, name=name[4:])

    def runCmdLoop(self, stdin=sys.stdin):
        '''
        Run commands from stdin until close or fini().
        '''

        while not self.isfini:

            # if our stdin closes, return from runCmdLoop

            # FIXME raw_input
            # FIXME history / completion

            try:

                self.vprint( self.cmdprompt, addnl=False )

                line = input()
                if not line:
                    continue

                line = line.strip()
                if not line:
                    self.vprint('FIXME EMPTY')
                    continue

                self.runCmdLine(line)

            except KeyboardInterrupt as e:
                self.vprint('<ctrl-c>')

            except Exception as e:
                traceback.print_exc()

    def runCmdLine(self, line):
        '''
        Run a single command line.

        Example:

            cli.runCmdLine('woot --help')

        '''
        ret = None
        name = line.split(None,1)[0]
        func = self.getCmdFunc(name)

        # FIXME partial match!
        if func == None:
            self.vprint('cmd not found: %s' % (name,))
            return

        self.fire('cli:cmd:run', line=line)

        try:

            ret = func(self,line)

        except CliFini as e:
            self.fini()

        except KeyboardInterrupt as e:
            self.vprint('<ctrl-c>')

        except CmdArgErr as e:
            self.vprint( e.pars.format_usage() )
            self.vprint( e.msg )

        except CmdArgExit as e:
            pass

        except Exception as e:
            self._printCliExc(e)

        self.fire('cli:cmd:ret', line=line, ret=ret)

        return ret

def _cliprint(mesg):
    '''
    Callback for printing cli:print events.
    '''
    msg = mesg[1].get('msg')

    addnl = mesg[1].get('addnl')
    if addnl:
        msg += '\n'

    sys.stdout.write(msg)
    sys.stdout.flush()

def cmd_quit(cli, line):
    '''
    Quit the CLI.
    '''
    cli.fini()
    raise CliFini()

def cmd_help(cli, line):
    '''
    Show command list and descriptions.
    '''
    argv = cli.getLineArgv(line)
    if argv:

        for name in argv:
            cli.vprint('=== %s' % (name,))

            doc = cli.getCmdDoc(name)
            if doc != None:
                cli.vprint(doc)

            cli.runCmdLine('%s --help' % name)

        return

    names = cli.getCmdNames()

    names.sort()
    padsize = max( [ len(n) for n in names ] )

    for name in names:
        brief = cli.getCmdBrief(name)
        padname = name.ljust(padsize)
        cli.vprint('%s - %s' % (padname,brief))

    cli.vprint('(%d cmds)' % (len(names),))

<|endoftext|>


<commit_before><commit_msg>Optimization in synch word finder<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python

"""
@package ion.agents.alarms.alarms
@file ion/agents/alarms/alarms.py
@author Edward Hunter
@brief Alarm objects to control construction of valid alarm expressions.
"""

__author__ = 'Edward Hunter'
__license__ = 'Apache 2.0'

# Pyon imports
from pyon.public import IonObject, log

# Alarm types and events.
from interface.objects import StreamAlarmType

class BaseAlarm(object):
    """
    Base alarm object.
    """
    def __init__(self, name, stream_id, value_id, message, type):
        """
        Populate fields used by all alarms.
        """

        if not isinstance(name, str):
            raise TypeError('Invalid name.')

        if not isinstance(stream_id, str):
            raise TypeError('Invalid stream id.')

        if not isinstance(value_id, str):
            raise TypeError('Invalid value id.')

        if not isinstance(message, str):
            raise TypeError('Invalid message.')

        if not isinstance(type, int):
            raise TypeError('Invalid alarm type.')

        if not type in StreamAlarmType._value_map.values():
            raise TypeError('Invalid alarm type.')

        self.name = name
        self.stream_id = stream_id
        self.value_id = value_id
        self.message = message
        self.type = type
        self.expr = ''
        self.status = None
        self.current_val = None

    def eval_alarm(self, x):
        """
        Evalueate boolean alarm expression.
        """
        self.current_val = x
        old_status = self.status
        self.status = eval(self.expr)

        retval = None
        if old_status != self.status:

            event_data = {
                'name' : self.name,
                'message' : self.message,
                'expr' : self.expr,
                'stream_id' : self.stream_id,
                'value_id' : self.value_id,
                'value' : x
            }

            if not self.status:
                event_data['event_type'] = 'StreamAllClearAlarrmEvent'
                event_data['message'] = 'The alarm %s has cleared.' % self.name
                retval = event_data

            elif self.type == StreamAlarmType.WARNING:
                event_data['event_type'] = 'StreamWarningAlaramEvent'
                retval = event_data

            elif self.type == StreamAlarmType.ALERT:
                event_data['event_type'] = 'StreamAlertAlarmEvent'
                retval = event_data

            else:
                log.error('Unknown alarm type.')

        return retval

    def __str__(self):
        """
        Pretty print the alarm object.
        """
        fmt = ('Alarm object of class:%s \n   name:%s \n   message:%s \n'
            '   expr:%s \n   stream_id:%s \n   value_id:%s \n   value:%s')

        s = fmt % (self.__class__.__name__,
                  self.name,
                  self.message,
                  self.expr,
                  self.stream_id,
                  self.value_id,
                  str(self.current_val))

        return s

class IntervalAlarm(BaseAlarm):
    """
    An alarm that specifies an interval range. Can be one sided or closed.
    """
    def __init__(self, name, stream_id, value_id, message, type,
                 lower_bound=None, lower_rel_op=None,
                 upper_bound=None, upper_rel_op=None):
        """
        Call superclass and construct interval expression.
        """
        super(IntervalAlarm, self).__init__(name, stream_id, value_id, message,
                                            type)

        self.lower_bound = lower_bound
        self.lower_rel_op = lower_rel_op
        self.upper_bound = upper_bound
        self.upper_rel_op = upper_rel_op

        if self.lower_bound:
            if not isinstance(self.lower_bound, (int, float)):
                raise TypeError('Bad lower bound value.')
            if self.lower_rel_op not in ('<','<='):
                raise TypeError('Bad lower bound relational op.')

        if self.upper_bound:
            if not isinstance(self.upper_bound, (int, float)):
                raise TypeError('Bad upper bound value.')
            if self.upper_rel_op not in ('<','<='):
                raise TypeError('Bad upper bound relational op.')

        if self.lower_bound and self.upper_bound:
            if self.lower_bound >= self.upper_bound:
                raise ValueError('Lower bound >= upper bound.')

        if self.lower_bound:
            self.expr += str(self.lower_bound)
            self.expr += self.lower_rel_op

        self.expr += 'x'

        if self.upper_bound:
            self.expr += self.upper_rel_op
            self.expr += str(self.upper_bound)

class DoubleIntervalAlarm(BaseAlarm):
    """
    An alarm providing a double interval. Either the left lower bound,
    the right upper bound or both may be open.
    """
    pass

class SetMembershipAlarm(BaseAlarm):
    """
    An alarm providing membership in a discrete set of objects.
    """
    pass

class UserDefinedAlarm(BaseAlarm):
    """
    An alarm provided by a user supplied expression.
    This must be used with caution.
    """
    def __init__(self, name, stream_id, value_id, message, type, expr=None):
        """
        """
        super(IntervalAlarm, self).__init__(name, stream_id, value_id, message,
                                            type)
        self.expr = expr

        if not isinstance(self.expr, str):
            raise TypeError('Alarm expression must be a string.')

<commit_msg>Minor reorder in constructor.<commit_after>#!/usr/bin/env python

"""
@package ion.agents.alarms.alarms
@file ion/agents/alarms/alarms.py
@author Edward Hunter
@brief Alarm objects to control construction of valid alarm expressions.
"""

__author__ = 'Edward Hunter'
__license__ = 'Apache 2.0'

# Pyon imports
from pyon.public import IonObject, log

# Alarm types and events.
from interface.objects import StreamAlarmType

class BaseAlarm(object):
    """
    Base alarm object.
    """
    def __init__(self, name, stream_id, value_id, message, type):
        """
        Populate fields used by all alarms.
        """

        if not isinstance(name, str):
            raise TypeError('Invalid name.')

        if not isinstance(stream_id, str):
            raise TypeError('Invalid stream id.')

        if not isinstance(value_id, str):
            raise TypeError('Invalid value id.')

        if not isinstance(message, str):
            raise TypeError('Invalid message.')

        if not isinstance(type, int):
            raise TypeError('Invalid alarm type.')

        if not type in StreamAlarmType._value_map.values():
            raise TypeError('Invalid alarm type.')

        self.name = name
        self.stream_id = stream_id
        self.value_id = value_id
        self.message = message
        self.type = type
        self.expr = ''
        self.status = None
        self.current_val = None

    def eval_alarm(self, x):
        """
        Evalueate boolean alarm expression.
        """
        self.current_val = x
        old_status = self.status
        self.status = eval(self.expr)

        retval = None
        if old_status != self.status:

            event_data = {
                'name' : self.name,
                'message' : self.message,
                'expr' : self.expr,
                'stream_id' : self.stream_id,
                'value_id' : self.value_id,
                'value' : x
            }

            if not self.status:
                event_data['event_type'] = 'StreamAllClearAlarrmEvent'
                event_data['message'] = 'The alarm %s has cleared.' % self.name
                retval = event_data

            elif self.type == StreamAlarmType.WARNING:
                event_data['event_type'] = 'StreamWarningAlaramEvent'
                retval = event_data

            elif self.type == StreamAlarmType.ALERT:
                event_data['event_type'] = 'StreamAlertAlarmEvent'
                retval = event_data

            else:
                log.error('Unknown alarm type.')

        return retval

    def __str__(self):
        """
        Pretty print the alarm object.
        """
        fmt = ('Alarm object of class:%s \n   name:%s \n   message:%s \n'
            '   expr:%s \n   stream_id:%s \n   value_id:%s \n   value:%s')

        s = fmt % (self.__class__.__name__,
                  self.name,
                  self.message,
                  self.expr,
                  self.stream_id,
                  self.value_id,
                  str(self.current_val))

        return s

class IntervalAlarm(BaseAlarm):
    """
    An alarm that specifies an interval range. Can be one sided or closed.
    """
    def __init__(self, name, stream_id, value_id, message, type,
                 lower_bound=None, lower_rel_op=None,
                 upper_bound=None, upper_rel_op=None):
        """
        Call superclass and construct interval expression.
        """
        super(IntervalAlarm, self).__init__(name, stream_id, value_id, message,
                                            type)

        self.lower_bound = lower_bound
        self.lower_rel_op = lower_rel_op
        self.upper_bound = upper_bound
        self.upper_rel_op = upper_rel_op

        if self.lower_bound:
            if not isinstance(self.lower_bound, (int, float)):
                raise TypeError('Bad lower bound value.')
            if self.lower_rel_op not in ('<','<='):
                raise TypeError('Bad lower bound relational op.')

        if self.upper_bound:
            if not isinstance(self.upper_bound, (int, float)):
                raise TypeError('Bad upper bound value.')
            if self.upper_rel_op not in ('<','<='):
                raise TypeError('Bad upper bound relational op.')

        if self.lower_bound and self.upper_bound:
            if self.lower_bound >= self.upper_bound:
                raise ValueError('Lower bound >= upper bound.')

        if self.lower_bound:
            self.expr += str(self.lower_bound)
            self.expr += self.lower_rel_op

        self.expr += 'x'

        if self.upper_bound:
            self.expr += self.upper_rel_op
            self.expr += str(self.upper_bound)

class DoubleIntervalAlarm(BaseAlarm):
    """
    An alarm providing a double interval. Either the left lower bound,
    the right upper bound or both may be open.
    """
    pass

class SetMembershipAlarm(BaseAlarm):
    """
    An alarm providing membership in a discrete set of objects.
    """
    pass

class UserDefinedAlarm(BaseAlarm):
    """
    An alarm provided by a user supplied expression.
    This must be used with caution.
    """
    def __init__(self, name, stream_id, value_id, message, type, expr=None):
        """
        """
        super(IntervalAlarm, self).__init__(name, stream_id, value_id, message,
                                            type)

        if not isinstance(self.expr, str):
            raise TypeError('Alarm expression must be a string.')

        self.expr = expr

<|endoftext|>


<commit_before>#
# Copyright 2014 WebFilings, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""Example of using Context events for work flow.

This example creates a context with events, then adds a few Async jobs to the
Context.
"""

import logging

import webapp2

class ContextCompletionHandler(webapp2.RequestHandler):
    """Demonstrate using Context Events to make work flows."""
    def get(self):
        from furious.async import Async
        from furious import context

        count = self.request.get('tasks', 5)

        # Create a new furious Context.
        with context.new(persist_async_results=True) as ctx:
            # Set a completion event handler.
            ctx.set_event_handler('complete',
                                  Async(context_complete, args=[ctx.id]))

            # Insert some Asyncs.
            for i in xrange(int(count)):
                ctx.add(target=async_worker, args=[ctx.id, i])
                logging.info('Added job %d to context.', i)

        # When the Context is exited, the tasks are inserted (if there are no
        # errors).

        logging.info('Async jobs for context batch inserted.')

        self.response.out.write('Successfully inserted a group of Async jobs.')

def async_worker(*args, **kwargs):
    """This function is called by furious tasks to demonstrate usage."""
    logging.info('Context %s, function %s', *args)

    return args

def context_complete(context_id):
    """Log out that the context is complete."""
    logging.info('Context %s is.......... DONE.', context_id)

    from furious.context import get_current_async
    from furious.context import Context

    async = get_current_async()
    if not (async and async.context_id):
        return context_id

    context = Context.load(async.context_id)

    if not context:
        return context_id

    for result in context.iter_results():
        logging.info("#########################")
        logging.info(result)

    return context_id
<commit_msg>Update example with error logging. Improve comment<commit_after>#
# Copyright 2014 WebFilings, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""Example of using Context events for work flow.

This example creates a context with events, then adds a few Async jobs to the
Context.
"""

import logging

import webapp2

class ContextCompletionHandler(webapp2.RequestHandler):
    """Demonstrate using Context Events to make work flows."""
    def get(self):
        from furious.async import Async
        from furious import context

        count = self.request.get('tasks', 5)

        # Create a new furious Context with persistance enabled.
        with context.new(persist_async_results=True) as ctx:
            # Set a completion event handler.
            ctx.set_event_handler('complete',
                                  Async(context_complete, args=[ctx.id]))

            # Insert some Asyncs.
            for i in xrange(int(count)):
                ctx.add(target=async_worker, args=[ctx.id, i])
                logging.info('Added job %d to context.', i)

        # When the Context is exited, the tasks are inserted (if there are no
        # errors).

        logging.info('Async jobs for context batch inserted.')

        self.response.out.write('Successfully inserted a group of Async jobs.')

def async_worker(*args, **kwargs):
    """This function is called by furious tasks to demonstrate usage."""
    logging.info('Context %s, function %s', *args)

    return args

def context_complete(context_id):
    """Log out that the context is complete."""
    logging.info('Context %s is.......... DONE.', context_id)

    from furious.context import get_current_async
    from furious.context import Context

    async = get_current_async()
    if not (async and async.context_id):
        logging.error("Could not find current async")
        return

    context = Context.load(async.context_id)

    if not context:
        logging.error("Could not load context")
        return

    for result in context.iter_results():
        logging.info("#########################")
        logging.info(result)

    return context_id
<|endoftext|>


<commit_before>"""Test functions for the sparse.linalg._onenormest module
"""

from __future__ import division, print_function, absolute_import

import numpy as np
from numpy.testing import (assert_allclose, assert_equal, assert_,
        decorators, TestCase, run_module_suite)
import scipy.linalg
import scipy.sparse.linalg
from scipy.sparse.linalg._onenormest import _onenormest_core

class MatrixProductOperator(scipy.sparse.linalg.LinearOperator):
    """
    This is purely for onenormest testing.
    """

    def __init__(self, A, B):
        if A.ndim != 2 or B.ndim != 2:
            raise ValueError('expected ndarrays representing matrices')
        if A.shape[1] != B.shape[0]:
            raise ValueError('incompatible shapes')
        self.A = A
        self.B = B
        self.ndim = 2
        self.shape = (A.shape[0], B.shape[1])

    def matvec(self, x):
        return np.dot(self.A, np.dot(self.B, x))

    def rmatvec(self, x):
        return np.dot(np.dot(x, self.A), self.B)

    def matmat(self, X):
        return np.dot(self.A, np.dot(self.B, X))

    @property
    def T(self):
        return MatrixProductOperator(self.B.T, self.A.T)

class TestOnenormest(TestCase):

    @decorators.slow
    @decorators.skipif(True, 'this test is annoyingly slow')
    def test_onenormest_table_3_t_2(self):
        # This will take multiple seconds if your computer is slow like mine.
        # It is stochastic, so the tolerance could be too strict.
        t = 2
        n = 100
        itmax = 5
        nsamples = 5000
        observed = []
        expected = []
        nmult_list = []
        nresample_list = []
        for i in range(nsamples):
            A = scipy.linalg.inv(np.random.randn(n, n))
            est, v, w, nmults, nresamples = _onenormest_core(A, A.T, t, itmax)
            observed.append(est)
            expected.append(scipy.linalg.norm(A, 1))
            nmult_list.append(nmults)
            nresample_list.append(nresamples)
        observed = np.array(observed, dtype=float)
        expected = np.array(expected, dtype=float)
        relative_errors = np.abs(observed - expected) / expected

        # check the mean underestimation ratio
        underestimation_ratio = observed / expected
        assert_(0.99 < np.mean(underestimation_ratio) < 1.0)

        # check the max and mean required column resamples
        assert_equal(np.max(nresample_list), 2)
        assert_(0.05 < np.mean(nresample_list) < 0.2)

        # check the proportion of norms computed exactly correctly
        nexact = np.count_nonzero(relative_errors < 1e-14)
        proportion_exact = nexact / float(nsamples)
        assert_(0.9 < proportion_exact < 0.95)

        # check the average number of matrix*vector multiplications
        assert_(3.5 < np.mean(nmult_list) < 4.5)

    def test_onenormest_table_5_t_1(self):
        # "note that there is no randomness and hence only one estimate for t=1"
        t = 1
        n = 100
        itmax = 5
        alpha = 1 - 1e-6
        A = -scipy.linalg.inv(np.identity(n) + alpha*np.eye(n, k=1))
        first_col = np.array([1] + [0]*(n-1))
        first_row = np.array([(-alpha)**i for i in range(n)])
        B = -scipy.linalg.toeplitz(first_col, first_row)
        assert_allclose(A, B)
        est, v, w, nmults, nresamples = _onenormest_core(B, B.T, t, itmax)
        exact_value = scipy.linalg.norm(B, 1)
        underest_ratio = est / exact_value
        assert_allclose(underest_ratio, 0.05, rtol=1e-4)
        assert_equal(nmults, 11)
        assert_equal(nresamples, 0)
        # check the non-underscored version of onenormest
        est_plain = scipy.sparse.linalg.onenormest(B, t=t, itmax=itmax)
        assert_allclose(est, est_plain)

    def _help_product_norm_slow(self, A, B):
        # for profiling
        C = np.dot(A, B)
        return scipy.linalg.norm(C, 1)

    def _help_product_norm_fast(self, A, B):
        # for profiling
        t = 2
        itmax = 5
        D = MatrixProductOperator(A, B)
        est, v, w, nmults, nresamples = _onenormest_core(D, D.T, t, itmax)
        return est

    @decorators.slow
    def test_onenormest_linear_operator(self):
        # Define a matrix through its product A B.
        # Depending on the shapes of A and B,
        # it could be easy to multiply this product by a small matrix,
        # but it could be annoying to look at all of
        # the entries of the product explicitly.
        n = 6000
        k = 3
        A = np.random.randn(n, k)
        B = np.random.randn(k, n)
        fast_estimate = self._help_product_norm_fast(A, B)
        exact_value = self._help_product_norm_slow(A, B)
        self.assert_(
                fast_estimate <= exact_value <= 3*fast_estimate,
                'fast: %g\nexact:%g' % (fast_estimate, exact_value))

if __name__ == '__main__':
    run_module_suite()

<commit_msg>TST: seed for random tests<commit_after>"""Test functions for the sparse.linalg._onenormest module
"""

from __future__ import division, print_function, absolute_import

import numpy as np
from numpy.testing import (assert_allclose, assert_equal, assert_,
        decorators, TestCase, run_module_suite)
import scipy.linalg
import scipy.sparse.linalg
from scipy.sparse.linalg._onenormest import _onenormest_core

class MatrixProductOperator(scipy.sparse.linalg.LinearOperator):
    """
    This is purely for onenormest testing.
    """

    def __init__(self, A, B):
        if A.ndim != 2 or B.ndim != 2:
            raise ValueError('expected ndarrays representing matrices')
        if A.shape[1] != B.shape[0]:
            raise ValueError('incompatible shapes')
        self.A = A
        self.B = B
        self.ndim = 2
        self.shape = (A.shape[0], B.shape[1])

    def matvec(self, x):
        return np.dot(self.A, np.dot(self.B, x))

    def rmatvec(self, x):
        return np.dot(np.dot(x, self.A), self.B)

    def matmat(self, X):
        return np.dot(self.A, np.dot(self.B, X))

    @property
    def T(self):
        return MatrixProductOperator(self.B.T, self.A.T)

class TestOnenormest(TestCase):

    @decorators.slow
    @decorators.skipif(True, 'this test is annoyingly slow')
    def test_onenormest_table_3_t_2(self):
        # This will take multiple seconds if your computer is slow like mine.
        # It is stochastic, so the tolerance could be too strict.
        np.random.seed(1234)
        t = 2
        n = 100
        itmax = 5
        nsamples = 5000
        observed = []
        expected = []
        nmult_list = []
        nresample_list = []
        for i in range(nsamples):
            A = scipy.linalg.inv(np.random.randn(n, n))
            est, v, w, nmults, nresamples = _onenormest_core(A, A.T, t, itmax)
            observed.append(est)
            expected.append(scipy.linalg.norm(A, 1))
            nmult_list.append(nmults)
            nresample_list.append(nresamples)
        observed = np.array(observed, dtype=float)
        expected = np.array(expected, dtype=float)
        relative_errors = np.abs(observed - expected) / expected

        # check the mean underestimation ratio
        underestimation_ratio = observed / expected
        assert_(0.99 < np.mean(underestimation_ratio) < 1.0)

        # check the max and mean required column resamples
        assert_equal(np.max(nresample_list), 2)
        assert_(0.05 < np.mean(nresample_list) < 0.2)

        # check the proportion of norms computed exactly correctly
        nexact = np.count_nonzero(relative_errors < 1e-14)
        proportion_exact = nexact / float(nsamples)
        assert_(0.9 < proportion_exact < 0.95)

        # check the average number of matrix*vector multiplications
        assert_(3.5 < np.mean(nmult_list) < 4.5)

    def test_onenormest_table_5_t_1(self):
        # "note that there is no randomness and hence only one estimate for t=1"
        t = 1
        n = 100
        itmax = 5
        alpha = 1 - 1e-6
        A = -scipy.linalg.inv(np.identity(n) + alpha*np.eye(n, k=1))
        first_col = np.array([1] + [0]*(n-1))
        first_row = np.array([(-alpha)**i for i in range(n)])
        B = -scipy.linalg.toeplitz(first_col, first_row)
        assert_allclose(A, B)
        est, v, w, nmults, nresamples = _onenormest_core(B, B.T, t, itmax)
        exact_value = scipy.linalg.norm(B, 1)
        underest_ratio = est / exact_value
        assert_allclose(underest_ratio, 0.05, rtol=1e-4)
        assert_equal(nmults, 11)
        assert_equal(nresamples, 0)
        # check the non-underscored version of onenormest
        est_plain = scipy.sparse.linalg.onenormest(B, t=t, itmax=itmax)
        assert_allclose(est, est_plain)

    def _help_product_norm_slow(self, A, B):
        # for profiling
        C = np.dot(A, B)
        return scipy.linalg.norm(C, 1)

    def _help_product_norm_fast(self, A, B):
        # for profiling
        t = 2
        itmax = 5
        D = MatrixProductOperator(A, B)
        est, v, w, nmults, nresamples = _onenormest_core(D, D.T, t, itmax)
        return est

    @decorators.slow
    def test_onenormest_linear_operator(self):
        # Define a matrix through its product A B.
        # Depending on the shapes of A and B,
        # it could be easy to multiply this product by a small matrix,
        # but it could be annoying to look at all of
        # the entries of the product explicitly.
        np.random.seed(1234)
        n = 6000
        k = 3
        A = np.random.randn(n, k)
        B = np.random.randn(k, n)
        fast_estimate = self._help_product_norm_fast(A, B)
        exact_value = self._help_product_norm_slow(A, B)
        self.assert_(
                fast_estimate <= exact_value <= 3*fast_estimate,
                'fast: %g\nexact:%g' % (fast_estimate, exact_value))

if __name__ == '__main__':
    run_module_suite()

<|endoftext|>


<commit_before>from   __future__ import unicode_literals
import linesep

try:
    from StringIO import StringIO
except ImportError:
    from io       import StringIO

# Based on <https://pytest.org/latest/example/parametrize.html#a-quick-port-of-testscenarios>
def pytest_generate_tests(metafunc):
    idlist = []
    argvalues = []
    for scenario in metafunc.module.scenarios:
        idlist.append(scenario[0])
        argvalues.append([scenario[1][argname] for argname in metafunc.fixturenames])
    metafunc.parametrize(metafunc.fixturenames, argvalues, ids=idlist, scope="module")

scenarios = [
    ('empty', {
        "text": '',
        "sep": '\n',
        "preceded": [],
        "terminated": [],
        "separated": [''],
        "preceded_retained": [],
        "terminated_retained": [],
        "separated_retained": [''],
    }),
    ('one_sep', {
        "text": '\n',
        "sep": '\n',
        "preceded": [''],
        "terminated": [''],
        "separated": ['', ''],
        "preceded_retained": ['\n'],
        "terminated_retained": ['\n'],
        "separated_retained": ['', '\n', ''],
    }),
]

def test_split_preceded(text, sep, preceded, **_):
    assert linesep.split_preceded(text, sep, retain=False) == preceded

def test_split_terminated(text, sep, terminated, **_):
    assert linesep.split_terminated(text, sep, retain=False) == terminated

def test_split_separated(text, sep, separated, **_):
    assert linesep.split_separated(text, sep, retain=False) == separated

def test_split_preceded_retained(text, sep, preceded_retained, **_):
    assert linesep.split_preceded(text, sep, retain=True) == preceded_retained

def test_split_terminated_retained(text, sep, terminated_retained, **_):
    assert linesep.split_terminated(text, sep, retain=True) == terminated_retained

def test_split_separated_retained(text, sep, separated_retained, **_):
    assert linesep.split_separated(text, sep, retain=True) == separated_retained

def test_read_preceded(text, sep, preceded, **_):
    assert list(linesep.read_preceded(StringIO(text), sep, retain=False)) == preceded

def test_read_terminated(text, sep, terminated, **_):
    assert list(linesep.read_terminated(StringIO(text), sep, retain=False)) == terminated

def test_read_separated(text, sep, separated, **_):
    assert list(linesep.read_separated(StringIO(text), sep, retain=False)) == separated

def test_read_preceded_retained(text, sep, preceded_retained, **_):
    assert list(linesep.read_preceded(StringIO(text), sep, retain=True)) == preceded_retained

def test_read_terminated_retained(text, sep, terminated_retained, **_):
    assert list(linesep.read_terminated(StringIO(text), sep, retain=True)) == terminated_retained

def test_read_separated_retained(text, sep, separated_retained, **_):
    assert list(linesep.read_separated(StringIO(text), sep, retain=True)) == separated_retained
<commit_msg>Another test<commit_after>from   __future__ import unicode_literals
import linesep

try:
    from StringIO import StringIO
except ImportError:
    from io       import StringIO

# Based on <https://pytest.org/latest/example/parametrize.html#a-quick-port-of-testscenarios>
def pytest_generate_tests(metafunc):
    idlist = []
    argvalues = []
    for scenario in metafunc.module.scenarios:
        idlist.append(scenario[0])
        argvalues.append([scenario[1][argname] for argname in metafunc.fixturenames])
    metafunc.parametrize(metafunc.fixturenames, argvalues, ids=idlist, scope="module")

scenarios = [

    ('empty', {
        "text": '',
        "sep": '\n',
        "preceded": [],
        "terminated": [],
        "separated": [''],
        "preceded_retained": [],
        "terminated_retained": [],
        "separated_retained": [''],
    }),

    ('one_sep', {
        "text": '\n',
        "sep": '\n',
        "preceded": [''],
        "terminated": [''],
        "separated": ['', ''],
        "preceded_retained": ['\n'],
        "terminated_retained": ['\n'],
        "separated_retained": ['', '\n', ''],
    }),

    ('text_sep_text', {
        "text": 'foo\nbar',
        "sep": '\n',
        "preceded": ['foo', 'bar'],
        "preceded_retained": ['foo', '\nbar'],
        "separated": ['foo', 'bar'],
        "separated_retained": ['foo', '\n', 'bar'],
        "terminated": ['foo', 'bar'],
        "terminated_retained": ['foo\n', 'bar']
    }),

]

def test_split_preceded(text, sep, preceded, **_):
    assert linesep.split_preceded(text, sep, retain=False) == preceded

def test_split_terminated(text, sep, terminated, **_):
    assert linesep.split_terminated(text, sep, retain=False) == terminated

def test_split_separated(text, sep, separated, **_):
    assert linesep.split_separated(text, sep, retain=False) == separated

def test_split_preceded_retained(text, sep, preceded_retained, **_):
    assert linesep.split_preceded(text, sep, retain=True) == preceded_retained

def test_split_terminated_retained(text, sep, terminated_retained, **_):
    assert linesep.split_terminated(text, sep, retain=True) == terminated_retained

def test_split_separated_retained(text, sep, separated_retained, **_):
    assert linesep.split_separated(text, sep, retain=True) == separated_retained

def test_read_preceded(text, sep, preceded, **_):
    assert list(linesep.read_preceded(StringIO(text), sep, retain=False)) == preceded

def test_read_terminated(text, sep, terminated, **_):
    assert list(linesep.read_terminated(StringIO(text), sep, retain=False)) == terminated

def test_read_separated(text, sep, separated, **_):
    assert list(linesep.read_separated(StringIO(text), sep, retain=False)) == separated

def test_read_preceded_retained(text, sep, preceded_retained, **_):
    assert list(linesep.read_preceded(StringIO(text), sep, retain=True)) == preceded_retained

def test_read_terminated_retained(text, sep, terminated_retained, **_):
    assert list(linesep.read_terminated(StringIO(text), sep, retain=True)) == terminated_retained

def test_read_separated_retained(text, sep, separated_retained, **_):
    assert list(linesep.read_separated(StringIO(text), sep, retain=True)) == separated_retained
<|endoftext|>


<commit_before>"""
Changes existing registered_meta on a node to new schema layout
required for the prereg-prize
"""
from nose.tools import *  # noqa PEP8 asserts
import json
import sys
import logging

from modularodm import Q

from framework.mongo import database as db
from framework.mongo.utils import from_mongo
from framework.transactions.context import TokuTransaction

from website.models import MetaSchema
from website.app import init_app
from website.project.model import ensure_schemas
from website.project.metadata.schemas import _id_to_name

from scripts import utils as scripts_utils

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

def prepare_nodes(_db=None):
    _db = _db or db
    _db['node'].update(
        {},
        {
            '$set': {
                'registered_schema': []
            }
        },
        multi=True
    )

def from_json_or_fail(schema):
    # Unstringify stored metadata
    try:
        schema = json.loads(schema) if schema else {}
    except TypeError as e:
        if isinstance(schema, dict):
            pass
        else:
            raise e
    return schema

def main(dev=False, _db=None):
    _db = _db or db
    init_app(routes=False)
    count = 0
    skipped = 0
    scripts_utils.add_file_logger(logger, __file__)
    logger.info("Iterating over all registrations")

    # nullify old registered_schema refs if set
    MetaSchema.remove(
        Q('schema_version', 'eq', 1)
    )
    ensure_schemas()
    # convert registered_schema to list field
    prepare_nodes()

    node_documents = _db['node'].find({'is_registration': True})
    for node in node_documents:
        registered_schemas = []
        registered_meta = {}
        schemas = node['registered_meta']
        if not schemas:
            logger.info('Node: {0} is registered but has no registered_meta'.format(node['_id']))
            continue
        for schema_id, schema in schemas.iteritems():
            name = _id_to_name(from_mongo(schema_id))
            schema = from_json_or_fail(schema)
            # append matching schema to node.registered_schema
            try:
                meta_schema = MetaSchema.find(
                    Q('name', 'eq', name)
                ).sort('-schema_version')[0]
            except IndexError as e:
                logger.error('No MetaSchema matching name: {0} found'.format(name))
                # Skip over missing schemas
                skipped += 1
                if dev:
                    continue
                else:
                    import ipdb; ipdb.set_trace()
                    raise e
            else:
                registered_meta[meta_schema._id] = {
                    key: {
                        'value': value
                    }
                    for key, value in schema.items()
                }
                registered_schemas.append(meta_schema._id)
        db['node'].update(
            {'_id': node['_id']},
            {'$set': {
                'registered_meta': registered_meta,
                'registered_schema': registered_schemas
            }}
        )
        count = count + 1
    logger.info('Done with {0} nodes migrated and {1} nodes skipped.'.format(count, skipped))

if __name__ == '__main__':
    dry_run = 'dry' in sys.argv
    dev = 'dev' in sys.argv
    with TokuTransaction():
        main(dev=dev)
        if dry_run:
            raise RuntimeError('Dry run, rolling back transaction.')
<commit_msg>No need to remove old MetaSchemas with new ensure_schemas def [skipci]<commit_after>"""
Changes existing registered_meta on a node to new schema layout
required for the prereg-prize
"""
from nose.tools import *  # noqa PEP8 asserts
import json
import sys
import logging

from modularodm import Q

from framework.mongo import database as db
from framework.mongo.utils import from_mongo
from framework.transactions.context import TokuTransaction

from website.models import MetaSchema
from website.app import init_app
from website.project.model import ensure_schemas
from website.project.metadata.schemas import _id_to_name

from scripts import utils as scripts_utils

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

def prepare_nodes(_db=None):
    _db = _db or db
    _db['node'].update(
        {},
        {
            '$set': {
                'registered_schema': []
            }
        },
        multi=True
    )

def from_json_or_fail(schema):
    # Unstringify stored metadata
    try:
        schema = json.loads(schema) if schema else {}
    except TypeError as e:
        if isinstance(schema, dict):
            pass
        else:
            raise e
    return schema

def main(dev=False, _db=None):
    _db = _db or db
    init_app(routes=False)
    count = 0
    skipped = 0
    scripts_utils.add_file_logger(logger, __file__)
    logger.info("Iterating over all registrations")

    # convert registered_schema to list field
    prepare_nodes()
    ensure_schemas()

    node_documents = _db['node'].find({'is_registration': True})
    for node in node_documents:
        registered_schemas = []
        registered_meta = {}
        schemas = node['registered_meta']
        if not schemas:
            logger.info('Node: {0} is registered but has no registered_meta'.format(node['_id']))
            continue
        for schema_id, schema in schemas.iteritems():
            name = _id_to_name(from_mongo(schema_id))
            schema = from_json_or_fail(schema)
            # append matching schema to node.registered_schema
            try:
                meta_schema = MetaSchema.find(
                    Q('name', 'eq', name)
                ).sort('-schema_version')[0]
            except IndexError as e:
                logger.error('No MetaSchema matching name: {0} found'.format(name))
                # Skip over missing schemas
                skipped += 1
                if dev:
                    continue
                else:
                    import ipdb; ipdb.set_trace()
                    raise e
            else:
                registered_meta[meta_schema._id] = {
                    key: {
                        'value': value
                    }
                    for key, value in schema.items()
                }
                registered_schemas.append(meta_schema._id)
        db['node'].update(
            {'_id': node['_id']},
            {'$set': {
                'registered_meta': registered_meta,
                'registered_schema': registered_schemas
            }}
        )
        count = count + 1
    logger.info('Done with {0} nodes migrated and {1} nodes skipped.'.format(count, skipped))

if __name__ == '__main__':
    dry_run = 'dry' in sys.argv
    dev = 'dev' in sys.argv
    with TokuTransaction():
        main(dev=dev)
        if dry_run:
            raise RuntimeError('Dry run, rolling back transaction.')
<|endoftext|>


<commit_before>import logging
import sqlalchemy
from sqlalchemy.orm import sessionmaker
from sqlalchemy.orm.exc import MultipleResultsFound, NoResultFound

from irma.common.exceptions import IrmaDatabaseError

log = logging.getLogger(__name__)

class SQLDatabase(object):
    """Internal database.

    This class handles the creation of the internal database and provides some
    functions for interacting with it.
    """

    ##########################################################################
    # Constructor and Destructor stuff
    ##########################################################################
    def __init__(self, engine):
        self._db = sqlalchemy.create_engine(engine)
        self._db.echo = False
        self._session = None
        self._connect()

    def __del__(self):
        self._disconnect()

    def __enter__(self):
        return self

    def __exit__(self, type, value, tb):
        self.__del__()

    ##########################################################################
    # Private methods
    ##########################################################################

    def _connect(self):
        Session = sessionmaker(bind=self._db)
        self._session = Session()

    def _disconnect(self):
        if self._session is None:
            return
        try:
            self._session.commit()
            self._session.close()
            self._session = None
        except Exception as e:
            raise IrmaDatabaseError("{0}".format(e))

    ##########################################################################
    # Public methods
    ##########################################################################
    def add(self, entry):
        self._session.add(entry)

    def add_all(self, entries):
        self._session.add_all(entries)

    def commit(self):
        self._session.commit()

    def rollback(self):
        self._session.rollback()

    def all(self, classname):
        return self._session.query(classname).all()

    def one_by(self, classname, **kwargs):
        try:
            return self._session.query(classname).filter_by(**kwargs).one()
        except NoResultFound:
            raise IrmaDatabaseError("No result found instead of one")
        except MultipleResultsFound:
            raise IrmaDatabaseError("Multiple results found instead of one")

    def one(self, classname, filter_expr):
        try:
            return self._session.query(classname).filter(filter_expr).one()
        except NoResultFound:
            raise IrmaDatabaseError("No result found instead of one")
        except MultipleResultsFound:
            raise IrmaDatabaseError("Multiple results found instead of one")

    def find_by(self, classname, **kwargs):
        return self._session.query(classname).filter_by(**kwargs).all()

    def find(self, classname, filter_expr):
        return self._session.query(classname).filter(filter_expr).all()

    def count(self, classname):
        return self._session.query(classname).count()

    def delete(self, obj):
        return self._session.delete(obj)

    def _sum(self, elts):
        res = 0
        for e in elts:
            res += e[0]
        return res

    def sum_by(self, field, **kwargs):
        t = self.find_by(field, **kwargs)
        return self._sum(t)

    def sum(self, field, filter_expr):
        t = self.find(field, filter_expr)
        return self._sum(t)
<commit_msg>removed close() for database<commit_after>import logging
import sqlalchemy
from sqlalchemy.orm import sessionmaker
from sqlalchemy.orm.exc import MultipleResultsFound, NoResultFound

from irma.common.exceptions import IrmaDatabaseError

log = logging.getLogger(__name__)

class SQLDatabase(object):
    """Internal database.

    This class handles the creation of the internal database and provides some
    functions for interacting with it.
    """

    ##########################################################################
    # Constructor and Destructor stuff
    ##########################################################################
    def __init__(self, engine):
        self._db = sqlalchemy.create_engine(engine)
        self._db.echo = False
        self._session = None
        self._connect()

    def __del__(self):
        self._disconnect()

    def __enter__(self):
        return self

    def __exit__(self, type, value, tb):
        self.__del__()

    ##########################################################################
    # Private methods
    ##########################################################################

    def _connect(self):
        Session = sessionmaker(bind=self._db)
        self._session = Session()

    def _disconnect(self):
        try:
            self._session.commit()
            # we do not close the session to keep objects persistent
        except Exception as e:
            raise IrmaDatabaseError("{0}".format(e))

    ##########################################################################
    # Public methods
    ##########################################################################
    def add(self, entry):
        self._session.add(entry)

    def add_all(self, entries):
        self._session.add_all(entries)

    def commit(self):
        self._session.commit()

    def rollback(self):
        self._session.rollback()

    def all(self, classname):
        return self._session.query(classname).all()

    def one_by(self, classname, **kwargs):
        try:
            return self._session.query(classname).filter_by(**kwargs).one()
        except NoResultFound:
            raise IrmaDatabaseError("No result found instead of one")
        except MultipleResultsFound:
            raise IrmaDatabaseError("Multiple results found instead of one")

    def one(self, classname, filter_expr):
        try:
            return self._session.query(classname).filter(filter_expr).one()
        except NoResultFound:
            raise IrmaDatabaseError("No result found instead of one")
        except MultipleResultsFound:
            raise IrmaDatabaseError("Multiple results found instead of one")

    def find_by(self, classname, **kwargs):
        return self._session.query(classname).filter_by(**kwargs).all()

    def find(self, classname, filter_expr):
        return self._session.query(classname).filter(filter_expr).all()

    def count(self, classname):
        return self._session.query(classname).count()

    def delete(self, obj):
        return self._session.delete(obj)

    def _sum(self, elts):
        res = 0
        for e in elts:
            res += e[0]
        return res

    def sum_by(self, field, **kwargs):
        t = self.find_by(field, **kwargs)
        return self._sum(t)

    def sum(self, field, filter_expr):
        t = self.find(field, filter_expr)
        return self._sum(t)
<|endoftext|>


<commit_before>def short_description(description):
    """
    Sets 'short_description' attribute (this attribute is used by list_display and formular).
    """
    def decorator(func):
        func.short_description = description
        return func
    return decorator

def filter_class(filter_class):
    """
    Sets 'filter' class (this attribute is used inside grid and rest).
    """
    def decorator(func):
        func.filter = filter_class
        return func
    return decorator

def order_by(field_name):
    """
    Sets 'field name' (this is used for grid ordering)
    """
    def decorator(func):
        func.filter = filter_class
        return func
    return decorator

<commit_msg>v1.3.32 fixed decorators<commit_after>def short_description(description):
    """
    Sets 'short_description' attribute (this attribute is used by list_display and formular).
    """
    def decorator(func):
        func.short_description = description
        return func
    return decorator

def filter_class(filter_class):
    """
    Sets 'filter' class (this attribute is used inside grid and rest).
    """
    def decorator(func):
        func.filter = filter_class
        return func
    return decorator

def order_by(field_name):
    """
    Sets 'field name' (this is used for grid ordering)
    """
    def decorator(func):
        func.order_by = field_name
        return func
    return decorator

<|endoftext|>


<commit_before>import unittest

from hypothesis import given, assume
from segpy.util import batched

class TestBatched(unittest.TestCase):

    @given([int], int)
    def test_batch_sizes_unpadded(self, items, batch_size):
        assume(batch_size > 0)
        batches = list(batched(items, batch_size))
        self.assertTrue(all(len(batch) == batch_size for batch in batches[:-1]))

    @given([int], int)
    def test_final_batch_sizes(self, items, batch_size):
        assume(len(items) > 0)
        assume(batch_size > 0)
        batches = list(batched(items, batch_size))
        self.assertTrue(len(batches[-1]) <= batch_size)

    @given([int], int, int)
    def test_batch_sizes_padded(self, items, batch_size, pad):
        assume(batch_size > 0)
        batches = list(batched(items, batch_size, padding=pad))
        self.assertTrue(all(len(batch) == batch_size for batch in batches))

    @given([int], int, int)
    def test_pad_contents(self, items, batch_size, pad):
        assume(len(items) > 0)
        assume(0 < batch_size < 1000)
        num_left_over = len(items) % batch_size
        pad_length = batch_size - num_left_over if num_left_over != 0 else 0
        assume(pad_length != 0)
        batches = list(batched(items, batch_size, padding=pad))
        self.assertEqual(batches[-1][batch_size - pad_length:], [pad] * pad_length)

    def test_pad(self):
        batches = list(batched([0, 0], 3, 42))
        self.assertEqual(batches[-1], [0, 0, 42])

if __name__ == '__main__':
    unittest.main()<commit_msg>Use integers_in_range to set the batch size rather than simply int to prevent colossal memory allocations.<commit_after>import unittest

from hypothesis import given, assume
from hypothesis.descriptors import integers_in_range
from segpy.util import batched

class TestBatched(unittest.TestCase):

    @given([int],
           integers_in_range(1, 1000))
    def test_batch_sizes_unpadded(self, items, batch_size):
        assume(batch_size > 0)
        batches = list(batched(items, batch_size))
        self.assertTrue(all(len(batch) == batch_size for batch in batches[:-1]))

    @given([int],
            integers_in_range(1, 1000))
    def test_final_batch_sizes(self, items, batch_size):
        assume(len(items) > 0)
        assume(batch_size > 0)
        batches = list(batched(items, batch_size))
        self.assertTrue(len(batches[-1]) <= batch_size)

    @given([int],
           integers_in_range(1, 1000),
           int)
    def test_batch_sizes_padded(self, items, batch_size, pad):
        assume(batch_size > 0)
        batches = list(batched(items, batch_size, padding=pad))
        self.assertTrue(all(len(batch) == batch_size for batch in batches))

    @given([int],
           integers_in_range(1, 1000),
           int)
    def test_pad_contents(self, items, batch_size, pad):
        assume(len(items) > 0)
        assume(0 < batch_size < 1000)
        num_left_over = len(items) % batch_size
        pad_length = batch_size - num_left_over if num_left_over != 0 else 0
        assume(pad_length != 0)
        batches = list(batched(items, batch_size, padding=pad))
        self.assertEqual(batches[-1][batch_size - pad_length:], [pad] * pad_length)

    def test_pad(self):
        batches = list(batched([0, 0], 3, 42))
        self.assertEqual(batches[-1], [0, 0, 42])

if __name__ == '__main__':
    unittest.main()<|endoftext|>


<commit_before># pyOCD debugger
# Copyright (c) 2015-2019 Arm Limited
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function

import logging
import os
import sys
import traceback
from xml.etree import ElementTree
import six
import subprocess
import tempfile
from pyocd.utility.compatibility import to_str_safe

isPy2 = (sys.version_info[0] == 2)

OBJCOPY = "arm-none-eabi-objcopy"

# Returns common option values passed in when creating test sessions.
def get_session_options():
    return {
        'frequency' : 1000000, # 1 MHz
        }

# Returns a dict containing some test parameters for the target in the passed-in session.
#
# 'test_clock' : the max supported SWD frequency for the target
# 'error_on_invalid_access' : whether invalid accesses cause a fault
#
def get_target_test_params(session):
    target_type = session.board.target_type
    error_on_invalid_access = True
    if target_type in ("nrf51", "nrf52", "nrf52840"):
        # Override clock since 10MHz is too fast
        test_clock = 1000000
        error_on_invalid_access = False
    elif target_type == "ncs36510":
        # Override clock since 10MHz is too fast
        test_clock = 1000000
    else:
        # Default of 10 MHz. Most probes will not actually run this fast, but this
        # sets them to their max supported frequency.
        test_clock = 10000000
    return {
            'test_clock': test_clock,
            'error_on_invalid_access': error_on_invalid_access,
            }

# Generate an Intel hex file from the binary test file.
def binary_to_hex_file(binary_file, base_address):
    temp_test_hex_name = tempfile.mktemp('.elf')
    objcopyOutput = subprocess.check_output([OBJCOPY,
        "-v", "-I", "binary", "-O", "ihex", "-B", "arm", "-S",
        "--set-start", "0x%x" % base_address,
        "--change-addresses", "0x%x" % base_address,
        binary_file, temp_test_hex_name], stderr=subprocess.STDOUT)
    print(to_str_safe(objcopyOutput))
    # Need to escape backslashes on Windows.
    if sys.platform.startswith('win'):
        temp_test_hex_name = temp_test_hex_name.replace('\\', '\\\\')
    return temp_test_hex_name

# Generate an elf from the binary test file.
def binary_to_elf_file(binary_file, base_address):
    temp_test_elf_name = tempfile.mktemp('.elf')
    objcopyOutput = subprocess.check_output([OBJCOPY,
        "-v", "-I", "binary", "-O", "elf32-littlearm", "-B", "arm", "-S",
        "--set-start", "0x%x" % base_address,
        "--change-addresses", "0x%x" % base_address,
        binary_file, temp_test_elf_name], stderr=subprocess.STDOUT)
    print(to_str_safe(objcopyOutput))
    # Need to escape backslashes on Windows.
    if sys.platform.startswith('win'):
        temp_test_elf_name = temp_test_elf_name.replace('\\', '\\\\')
    return temp_test_elf_name

class IOTee(object):
    def __init__(self, *args):
        self.outputs = list(args)

    def add(self, output):
        self.outputs.append(output)

    def write(self, message):
        if isPy2 and isinstance(message, str):
            message = message.decode('UTF-8')
        for out in self.outputs:
            out.write(message)

    def flush(self):
        for out in self.outputs:
            out.flush()

class RecordingLogHandler(logging.Handler):
    def __init__(self, iostream, level=logging.NOTSET):
        super(RecordingLogHandler, self).__init__(level)
        self.stream = iostream

    def emit(self, record):
        try:
            message = self.format(record)
            if isPy2 and isinstance(message, unicode):
                message = message.encode('UTF-8')
            self.stream.write(six.u(message + "\n"))
        except:
            self.handleError(record)

class TestResult(object):

    def __init__(self, test_board, test, result):
        self.passed = result
        self._board = test_board.target_type if test_board else 'unknown'
        self.board_name = test_board.name if test_board else ""
        self.test = test
        self.name = "test"
        self.time = 0
        self.output = ""

    @property
    def board(self):
        return self._board

    @board.setter
    def board(self, newBoard):
        self._board = newBoard.target_type if newBoard else 'unknown'
        self.board_name = newBoard.name

    def get_test_case(self):
        case = ElementTree.Element('testcase',
                    name=self.name,
                    classname="{}.{}.{}".format(self.board_name, self.board, self.name),
                    status=("passed" if self.passed else "failed"),
                    time="%.3f" % self.time
                    )
        case.text = "\n"
        case.tail = "\n"
        if not self.passed:
            failed = ElementTree.SubElement(case, 'failure',
                        message="failure",
                        type="failure"
                        )
        system_out = ElementTree.SubElement(case, 'system-out')
        system_out.text = self.output
        return case

class Test(object):

    def __init__(self, name, function):
        self.name = name
        self.test_function = function

    def run(self, board):
        """
        Run test and return the result

        Override this function to return a custom result
        """
        passed = False
        try:
            self.test_function(board.unique_id)
            passed = True
        except Exception as e:
            print("Exception %s when testing board %s" % (e, board.unique_id))
            traceback.print_exc(file=sys.stdout)
        result = TestResult(board, self, passed)
        result.name = self.name
        return result

    def print_perf_info(self, result_list, output_file=None):
        """
        Print performance info if any
        """
        pass

    @staticmethod
    def print_results(result_list, output_file=None):
        msg_format_str = "{:<15}{:<21}{:<15}{:<15}"
        print("\n\n------ TEST RESULTS ------")
        print(msg_format_str .format("Target", "Test", "Result", "Time"),
              file=output_file)
        print("", file=output_file)
        for result in result_list:
            status_str = "Pass" if result.passed else "Fail"
            print(msg_format_str.format(result.board,
                                        result.test.name,
                                        status_str, "%.3f" % result.time),
                  file=output_file)

    @staticmethod
    def all_tests_pass(result_list):
        passed = True
        for result in result_list:
            if not result.passed:
                passed = False
                break
        if len(result_list) <= 0:
            passed = False
        return passed

<commit_msg>Added tox env name to xml test results test class name.<commit_after># pyOCD debugger
# Copyright (c) 2015-2019 Arm Limited
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function

import logging
import os
import sys
import traceback
from xml.etree import ElementTree
import six
import subprocess
import tempfile
from pyocd.utility.compatibility import to_str_safe

isPy2 = (sys.version_info[0] == 2)

OBJCOPY = "arm-none-eabi-objcopy"

# Returns common option values passed in when creating test sessions.
def get_session_options():
    return {
        'frequency' : 1000000, # 1 MHz
        }

# Returns a dict containing some test parameters for the target in the passed-in session.
#
# 'test_clock' : the max supported SWD frequency for the target
# 'error_on_invalid_access' : whether invalid accesses cause a fault
#
def get_target_test_params(session):
    target_type = session.board.target_type
    error_on_invalid_access = True
    if target_type in ("nrf51", "nrf52", "nrf52840"):
        # Override clock since 10MHz is too fast
        test_clock = 1000000
        error_on_invalid_access = False
    elif target_type == "ncs36510":
        # Override clock since 10MHz is too fast
        test_clock = 1000000
    else:
        # Default of 10 MHz. Most probes will not actually run this fast, but this
        # sets them to their max supported frequency.
        test_clock = 10000000
    return {
            'test_clock': test_clock,
            'error_on_invalid_access': error_on_invalid_access,
            }

# Generate an Intel hex file from the binary test file.
def binary_to_hex_file(binary_file, base_address):
    temp_test_hex_name = tempfile.mktemp('.elf')
    objcopyOutput = subprocess.check_output([OBJCOPY,
        "-v", "-I", "binary", "-O", "ihex", "-B", "arm", "-S",
        "--set-start", "0x%x" % base_address,
        "--change-addresses", "0x%x" % base_address,
        binary_file, temp_test_hex_name], stderr=subprocess.STDOUT)
    print(to_str_safe(objcopyOutput))
    # Need to escape backslashes on Windows.
    if sys.platform.startswith('win'):
        temp_test_hex_name = temp_test_hex_name.replace('\\', '\\\\')
    return temp_test_hex_name

# Generate an elf from the binary test file.
def binary_to_elf_file(binary_file, base_address):
    temp_test_elf_name = tempfile.mktemp('.elf')
    objcopyOutput = subprocess.check_output([OBJCOPY,
        "-v", "-I", "binary", "-O", "elf32-littlearm", "-B", "arm", "-S",
        "--set-start", "0x%x" % base_address,
        "--change-addresses", "0x%x" % base_address,
        binary_file, temp_test_elf_name], stderr=subprocess.STDOUT)
    print(to_str_safe(objcopyOutput))
    # Need to escape backslashes on Windows.
    if sys.platform.startswith('win'):
        temp_test_elf_name = temp_test_elf_name.replace('\\', '\\\\')
    return temp_test_elf_name

class IOTee(object):
    def __init__(self, *args):
        self.outputs = list(args)

    def add(self, output):
        self.outputs.append(output)

    def write(self, message):
        if isPy2 and isinstance(message, str):
            message = message.decode('UTF-8')
        for out in self.outputs:
            out.write(message)

    def flush(self):
        for out in self.outputs:
            out.flush()

class RecordingLogHandler(logging.Handler):
    def __init__(self, iostream, level=logging.NOTSET):
        super(RecordingLogHandler, self).__init__(level)
        self.stream = iostream

    def emit(self, record):
        try:
            message = self.format(record)
            if isPy2 and isinstance(message, unicode):
                message = message.encode('UTF-8')
            self.stream.write(six.u(message + "\n"))
        except:
            self.handleError(record)

class TestResult(object):

    def __init__(self, test_board, test, result):
        self.passed = result
        self._board = test_board.target_type if test_board else 'unknown'
        self.board_name = test_board.name if test_board else ""
        self.test = test
        self.name = "test"
        self.time = 0
        self.output = ""

    @property
    def board(self):
        return self._board

    @board.setter
    def board(self, newBoard):
        self._board = newBoard.target_type if newBoard else 'unknown'
        self.board_name = newBoard.name

    def get_test_case(self):
        if 'TOX_ENV_NAME' in os.environ:
            classname = "{}.{}.{}.{}".format(os.environ['TOX_ENV_NAME'], self.board_name, self.board, self.name)
        else:
            classname = "{}.{}.{}".format(self.board_name, self.board, self.name)
        case = ElementTree.Element('testcase',
                    name=self.name,
                    classname=classname,
                    status=("passed" if self.passed else "failed"),
                    time="%.3f" % self.time
                    )
        case.text = "\n"
        case.tail = "\n"
        if not self.passed:
            failed = ElementTree.SubElement(case, 'failure',
                        message="failure",
                        type="failure"
                        )
        system_out = ElementTree.SubElement(case, 'system-out')
        system_out.text = self.output
        return case

class Test(object):

    def __init__(self, name, function):
        self.name = name
        self.test_function = function

    def run(self, board):
        """
        Run test and return the result

        Override this function to return a custom result
        """
        passed = False
        try:
            self.test_function(board.unique_id)
            passed = True
        except Exception as e:
            print("Exception %s when testing board %s" % (e, board.unique_id))
            traceback.print_exc(file=sys.stdout)
        result = TestResult(board, self, passed)
        result.name = self.name
        return result

    def print_perf_info(self, result_list, output_file=None):
        """
        Print performance info if any
        """
        pass

    @staticmethod
    def print_results(result_list, output_file=None):
        msg_format_str = "{:<15}{:<21}{:<15}{:<15}"
        print("\n\n------ TEST RESULTS ------")
        print(msg_format_str .format("Target", "Test", "Result", "Time"),
              file=output_file)
        print("", file=output_file)
        for result in result_list:
            status_str = "Pass" if result.passed else "Fail"
            print(msg_format_str.format(result.board,
                                        result.test.name,
                                        status_str, "%.3f" % result.time),
                  file=output_file)

    @staticmethod
    def all_tests_pass(result_list):
        passed = True
        for result in result_list:
            if not result.passed:
                passed = False
                break
        if len(result_list) <= 0:
            passed = False
        return passed

<|endoftext|>


<commit_before># vim:nowrap:textwidth=0
#
# Sheared -- non-blocking network programming library for Python
# Copyright (C) 2003  Sune Kirkeby <sune@mel.interspace.dk>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#

import unittest, os, random, signal, time

from sheared import reactor
from sheared import error
from sheared.protocol import http
from sheared.web import server, subserver, virtualhost, resource
from sheared.web.collections.collection import Collection
from sheared.web.server.request import HTTPRequest
from sheared.python import commands

from tests import transport

class FakeRequest(HTTPRequest):
    def __init__(self, requestline='GET / HTTP/1.0', headers='', body=''):
        requestline = http.HTTPRequestLine(requestline)
        headers = http.HTTPHeaders(headers)
        HTTPRequest.__init__(self, requestline, headers, body)

class FakeReply:
    def __init__(self, head_only=0):
        self.headers = http.HTTPHeaders()
        self.sent = ''
        self.status = 200
        self.head_only = head_only

    def setStatusCode(self, code):
        self.status = code

    def send(self, data):
        self.sent = self.sent + data
    def sendfile(self, file):
        self.send(file.read())
    def done(self):
        self.done = 1

class FakeResource(resource.GettableResource):
    def __init__(self, name='foo'):
        resource.GettableResource.__init__(self)
        self.name = name

    def handle(self, request, reply, subpath):
        content = """Welcome to %s!\r\n""" % self.name
        last_mod = http.HTTPDateTime(300229200)

        reply.headers.setHeader('Content-Type', 'text/plain')
        reply.headers.setHeader('Content-Length', len(content))
        reply.headers.setHeader('Last-Modified', last_mod)
        reply.headers.setHeader('ETag', 'abc')

        if not request.head_only:
            if request.headers.has_key('If-None-Match'):
                if request.headers['If-None-Match'] == 'abc':
                    raise error.web.NotModified

            if request.headers.has_key('If-Modified-Since'):
                when = http.HTTPDateTime(request.headers['If-Modified-Since'])
                if not last_mod > when:
                    raise error.web.NotModified

        reply.send(content)
        reply.done()

class SimpleCollection(Collection):
    def __init__(self, name):
        resource.GettableResource.__init__(self)
        self.name = name
        self.resource = FakeResource(name)

    def getChild(self, request, reply, subpath):
        if subpath == '':
            return self.resource
        elif subpath == 'moved':
            reply.headers.setHeader('Location', '/')
            raise error.web.MovedPermanently
        elif subpath == 'abuse-me':
            reply.headers.setHeader('Foo', 'fubar')
            raise error.web.ForbiddenError, 'Sod off, cretin!'
        else:
            raise error.web.NotFoundError

def parseReply(reply):
    try:
        headers, body = reply.split('\r\n\r\n', 1)
        status, headers = headers.split('\r\n', 1)
        status = http.HTTPStatusLine(status)
        headers = http.HTTPHeaders(headers)

    except ValueError:
        raise 'Bad reply: %r' % reply

    return status, headers, body
<commit_msg>Add test-case for posts.<commit_after># vim:nowrap:textwidth=0
#
# Sheared -- non-blocking network programming library for Python
# Copyright (C) 2003  Sune Kirkeby <sune@mel.interspace.dk>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#

import unittest, os, random, signal, time

from sheared import reactor
from sheared import error
from sheared.protocol import http
from sheared.web import server, subserver, virtualhost, resource
from sheared.web.collections.collection import Collection
from sheared.web.server.request import HTTPRequest
from sheared.python import commands

from tests import transport

class FakeRequest(HTTPRequest):
    def __init__(self, requestline='GET / HTTP/1.0', headers='', body=''):
        requestline = http.HTTPRequestLine(requestline)
        headers = http.HTTPHeaders(headers)
        HTTPRequest.__init__(self, requestline, headers, body)

class FakeReply:
    def __init__(self, head_only=0):
        self.headers = http.HTTPHeaders()
        self.sent = ''
        self.status = 200
        self.head_only = head_only

    def setStatusCode(self, code):
        self.status = code

    def send(self, data):
        self.sent = self.sent + data
    def sendfile(self, file):
        self.send(file.read())
    def done(self):
        self.done = 1

class FakeResource(resource.NormalResource):
    def __init__(self, name='foo'):
        resource.NormalResource.__init__(self)
        self.name = name

    def handle(self, request, reply, subpath):
        content = """Welcome to %s!\r\n""" % self.name
        last_mod = http.HTTPDateTime(300229200)

        reply.headers.setHeader('Content-Type', 'text/plain')
        reply.headers.setHeader('Content-Length', len(content))
        reply.headers.setHeader('Last-Modified', last_mod)
        reply.headers.setHeader('ETag', 'abc')

        if not request.head_only:
            if request.headers.has_key('If-None-Match'):
                if request.headers['If-None-Match'] == 'abc':
                    raise error.web.NotModified

            if request.headers.has_key('If-Modified-Since'):
                when = http.HTTPDateTime(request.headers['If-Modified-Since'])
                if not last_mod > when:
                    raise error.web.NotModified

        reply.send(content)
        reply.done()

    def authenticate(self, request, reply):
        pass

class SimpleCollection(Collection):
    def __init__(self, name):
        resource.GettableResource.__init__(self)
        self.name = name
        self.resource = FakeResource(name)

    def getChild(self, request, reply, subpath):
        if subpath == '':
            return self.resource
        elif subpath == 'moved':
            reply.headers.setHeader('Location', '/')
            raise error.web.MovedPermanently
        elif subpath == 'abuse-me':
            reply.headers.setHeader('Foo', 'fubar')
            raise error.web.ForbiddenError, 'Sod off, cretin!'
        elif subpath == 'post':
            return self.resource
        else:
            raise error.web.NotFoundError

def parseReply(reply):
    try:
        headers, body = reply.split('\r\n\r\n', 1)
        status, headers = headers.split('\r\n', 1)
        status = http.HTTPStatusLine(status)
        headers = http.HTTPHeaders(headers)

    except ValueError:
        raise 'Bad reply: %r' % reply

    return status, headers, body
<|endoftext|>


<commit_before>#!/usr/bin/python
#
# Copyright (c) 2010 rPath, Inc.
#
# This program is distributed under the terms of the MIT License as found
# in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/mit-license.php.
#
# This program is distributed in the hope that it will be useful, but
# without any waranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the MIT License for full details.
#
import os
import sys
import unittest

import bootstrap

from testrunner import pathManager, testhelp

EXCLUDED_PATHS = ['scripts/.*', 'epdb.py', 'stackutil.py', 'test/.*']

def setup():
    xobjPath = pathManager.addExecPath('XOBJ_PATH')
    robjPath = pathManager.addExecPath('ROBJ_PATH')
    pathManager.addResourcePath('TEST_PATH', robjPath + '/test')

def main(argv=None, individual=True):
    if argv is None:
        argv = list(sys.argv)

    from conary.lib import util
    from conary.lib import coveragehook
    sys.excepthook = util.genExcepthook(True, catchSIGUSR1=False)

    handlerClass = testhelp.getHandlerClass(testhelp.ConaryTestSuite,
            lambda handler, environ: os.getenv('ROBJ_PATH'),
            lambda handler, environ: EXCLUDED_PATHS)

    handler = handlerClass(individual=individual)
    results = handler.main(argv)
    return results.getExitCode()

class TestCase(testhelp.TestCase):
    def setUp(self):
        testhelp.TestCase.setUp(self)

        from robj.lib import log
        log.setupLogging()

        import testserver
        ports = testhelp.findPorts(num=1, closeSockets=True)
        self.server = testserver.ThreadServer(port=ports[0])

    def tearDown(self):
        self.server.shutdown()

if __name__ == '__main__':
    setup()
    sys.exit(main(sys.argv, individual=False))
<commit_msg>add some helper methods and archive handling<commit_after>#!/usr/bin/python
#
# Copyright (c) 2010 rPath, Inc.
#
# This program is distributed under the terms of the MIT License as found
# in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/mit-license.php.
#
# This program is distributed in the hope that it will be useful, but
# without any waranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the MIT License for full details.
#
import os
import sys
import unittest

import bootstrap

from testrunner import pathManager, testhelp

EXCLUDED_PATHS = ['scripts/.*', 'epdb.py', 'stackutil.py', 'test/.*']

def setup():
    xobjPath = pathManager.addExecPath('XOBJ_PATH')
    robjPath = pathManager.addExecPath('ROBJ_PATH')

    pathManager.addResourcePath('TEST_PATH', os.path.join(robjPath, 'test'))
    pathManager.addResourcePath('ROBJ_ARCHIVE_PATH',
        path=os.path.join(robjPath, 'test', 'archive'))

def main(argv=None, individual=True):
    if argv is None:
        argv = list(sys.argv)

    from conary.lib import util
    from conary.lib import coveragehook
    sys.excepthook = util.genExcepthook(True, catchSIGUSR1=False)

    handlerClass = testhelp.getHandlerClass(testhelp.ConaryTestSuite,
            lambda handler, environ: os.getenv('ROBJ_PATH'),
            lambda handler, environ: EXCLUDED_PATHS)

    handler = handlerClass(individual=individual)
    results = handler.main(argv)
    return results.getExitCode()

class TestCase(testhelp.TestCase):
    def getArchiveContents(self, filename):
        path = os.path.join(self.archivePath, filename)
        return open(path).read()

    def getResponse(self, uri):
        cclass, pathvars = self.server.controllers.get(uri)
        controller = cclass(self.server.data, None, pathvars)
        response = controller.do_GET()
        return response

    def getModel(self, uri):
        response = self.getResponse(uri)
        return response.model

    def getXML(self, uri):
        response = self.getResponse(uri)
        return response.message

    def setUp(self):
        testhelp.TestCase.setUp(self)

        self.archivePath = pathManager.getPath('ROBJ_ARCHIVE_PATH')

        from robj.lib import log
        log.setupLogging()

        import testserver
        ports = testhelp.findPorts(num=1, closeSockets=True)
        self.server = testserver.ThreadServer(port=ports[0])

    def tearDown(self):
        self.server.shutdown()

if __name__ == '__main__':
    setup()
    sys.exit(main(sys.argv, individual=False))
<|endoftext|>


<commit_before>import sys
import unittest
import os
import tempfile
import numpy as NP
from numpy.testing import assert_array_equal, assert_array_almost_equal
from numpy.random.mtrand import uniform
import netCDF4

# test primitive data types.

# create an n1dim by n2dim random ranarr.
FILE_NAME = tempfile.mktemp(".nc")
n1dim = 5
n2dim = 10
ranarr = 100.*uniform(size=(n1dim,n2dim))
zlib=False;complevel=0;shuffle=0;least_significant_digit=None
datatypes = ['f8','f4','i1','i2','i4','i8','u1','u2','u4','u8','S1']
FillValue = 1.0

class PrimitiveTypesTestCase(unittest.TestCase):

    def setUp(self):
        self.file = FILE_NAME
        file = netCDF4.Dataset(self.file,'w')
        file.createDimension('n1', None)
        file.createDimension('n2', n2dim)
        for typ in datatypes:
            foo = file.createVariable('data_'+typ, typ, ('n1','n2',),zlib=zlib,complevel=complevel,shuffle=shuffle,least_significant_digit=least_significant_digit,fill_value=FillValue)
            #foo._FillValue = FillValue
            # test writing of _FillValue attribute for diff types
            # (should be cast to type of variable silently)
            foo[1:n1dim] = ranarr[1:n1dim]
        v = file.createVariable('issue271', NP.dtype('S1'), [], fill_value=b'Z')
        file.close()

    def tearDown(self):
        # Remove the temporary files
        os.remove(self.file)

    def runTest(self):
        """testing primitive data type """
        file = netCDF4.Dataset(self.file)
        for typ in datatypes:
            data = file.variables['data_'+typ]
            data.set_auto_maskandscale(False)
            datarr = data[1:n1dim]
            # fill missing data with _FillValue
            # ('S1' array will have some missing values)
            if hasattr(datarr, 'mask'):
                datarr = datarr.filled()
            datfilled = data[0]
            # check to see that data type is correct
            if typ == 'S1':
                self.assertTrue(data.dtype.str[1:] in ['S1','U1'])
            else:
                self.assertTrue(data.dtype.str[1:] == typ)
            # check data in variable.
            if data.dtype.str[1:] != 'S1':
                #assert NP.allclose(datarr, ranarr[1:n1dim].astype(data.dtype))
                assert_array_almost_equal(datarr,ranarr[1:n1dim].astype(data.dtype))
            else:
                assert datarr.tostring() == ranarr[1:n1dim].astype(data.dtype).tostring()
            # check that variable elements not yet written are filled
            # with the specified _FillValue.
            assert_array_equal(datfilled,NP.asarray(data._FillValue,datfilled.dtype))
        # issue 271 (_FillValue should be a byte for character arrays on
        # Python 3)
        v = file.variables['issue271']
        assert(v._FillValue == b'Z')
        file.close()

if __name__ == '__main__':
    unittest.main()
<commit_msg>add test for issue 273<commit_after>import sys
import unittest
import os
import tempfile
import numpy as NP
from numpy.testing import assert_array_equal, assert_array_almost_equal
from numpy.random.mtrand import uniform
import netCDF4

# test primitive data types.

# create an n1dim by n2dim random ranarr.
FILE_NAME = tempfile.mktemp(".nc")
n1dim = 5
n2dim = 10
ranarr = 100.*uniform(size=(n1dim,n2dim))
zlib=False;complevel=0;shuffle=0;least_significant_digit=None
datatypes = ['f8','f4','i1','i2','i4','i8','u1','u2','u4','u8','S1']
FillValue = 1.0
issue273_data = NP.ma.array(['z']*10,dtype='S1',\
mask=[False,False,False,False,False,True,False,False,False,False])

class PrimitiveTypesTestCase(unittest.TestCase):

    def setUp(self):
        self.file = FILE_NAME
        file = netCDF4.Dataset(self.file,'w')
        file.createDimension('n1', None)
        file.createDimension('n2', n2dim)
        for typ in datatypes:
            foo = file.createVariable('data_'+typ, typ, ('n1','n2',),zlib=zlib,complevel=complevel,shuffle=shuffle,least_significant_digit=least_significant_digit,fill_value=FillValue)
            #foo._FillValue = FillValue
            # test writing of _FillValue attribute for diff types
            # (should be cast to type of variable silently)
            foo[1:n1dim] = ranarr[1:n1dim]
        v = file.createVariable('issue271', NP.dtype('S1'), [], fill_value=b'Z')
        v2 = file.createVariable('issue273', NP.dtype('S1'), 'n2',\
                fill_value='\x00')
        v2[:] = issue273_data
        file.close()

    def tearDown(self):
        # Remove the temporary files
        os.remove(self.file)

    def runTest(self):
        """testing primitive data type """
        file = netCDF4.Dataset(self.file)
        for typ in datatypes:
            data = file.variables['data_'+typ]
            data.set_auto_maskandscale(False)
            datarr = data[1:n1dim]
            # fill missing data with _FillValue
            # ('S1' array will have some missing values)
            if hasattr(datarr, 'mask'):
                datarr = datarr.filled()
            datfilled = data[0]
            # check to see that data type is correct
            if typ == 'S1':
                self.assertTrue(data.dtype.str[1:] in ['S1','U1'])
            else:
                self.assertTrue(data.dtype.str[1:] == typ)
            # check data in variable.
            if data.dtype.str[1:] != 'S1':
                #assert NP.allclose(datarr, ranarr[1:n1dim].astype(data.dtype))
                assert_array_almost_equal(datarr,ranarr[1:n1dim].astype(data.dtype))
            else:
                assert datarr.tostring() == ranarr[1:n1dim].astype(data.dtype).tostring()
            # check that variable elements not yet written are filled
            # with the specified _FillValue.
            assert_array_equal(datfilled,NP.asarray(data._FillValue,datfilled.dtype))
        # issue 271 (_FillValue should be a byte for character arrays on
        # Python 3)
        v = file.variables['issue271']
        if type(v._FillValue) == bytes:
            assert(v._FillValue == b'Z') # python 3
        else:
            assert(v._FillValue == u'Z') # python 2
        # issue 273 (setting _FillValue to null byte manually)
        v2 = file.variables['issue273']
        if type(v2._FillValue) == bytes:
            assert(v2._FillValue == b'\x00') # python 3
        else:
            assert(v2._FillValue == u'') # python 2
        assert(str(issue273_data) == str(v2[:]))
        file.close()

if __name__ == '__main__':
    unittest.main()
<|endoftext|>