<commit_before>#  Copyright 2008 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

from robot import utils
from robot.common import BaseTestSuite, BaseTestCase
from robot.parsing import TestSuiteData
from robot.errors import ExecutionFailed
from robot.variables import GLOBAL_VARIABLES

from fixture import Setup, Teardown
from timeouts import TestTimeout
from keywords import KeywordFactory
from namespace import Namespace
from userkeyword import UserLibrary

def TestSuite(datasources, settings, syslog):
    suitedata = TestSuiteData(datasources, settings, syslog)
    suite = RunnableTestSuite(suitedata)
    suite.set_options(settings)
    return suite

class RunnableTestSuite(BaseTestSuite):

    def __init__(self, suitedata, parentdatas=[]):
        parentdatas = parentdatas[:] + [suitedata]
        BaseTestSuite.__init__(self, suitedata.name, suitedata.source)
        self.variables = GLOBAL_VARIABLES.copy()
        self.variables.set_from_variable_table(suitedata.variables)
        self.doc = suitedata.doc is not None and suitedata.doc or ''
        self.metadata = suitedata.metadata
        self.imports = suitedata.imports
        self.user_keywords = UserLibrary(suitedata.user_keywords)
        self.setup = utils.get_not_none(suitedata.suite_setup, [])
        self.teardown = utils.get_not_none(suitedata.suite_teardown, [])
        self.suites = [ RunnableTestSuite(suite, parentdatas)
                        for suite in suitedata.suites ]
        self.tests = [ RunnableTestCase(test, parentdatas)
                       for test in suitedata.tests ]
        if self.name == '':   # suitedata was multisource suite
            self.name = ' & '.join([suite.name for suite in self.suites])
        self.message = ''
        self._exit_on_failure = False

    def run(self, output, parent=None, error=None):
        self.starttime = utils.get_timestamp()
        self.namespace = Namespace(self, parent, output.syslog)
        self.namespace.variables['${SUITE_NAME}'] = self.longname
        init_err = self._init_suite(self.namespace.variables)
        output.start_suite(self)
        self.status = 'RUNNING'
        setup_err = self._run_fixture(self.setup, output, self.namespace, error, init_err)
        child_err = self._get_child_error(error, init_err, setup_err)
        for suite in self.suites:
            suite.run(output, self, child_err)
            if self._exit_on_failure and not child_err and suite.critical_stats.failed:
                child_err = 'Critical failure occurred and ExitOnFailure option is in use'
        for test in self.tests:
            test.run(output, self.namespace, child_err)
            if self._exit_on_failure and not child_err and \
                    test.status == 'FAIL' and test.critical == 'yes':
                child_err = 'Critical failure occurred and ExitOnFailure option is in use'
            self._set_prev_test_variables(self.namespace.variables, test)
        self.set_status()
        self.message = self._get_my_error(error, init_err, setup_err)
        self.namespace.variables['${SUITE_STATUS}'] = self.status
        self.namespace.variables['${SUITE_MESSAGE}'] = self.get_full_message()
        teardown_err = self._run_fixture(self.teardown, output, self.namespace,
                                         error, init_err)
        if teardown_err is not None:
            err_msg = 'Suite teardown failed:\n%s' % teardown_err
            self.suite_teardown_failed(err_msg)
        self.endtime = utils.get_timestamp()
        self.elapsedmillis = utils.get_elapsed_millis(self.starttime, self.endtime)
        self.elapsedtime = utils.elapsed_millis_to_string(self.elapsedmillis)
        self._set_prev_test_variables(GLOBAL_VARIABLES, varz=self.namespace.variables)
        output.end_suite(self)
        self.namespace.end_suite()

    def _init_suite(self, varz):
        errs = []
        self.doc = varz.replace_from_meta('Documentation', self.doc, errs)
        self.setup = Setup(varz.replace_from_meta('Setup', self.setup, errs))
        self.teardown = Teardown(varz.replace_from_meta('Teardown', self.teardown, errs))
        for name, value in self.metadata.items():
            self.metadata[name] = varz.replace_from_meta(name, value, errs)
        if errs:
            return '\n'.join(errs)
        return None

    def _get_child_error(self, error, init_error, setup_error):
        if error is not None:
            return error
        if init_error is not None:
            return 'Initialization of the parent suite failed.'
        if setup_error is not None:
            return 'Setup of the parent suite failed.'
        return None

    def _get_my_error(self, error, init_error, setup_error):
        if error is not None:
            return error
        if init_error is not None:
            return 'Suite initialization failed:\n%s' % init_error
        if setup_error is not None:
            return 'Suite setup failed:\n%s' % setup_error
        return ''

    def _run_fixture(self, fixture, output, namespace, error, init_error):
        if fixture is None or error is not None or init_error is not None:
            return None
        try:
            fixture.run(output, namespace)
        except ExecutionFailed:
            return utils.get_error_message()

    def _set_prev_test_variables(self, destination, test=None, varz=None):
        if test is not None:
            name, status, message = test.name, test.status, test.message
        else:
            name, status, message = varz['${PREV_TEST_NAME}'], \
                    varz['${PREV_TEST_STATUS}'], varz['${PREV_TEST_MESSAGE}']
        destination['${PREV_TEST_NAME}'] = name
        destination['${PREV_TEST_STATUS}'] = status
        destination['${PREV_TEST_MESSAGE}'] = message

class RunnableTestCase(BaseTestCase):

    def __init__(self, testdata, parentdatas):
        BaseTestCase.__init__(self, testdata.name)
        self.doc = testdata.doc is not None and testdata.doc or ''
        test_setup, test_teardown, force_tags, default_tags, test_timeout \
                = self._process_parents(parentdatas)
        self.setup = utils.get_not_none(testdata.setup, test_setup, [])
        self.teardown = utils.get_not_none(testdata.teardown, test_teardown, [])
        self.tags = force_tags + utils.get_not_none(testdata.tags, default_tags, [])
        self.timeout = utils.get_not_none(testdata.timeout, test_timeout, [])
        self.keywords = [ KeywordFactory(kw) for kw in testdata.keywords ]
        self.message = ''

    def run(self, output, namespace, error=None):
        self.starttime = utils.get_timestamp()
        init_err = self._init_test(namespace.variables)
        if error is None:
            error = init_err
        if error is None and len(self.keywords) == 0:
            error = 'Test case contains no keywords'
        namespace.start_test(self)
        output.start_test(self)
        if error is None:
            self._run(output, namespace)
        else:
            self.status = 'FAIL'
            self.message = error
        self.endtime = utils.get_timestamp()
        self.elapsedmillis = utils.get_elapsed_millis(self.starttime, self.endtime)
        self.elapsedtime = utils.elapsed_millis_to_string(self.elapsedmillis)
        output.end_test(self)
        namespace.end_test()

    def _init_test(self, varz):
        errs = []
        self.doc = varz.replace_from_meta('Documentation', self.doc, errs)
        self.setup = Setup(varz.replace_from_meta('Setup', self.setup, errs))
        self.teardown = Teardown(varz.replace_from_meta('Teardown', self.teardown, errs))
        self.tags = utils.normalize_list(varz.replace_from_meta('Tags', self.tags, errs))
        self.timeout = TestTimeout(*varz.replace_from_meta('Timeout', self.timeout, errs))
        if errs:
            return 'Test case initialization failed:\n%s' % '\n'.join(errs)
        return None

    def _run(self, output, namespace):
        namespace.variables['${TEST_NAME}'] = self.name
        namespace.variables['@{TEST_TAGS}'] = self.tags
        self.status = 'RUNNING'
        self.timeout.start()
        setup_err = self._run_fixture(self.setup, output, namespace)
        kw_err = self._run_keywords(output, namespace, setup_err)
        self.message = self._get_message(setup_err, kw_err)
        self.status = self.message == '' and 'PASS' or 'FAIL'
        namespace.variables['${TEST_STATUS}'] = self.status
        namespace.variables['${TEST_MESSAGE}'] = self.message
        teardown_err = self._run_fixture(self.teardown, output, namespace)
        if teardown_err is not None:
            self.message = self._get_message_with_teardown_err(self.message,
                                                               teardown_err)
            self.status = 'FAIL'
        if self.status == 'PASS' and self.timeout.timed_out():
            self.status = 'FAIL'
            self.message = self.timeout.get_message()

    def _run_fixture(self, fixture, output, namespace):
        if fixture is None:
            return None
        try:
            fixture.run(output, namespace)
        except ExecutionFailed:
            return utils.get_error_message()

    def _run_keywords(self, output, namespace, setup_err):
        if setup_err is None:
            for kw in self.keywords:
                try:
                    kw.run(output, namespace)
                except ExecutionFailed:
                    return utils.get_error_message()
        return None

    def _get_message(self, setup_err, kw_err):
        if setup_err is None and kw_err is None:
            return ''
        if setup_err is not None:
            return 'Setup failed:\n%s' % setup_err
        return kw_err

    def _get_message_with_teardown_err(self, message, teardown_err):
        if message == '':
            return 'Teardown failed:\n%s' % teardown_err
        return '%s\n\nAlso teardown failed:\n%s' % (message, teardown_err)

    def _process_parents(self, parentdatas):
        test_setup = test_teardown = default_tags = test_timeout = None
        force_tags = []
        parentdatas.reverse()
        for parent in parentdatas:
            if parent.test_setup is not None and test_setup is None:
                test_setup = parent.test_setup
            if parent.test_teardown is not None and test_teardown is None:
                test_teardown = parent.test_teardown
            if parent.force_tags is not None:
                force_tags.extend(parent.force_tags)
            if parent.default_tags is not None and default_tags is None:
                default_tags = parent.default_tags
            if parent.test_timeout is not None and test_timeout is None:
                test_timeout = parent.test_timeout
        return test_setup, test_teardown, force_tags, default_tags, test_timeout
<commit_msg>Fixed issue 152.<commit_after>#  Copyright 2008 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

from robot import utils
from robot.common import BaseTestSuite, BaseTestCase
from robot.parsing import TestSuiteData
from robot.errors import ExecutionFailed
from robot.variables import GLOBAL_VARIABLES

from fixture import Setup, Teardown
from timeouts import TestTimeout
from keywords import KeywordFactory
from namespace import Namespace
from userkeyword import UserLibrary

def TestSuite(datasources, settings, syslog):
    suitedata = TestSuiteData(datasources, settings, syslog)
    suite = RunnableTestSuite(suitedata)
    suite.set_options(settings)
    return suite

class RunnableTestSuite(BaseTestSuite):

    def __init__(self, suitedata, parentdatas=None):
        if not parentdatas:
            parentdatas = []
        parentdatas = [suitedata] + parentdatas[:]
        BaseTestSuite.__init__(self, suitedata.name, suitedata.source)
        self.variables = GLOBAL_VARIABLES.copy()
        self.variables.set_from_variable_table(suitedata.variables)
        self.doc = suitedata.doc is not None and suitedata.doc or ''
        self.metadata = suitedata.metadata
        self.imports = suitedata.imports
        self.user_keywords = UserLibrary(suitedata.user_keywords)
        self.setup = utils.get_not_none(suitedata.suite_setup, [])
        self.teardown = utils.get_not_none(suitedata.suite_teardown, [])
        self.suites = [ RunnableTestSuite(suite, parentdatas)
                        for suite in suitedata.suites ]
        self.tests = [ RunnableTestCase(test, parentdatas)
                       for test in suitedata.tests ]
        if self.name == '':   # suitedata was multisource suite
            self.name = ' & '.join([suite.name for suite in self.suites])
        self.message = ''
        self._exit_on_failure = False

    def run(self, output, parent=None, error=None):
        self.starttime = utils.get_timestamp()
        self.namespace = Namespace(self, parent, output.syslog)
        self.namespace.variables['${SUITE_NAME}'] = self.longname
        init_err = self._init_suite(self.namespace.variables)
        output.start_suite(self)
        self.status = 'RUNNING'
        setup_err = self._run_fixture(self.setup, output, self.namespace, error, init_err)
        child_err = self._get_child_error(error, init_err, setup_err)
        for suite in self.suites:
            suite.run(output, self, child_err)
            if self._exit_on_failure and not child_err and suite.critical_stats.failed:
                child_err = 'Critical failure occurred and ExitOnFailure option is in use'
        for test in self.tests:
            test.run(output, self.namespace, child_err)
            if self._exit_on_failure and not child_err and \
                    test.status == 'FAIL' and test.critical == 'yes':
                child_err = 'Critical failure occurred and ExitOnFailure option is in use'
            self._set_prev_test_variables(self.namespace.variables, test)
        self.set_status()
        self.message = self._get_my_error(error, init_err, setup_err)
        self.namespace.variables['${SUITE_STATUS}'] = self.status
        self.namespace.variables['${SUITE_MESSAGE}'] = self.get_full_message()
        teardown_err = self._run_fixture(self.teardown, output, self.namespace,
                                         error, init_err)
        if teardown_err is not None:
            err_msg = 'Suite teardown failed:\n%s' % teardown_err
            self.suite_teardown_failed(err_msg)
        self.endtime = utils.get_timestamp()
        self.elapsedmillis = utils.get_elapsed_millis(self.starttime, self.endtime)
        self.elapsedtime = utils.elapsed_millis_to_string(self.elapsedmillis)
        self._set_prev_test_variables(GLOBAL_VARIABLES, varz=self.namespace.variables)
        output.end_suite(self)
        self.namespace.end_suite()

    def _init_suite(self, varz):
        errs = []
        self.doc = varz.replace_from_meta('Documentation', self.doc, errs)
        self.setup = Setup(varz.replace_from_meta('Setup', self.setup, errs))
        self.teardown = Teardown(varz.replace_from_meta('Teardown', self.teardown, errs))
        for name, value in self.metadata.items():
            self.metadata[name] = varz.replace_from_meta(name, value, errs)
        if errs:
            return '\n'.join(errs)
        return None

    def _get_child_error(self, error, init_error, setup_error):
        if error is not None:
            return error
        if init_error is not None:
            return 'Initialization of the parent suite failed.'
        if setup_error is not None:
            return 'Setup of the parent suite failed.'
        return None

    def _get_my_error(self, error, init_error, setup_error):
        if error is not None:
            return error
        if init_error is not None:
            return 'Suite initialization failed:\n%s' % init_error
        if setup_error is not None:
            return 'Suite setup failed:\n%s' % setup_error
        return ''

    def _run_fixture(self, fixture, output, namespace, error, init_error):
        if fixture is None or error is not None or init_error is not None:
            return None
        try:
            fixture.run(output, namespace)
        except ExecutionFailed:
            return utils.get_error_message()

    def _set_prev_test_variables(self, destination, test=None, varz=None):
        if test is not None:
            name, status, message = test.name, test.status, test.message
        else:
            name, status, message = varz['${PREV_TEST_NAME}'], \
                    varz['${PREV_TEST_STATUS}'], varz['${PREV_TEST_MESSAGE}']
        destination['${PREV_TEST_NAME}'] = name
        destination['${PREV_TEST_STATUS}'] = status
        destination['${PREV_TEST_MESSAGE}'] = message

class RunnableTestCase(BaseTestCase):

    def __init__(self, testdata, parentdatas):
        BaseTestCase.__init__(self, testdata.name)
        self.doc = testdata.doc is not None and testdata.doc or ''
        test_setup, test_teardown, force_tags, default_tags, test_timeout \
                = self._process_parents(parentdatas)
        self.setup = utils.get_not_none(testdata.setup, test_setup, [])
        self.teardown = utils.get_not_none(testdata.teardown, test_teardown, [])
        self.tags = force_tags + utils.get_not_none(testdata.tags, default_tags, [])
        self.timeout = utils.get_not_none(testdata.timeout, test_timeout, [])
        self.keywords = [ KeywordFactory(kw) for kw in testdata.keywords ]
        self.message = ''

    def run(self, output, namespace, error=None):
        self.starttime = utils.get_timestamp()
        init_err = self._init_test(namespace.variables)
        if error is None:
            error = init_err
        if error is None and len(self.keywords) == 0:
            error = 'Test case contains no keywords'
        namespace.start_test(self)
        output.start_test(self)
        if error is None:
            self._run(output, namespace)
        else:
            self.status = 'FAIL'
            self.message = error
        self.endtime = utils.get_timestamp()
        self.elapsedmillis = utils.get_elapsed_millis(self.starttime, self.endtime)
        self.elapsedtime = utils.elapsed_millis_to_string(self.elapsedmillis)
        output.end_test(self)
        namespace.end_test()

    def _init_test(self, varz):
        errs = []
        self.doc = varz.replace_from_meta('Documentation', self.doc, errs)
        self.setup = Setup(varz.replace_from_meta('Setup', self.setup, errs))
        self.teardown = Teardown(varz.replace_from_meta('Teardown', self.teardown, errs))
        self.tags = utils.normalize_list(varz.replace_from_meta('Tags', self.tags, errs))
        self.timeout = TestTimeout(*varz.replace_from_meta('Timeout', self.timeout, errs))
        if errs:
            return 'Test case initialization failed:\n%s' % '\n'.join(errs)
        return None

    def _run(self, output, namespace):
        namespace.variables['${TEST_NAME}'] = self.name
        namespace.variables['@{TEST_TAGS}'] = self.tags
        self.status = 'RUNNING'
        self.timeout.start()
        setup_err = self._run_fixture(self.setup, output, namespace)
        kw_err = self._run_keywords(output, namespace, setup_err)
        self.message = self._get_message(setup_err, kw_err)
        self.status = self.message == '' and 'PASS' or 'FAIL'
        namespace.variables['${TEST_STATUS}'] = self.status
        namespace.variables['${TEST_MESSAGE}'] = self.message
        teardown_err = self._run_fixture(self.teardown, output, namespace)
        if teardown_err is not None:
            self.message = self._get_message_with_teardown_err(self.message,
                                                               teardown_err)
            self.status = 'FAIL'
        if self.status == 'PASS' and self.timeout.timed_out():
            self.status = 'FAIL'
            self.message = self.timeout.get_message()

    def _run_fixture(self, fixture, output, namespace):
        if fixture is None:
            return None
        try:
            fixture.run(output, namespace)
        except ExecutionFailed:
            return utils.get_error_message()

    def _run_keywords(self, output, namespace, setup_err):
        if setup_err is None:
            for kw in self.keywords:
                try:
                    kw.run(output, namespace)
                except ExecutionFailed:
                    return utils.get_error_message()
        return None

    def _get_message(self, setup_err, kw_err):
        if setup_err is None and kw_err is None:
            return ''
        if setup_err is not None:
            return 'Setup failed:\n%s' % setup_err
        return kw_err

    def _get_message_with_teardown_err(self, message, teardown_err):
        if message == '':
            return 'Teardown failed:\n%s' % teardown_err
        return '%s\n\nAlso teardown failed:\n%s' % (message, teardown_err)

    def _process_parents(self, parentdatas):
        test_setup = test_teardown = default_tags = test_timeout = None
        force_tags = []
        for parent in parentdatas:
            if parent.test_setup is not None and test_setup is None:
                test_setup = parent.test_setup
            if parent.test_teardown is not None and test_teardown is None:
                test_teardown = parent.test_teardown
            if parent.force_tags is not None:
                force_tags.extend(parent.force_tags)
            if parent.default_tags is not None and default_tags is None:
                default_tags = parent.default_tags
            if parent.test_timeout is not None and test_timeout is None:
                test_timeout = parent.test_timeout
        return test_setup, test_teardown, force_tags, default_tags, test_timeout
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    pygments.cmdline
    ~~~~~~~~~~~~~~~~

    Command line interface.

    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""
import sys
import getopt
from textwrap import dedent

from pygments import __version__, highlight
from pygments.util import ClassNotFound, OptionError, docstring_headline
from pygments.lexers import get_all_lexers, get_lexer_by_name, get_lexer_for_filename, \
     find_lexer_class, guess_lexer, TextLexer
from pygments.formatters import get_all_formatters, get_formatter_by_name, \
     get_formatter_for_filename, find_formatter_class, \
     TerminalFormatter  # pylint:disable-msg=E0611
from pygments.filters import get_all_filters, find_filter_class
from pygments.styles import get_all_styles, get_style_by_name

USAGE = """\
Usage: %s [-l <lexer> | -g] [-F <filter>[:<options>]] [-f <formatter>]
          [-O <options>] [-P <option=value>] [-o <outfile>] [<infile>]

       %s -S <style> -f <formatter> [-a <arg>] [-O <options>] [-P <option=value>]
       %s -L [<which> ...]
       %s -N <filename>
       %s -H <type> <name>
       %s -h | -V

Highlight the input file and write the result to <outfile>.

If no input file is given, use stdin, if -o is not given, use stdout.

<lexer> is a lexer name (query all lexer names with -L). If -l is not
given, the lexer is guessed from the extension of the input file name
(this obviously doesn't work if the input is stdin).  If -g is passed,
attempt to guess the lexer from the file contents, or pass through as
plain text if this fails (this can work for stdin).

Likewise, <formatter> is a formatter name, and will be guessed from
the extension of the output file name. If no output file is given,
the terminal formatter will be used by default.

With the -O option, you can give the lexer and formatter a comma-
separated list of options, e.g. ``-O bg=light,python=cool``.

The -P option adds lexer and formatter options like the -O option, but
you can only give one option per -P. That way, the option value may
contain commas and equals signs, which it can't with -O, e.g.
``-P "heading=Pygments, the Python highlighter".

With the -F option, you can add filters to the token stream, you can
give options in the same way as for -O after a colon (note: there must
not be spaces around the colon).

The -O, -P and -F options can be given multiple times.

With the -S option, print out style definitions for style <style>
for formatter <formatter>. The argument given by -a is formatter
dependent.

The -L option lists lexers, formatters, styles or filters -- set
`which` to the thing you want to list (e.g. "styles"), or omit it to
list everything.

The -N option guesses and prints out a lexer name based solely on
the given filename. It does not take input or highlight anything.
If no specific lexer can be determined "text" is returned.

The -H option prints detailed help for the object <name> of type <type>,
where <type> is one of "lexer", "formatter" or "filter".

The -h option prints this help.
The -V option prints the package version.
"""

def _parse_options(o_strs):
    opts = {}
    if not o_strs:
        return opts
    for o_str in o_strs:
        if not o_str:
            continue
        o_args = o_str.split(',')
        for o_arg in o_args:
            o_arg = o_arg.strip()
            try:
                o_key, o_val = o_arg.split('=')
                o_key = o_key.strip()
                o_val = o_val.strip()
            except ValueError:
                opts[o_arg] = True
            else:
                opts[o_key] = o_val
    return opts

def _parse_filters(f_strs):
    filters = []
    if not f_strs:
        return filters
    for f_str in f_strs:
        if ':' in f_str:
            fname, fopts = f_str.split(':', 1)
            filters.append((fname, _parse_options([fopts])))
        else:
            filters.append((f_str, {}))
    return filters

def _print_help(what, name):
    try:
        if what == 'lexer':
            cls = find_lexer_class(name)
            print "Help on the %s lexer:" % cls.name
            print dedent(cls.__doc__)
        elif what == 'formatter':
            cls = find_formatter_class(name)
            print "Help on the %s formatter:" % cls.name
            print dedent(cls.__doc__)
        elif what == 'filter':
            cls = find_filter_class(name)
            print "Help on the %s filter:" % name
            print dedent(cls.__doc__)
    except AttributeError:
        print >>sys.stderr, "%s not found!" % what

def _print_list(what):
    if what == 'lexer':
        print
        print "Lexers:"
        print "~~~~~~~"

        info = []
        for fullname, names, exts, _ in get_all_lexers():
            tup = (', '.join(names)+':', fullname,
                   exts and '(filenames ' + ', '.join(exts) + ')' or '')
            info.append(tup)
        info.sort()
        for i in info:
            print ('* %s\n    %s %s') % i

    elif what == 'formatter':
        print
        print "Formatters:"
        print "~~~~~~~~~~~"

        info = []
        for cls in get_all_formatters():
            doc = docstring_headline(cls)
            tup = (', '.join(cls.aliases) + ':', doc, cls.filenames and
                   '(filenames ' + ', '.join(cls.filenames) + ')' or '')
            info.append(tup)
        info.sort()
        for i in info:
            print ('* %s\n    %s %s') % i

    elif what == 'filter':
        print
        print "Filters:"
        print "~~~~~~~~"

        for name in get_all_filters():
            cls = find_filter_class(name)
            print "* " + name + ':'
            print "    %s" % docstring_headline(cls)

    elif what == 'style':
        print
        print "Styles:"
        print "~~~~~~~"

        for name in get_all_styles():
            cls = get_style_by_name(name)
            print "* " + name + ':'
            print "    %s" % docstring_headline(cls)

def main(args=sys.argv):
    """
    Main command line entry point.
    """
    # pylint: disable-msg=R0911,R0912,R0915

    usage = USAGE % ((args[0],) * 6)

    if sys.platform in ['win32', 'cygwin']:
        try:
            # Provide coloring under Windows, if possible
            import colorama
            colorama.init()
        except ImportError:
            pass

    try:
        popts, args = getopt.getopt(args[1:], "l:f:F:o:O:P:LS:a:N:hVHg")
    except getopt.GetoptError, err:
        print >>sys.stderr, usage
        return 2
    opts = {}
    O_opts = []
    P_opts = []
    F_opts = []
    for opt, arg in popts:
        if opt == '-O':
            O_opts.append(arg)
        elif opt == '-P':
            P_opts.append(arg)
        elif opt == '-F':
            F_opts.append(arg)
        opts[opt] = arg

    if not opts and not args:
        print usage
        return 0

    if opts.pop('-h', None) is not None:
        print usage
        return 0

    if opts.pop('-V', None) is not None:
        print 'Pygments version %s, (c) 2006-2013 by Georg Brandl.' % __version__
        return 0

    # handle ``pygmentize -L``
    L_opt = opts.pop('-L', None)
    if L_opt is not None:
        if opts:
            print >>sys.stderr, usage
            return 2

        # print version
        main(['', '-V'])
        if not args:
            args = ['lexer', 'formatter', 'filter', 'style']
        for arg in args:
            _print_list(arg.rstrip('s'))
        return 0

    # handle ``pygmentize -H``
    H_opt = opts.pop('-H', None)
    if H_opt is not None:
        if opts or len(args) != 2:
            print >>sys.stderr, usage
            return 2

        what, name = args
        if what not in ('lexer', 'formatter', 'filter'):
            print >>sys.stderr, usage
            return 2

        _print_help(what, name)
        return 0

    # parse -O options
    parsed_opts = _parse_options(O_opts)
    opts.pop('-O', None)

    # parse -P options
    for p_opt in P_opts:
        try:
            name, value = p_opt.split('=', 1)
        except ValueError:
            parsed_opts[p_opt] = True
        else:
            parsed_opts[name] = value
    opts.pop('-P', None)

    # handle ``pygmentize -N``
    infn = opts.pop('-N', None)
    if infn is not None:
        try:
            lexer = get_lexer_for_filename(infn, **parsed_opts)
        except ClassNotFound, err:
            lexer = TextLexer()
        except OptionError, err:
            print >>sys.stderr, 'Error:', err
            return 1

        print lexer.aliases[0]
        return 0

    # handle ``pygmentize -S``
    S_opt = opts.pop('-S', None)
    a_opt = opts.pop('-a', None)
    if S_opt is not None:
        f_opt = opts.pop('-f', None)
        if not f_opt:
            print >>sys.stderr, usage
            return 2
        if opts or args:
            print >>sys.stderr, usage
            return 2

        try:
            parsed_opts['style'] = S_opt
            fmter = get_formatter_by_name(f_opt, **parsed_opts)
        except ClassNotFound, err:
            print >>sys.stderr, err
            return 1

        arg = a_opt or ''
        try:
            print fmter.get_style_defs(arg)
        except Exception, err:
            print >>sys.stderr, 'Error:', err
            return 1
        return 0

    # if no -S is given, -a is not allowed
    if a_opt is not None:
        print >>sys.stderr, usage
        return 2

    # parse -F options
    F_opts = _parse_filters(F_opts)
    opts.pop('-F', None)

    # select formatter
    outfn = opts.pop('-o', None)
    fmter = opts.pop('-f', None)
    if fmter:
        try:
            fmter = get_formatter_by_name(fmter, **parsed_opts)
        except (OptionError, ClassNotFound), err:
            print >>sys.stderr, 'Error:', err
            return 1

    if outfn:
        if not fmter:
            try:
                fmter = get_formatter_for_filename(outfn, **parsed_opts)
            except (OptionError, ClassNotFound), err:
                print >>sys.stderr, 'Error:', err
                return 1
        try:
            outfile = open(outfn, 'wb')
        except Exception, err:
            print >>sys.stderr, 'Error: cannot open outfile:', err
            return 1
    else:
        if not fmter:
            fmter = TerminalFormatter(**parsed_opts)
        outfile = sys.stdout

    # select lexer
    lexer = opts.pop('-l', None)
    if lexer:
        try:
            lexer = get_lexer_by_name(lexer, **parsed_opts)
        except (OptionError, ClassNotFound), err:
            print >>sys.stderr, 'Error:', err
            return 1

    if args:
        if len(args) > 1:
            print >>sys.stderr, usage
            return 2

        infn = args[0]
        try:
            code = open(infn, 'rb').read()
        except Exception, err:
            print >>sys.stderr, 'Error: cannot read infile:', err
            return 1

        if not lexer:
            try:
                lexer = get_lexer_for_filename(infn, code, **parsed_opts)
            except ClassNotFound, err:
                if '-g' in opts:
                    try:
                        lexer = guess_lexer(code, **parsed_opts)
                    except ClassNotFound:
                        lexer = TextLexer(**parsed_opts)
                else:
                    print >>sys.stderr, 'Error:', err
                    return 1
            except OptionError, err:
                print >>sys.stderr, 'Error:', err
                return 1

    else:
        if '-g' in opts:
            code = sys.stdin.read()
            try:
                lexer = guess_lexer(code, **parsed_opts)
            except ClassNotFound:
                lexer = TextLexer(**parsed_opts)
        elif not lexer:
            print >>sys.stderr, 'Error: no lexer name given and reading ' + \
                                'from stdin (try using -g or -l <lexer>)'
            return 2
        else:
            code = sys.stdin.read()

    # No encoding given? Use latin1 if output file given,
    # stdin/stdout encoding otherwise.
    # (This is a compromise, I'm not too happy with it...)
    if 'encoding' not in parsed_opts and 'outencoding' not in parsed_opts:
        if outfn:
            # encoding pass-through
            fmter.encoding = 'latin1'
        else:
            if sys.version_info < (3,):
                # use terminal encoding; Python 3's terminals already do that
                lexer.encoding = getattr(sys.stdin, 'encoding',
                                         None) or 'ascii'
                fmter.encoding = getattr(sys.stdout, 'encoding',
                                         None) or 'ascii'
    elif not outfn and sys.version_info > (3,):
        # output to terminal with encoding -> use .buffer
        outfile = sys.stdout.buffer

    # ... and do it!
    try:
        # process filters
        for fname, fopts in F_opts:
            lexer.add_filter(fname, **fopts)
        highlight(code, lexer, fmter, outfile)
    except Exception, err:
        import traceback
        info = traceback.format_exception(*sys.exc_info())
        msg = info[-1].strip()
        if len(info) >= 3:
            # extract relevant file and position info
            msg += '\n   (f%s)' % info[-2].split('\n')[0].strip()[1:]
        print >>sys.stderr
        print >>sys.stderr, '*** Error while highlighting:'
        print >>sys.stderr, msg
        return 1

    return 0
<commit_msg>Fix options with "=" in them.<commit_after># -*- coding: utf-8 -*-
"""
    pygments.cmdline
    ~~~~~~~~~~~~~~~~

    Command line interface.

    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""
import sys
import getopt
from textwrap import dedent

from pygments import __version__, highlight
from pygments.util import ClassNotFound, OptionError, docstring_headline
from pygments.lexers import get_all_lexers, get_lexer_by_name, get_lexer_for_filename, \
     find_lexer_class, guess_lexer, TextLexer
from pygments.formatters import get_all_formatters, get_formatter_by_name, \
     get_formatter_for_filename, find_formatter_class, \
     TerminalFormatter  # pylint:disable-msg=E0611
from pygments.filters import get_all_filters, find_filter_class
from pygments.styles import get_all_styles, get_style_by_name

USAGE = """\
Usage: %s [-l <lexer> | -g] [-F <filter>[:<options>]] [-f <formatter>]
          [-O <options>] [-P <option=value>] [-o <outfile>] [<infile>]

       %s -S <style> -f <formatter> [-a <arg>] [-O <options>] [-P <option=value>]
       %s -L [<which> ...]
       %s -N <filename>
       %s -H <type> <name>
       %s -h | -V

Highlight the input file and write the result to <outfile>.

If no input file is given, use stdin, if -o is not given, use stdout.

<lexer> is a lexer name (query all lexer names with -L). If -l is not
given, the lexer is guessed from the extension of the input file name
(this obviously doesn't work if the input is stdin).  If -g is passed,
attempt to guess the lexer from the file contents, or pass through as
plain text if this fails (this can work for stdin).

Likewise, <formatter> is a formatter name, and will be guessed from
the extension of the output file name. If no output file is given,
the terminal formatter will be used by default.

With the -O option, you can give the lexer and formatter a comma-
separated list of options, e.g. ``-O bg=light,python=cool``.

The -P option adds lexer and formatter options like the -O option, but
you can only give one option per -P. That way, the option value may
contain commas and equals signs, which it can't with -O, e.g.
``-P "heading=Pygments, the Python highlighter".

With the -F option, you can add filters to the token stream, you can
give options in the same way as for -O after a colon (note: there must
not be spaces around the colon).

The -O, -P and -F options can be given multiple times.

With the -S option, print out style definitions for style <style>
for formatter <formatter>. The argument given by -a is formatter
dependent.

The -L option lists lexers, formatters, styles or filters -- set
`which` to the thing you want to list (e.g. "styles"), or omit it to
list everything.

The -N option guesses and prints out a lexer name based solely on
the given filename. It does not take input or highlight anything.
If no specific lexer can be determined "text" is returned.

The -H option prints detailed help for the object <name> of type <type>,
where <type> is one of "lexer", "formatter" or "filter".

The -h option prints this help.
The -V option prints the package version.
"""

def _parse_options(o_strs):
    opts = {}
    if not o_strs:
        return opts
    for o_str in o_strs:
        if not o_str:
            continue
        o_args = o_str.split(',')
        for o_arg in o_args:
            o_arg = o_arg.strip()
            try:
                o_key, o_val = o_arg.split('=', 1)
                o_key = o_key.strip()
                o_val = o_val.strip()
            except ValueError:
                opts[o_arg] = True
            else:
                opts[o_key] = o_val
    return opts

def _parse_filters(f_strs):
    filters = []
    if not f_strs:
        return filters
    for f_str in f_strs:
        if ':' in f_str:
            fname, fopts = f_str.split(':', 1)
            filters.append((fname, _parse_options([fopts])))
        else:
            filters.append((f_str, {}))
    return filters

def _print_help(what, name):
    try:
        if what == 'lexer':
            cls = find_lexer_class(name)
            print "Help on the %s lexer:" % cls.name
            print dedent(cls.__doc__)
        elif what == 'formatter':
            cls = find_formatter_class(name)
            print "Help on the %s formatter:" % cls.name
            print dedent(cls.__doc__)
        elif what == 'filter':
            cls = find_filter_class(name)
            print "Help on the %s filter:" % name
            print dedent(cls.__doc__)
    except AttributeError:
        print >>sys.stderr, "%s not found!" % what

def _print_list(what):
    if what == 'lexer':
        print
        print "Lexers:"
        print "~~~~~~~"

        info = []
        for fullname, names, exts, _ in get_all_lexers():
            tup = (', '.join(names)+':', fullname,
                   exts and '(filenames ' + ', '.join(exts) + ')' or '')
            info.append(tup)
        info.sort()
        for i in info:
            print ('* %s\n    %s %s') % i

    elif what == 'formatter':
        print
        print "Formatters:"
        print "~~~~~~~~~~~"

        info = []
        for cls in get_all_formatters():
            doc = docstring_headline(cls)
            tup = (', '.join(cls.aliases) + ':', doc, cls.filenames and
                   '(filenames ' + ', '.join(cls.filenames) + ')' or '')
            info.append(tup)
        info.sort()
        for i in info:
            print ('* %s\n    %s %s') % i

    elif what == 'filter':
        print
        print "Filters:"
        print "~~~~~~~~"

        for name in get_all_filters():
            cls = find_filter_class(name)
            print "* " + name + ':'
            print "    %s" % docstring_headline(cls)

    elif what == 'style':
        print
        print "Styles:"
        print "~~~~~~~"

        for name in get_all_styles():
            cls = get_style_by_name(name)
            print "* " + name + ':'
            print "    %s" % docstring_headline(cls)

def main(args=sys.argv):
    """
    Main command line entry point.
    """
    # pylint: disable-msg=R0911,R0912,R0915

    usage = USAGE % ((args[0],) * 6)

    if sys.platform in ['win32', 'cygwin']:
        try:
            # Provide coloring under Windows, if possible
            import colorama
            colorama.init()
        except ImportError:
            pass

    try:
        popts, args = getopt.getopt(args[1:], "l:f:F:o:O:P:LS:a:N:hVHg")
    except getopt.GetoptError, err:
        print >>sys.stderr, usage
        return 2
    opts = {}
    O_opts = []
    P_opts = []
    F_opts = []
    for opt, arg in popts:
        if opt == '-O':
            O_opts.append(arg)
        elif opt == '-P':
            P_opts.append(arg)
        elif opt == '-F':
            F_opts.append(arg)
        opts[opt] = arg

    if not opts and not args:
        print usage
        return 0

    if opts.pop('-h', None) is not None:
        print usage
        return 0

    if opts.pop('-V', None) is not None:
        print 'Pygments version %s, (c) 2006-2013 by Georg Brandl.' % __version__
        return 0

    # handle ``pygmentize -L``
    L_opt = opts.pop('-L', None)
    if L_opt is not None:
        if opts:
            print >>sys.stderr, usage
            return 2

        # print version
        main(['', '-V'])
        if not args:
            args = ['lexer', 'formatter', 'filter', 'style']
        for arg in args:
            _print_list(arg.rstrip('s'))
        return 0

    # handle ``pygmentize -H``
    H_opt = opts.pop('-H', None)
    if H_opt is not None:
        if opts or len(args) != 2:
            print >>sys.stderr, usage
            return 2

        what, name = args
        if what not in ('lexer', 'formatter', 'filter'):
            print >>sys.stderr, usage
            return 2

        _print_help(what, name)
        return 0

    # parse -O options
    parsed_opts = _parse_options(O_opts)
    opts.pop('-O', None)

    # parse -P options
    for p_opt in P_opts:
        try:
            name, value = p_opt.split('=', 1)
        except ValueError:
            parsed_opts[p_opt] = True
        else:
            parsed_opts[name] = value
    opts.pop('-P', None)

    # handle ``pygmentize -N``
    infn = opts.pop('-N', None)
    if infn is not None:
        try:
            lexer = get_lexer_for_filename(infn, **parsed_opts)
        except ClassNotFound, err:
            lexer = TextLexer()
        except OptionError, err:
            print >>sys.stderr, 'Error:', err
            return 1

        print lexer.aliases[0]
        return 0

    # handle ``pygmentize -S``
    S_opt = opts.pop('-S', None)
    a_opt = opts.pop('-a', None)
    if S_opt is not None:
        f_opt = opts.pop('-f', None)
        if not f_opt:
            print >>sys.stderr, usage
            return 2
        if opts or args:
            print >>sys.stderr, usage
            return 2

        try:
            parsed_opts['style'] = S_opt
            fmter = get_formatter_by_name(f_opt, **parsed_opts)
        except ClassNotFound, err:
            print >>sys.stderr, err
            return 1

        arg = a_opt or ''
        try:
            print fmter.get_style_defs(arg)
        except Exception, err:
            print >>sys.stderr, 'Error:', err
            return 1
        return 0

    # if no -S is given, -a is not allowed
    if a_opt is not None:
        print >>sys.stderr, usage
        return 2

    # parse -F options
    F_opts = _parse_filters(F_opts)
    opts.pop('-F', None)

    # select formatter
    outfn = opts.pop('-o', None)
    fmter = opts.pop('-f', None)
    if fmter:
        try:
            fmter = get_formatter_by_name(fmter, **parsed_opts)
        except (OptionError, ClassNotFound), err:
            print >>sys.stderr, 'Error:', err
            return 1

    if outfn:
        if not fmter:
            try:
                fmter = get_formatter_for_filename(outfn, **parsed_opts)
            except (OptionError, ClassNotFound), err:
                print >>sys.stderr, 'Error:', err
                return 1
        try:
            outfile = open(outfn, 'wb')
        except Exception, err:
            print >>sys.stderr, 'Error: cannot open outfile:', err
            return 1
    else:
        if not fmter:
            fmter = TerminalFormatter(**parsed_opts)
        outfile = sys.stdout

    # select lexer
    lexer = opts.pop('-l', None)
    if lexer:
        try:
            lexer = get_lexer_by_name(lexer, **parsed_opts)
        except (OptionError, ClassNotFound), err:
            print >>sys.stderr, 'Error:', err
            return 1

    if args:
        if len(args) > 1:
            print >>sys.stderr, usage
            return 2

        infn = args[0]
        try:
            code = open(infn, 'rb').read()
        except Exception, err:
            print >>sys.stderr, 'Error: cannot read infile:', err
            return 1

        if not lexer:
            try:
                lexer = get_lexer_for_filename(infn, code, **parsed_opts)
            except ClassNotFound, err:
                if '-g' in opts:
                    try:
                        lexer = guess_lexer(code, **parsed_opts)
                    except ClassNotFound:
                        lexer = TextLexer(**parsed_opts)
                else:
                    print >>sys.stderr, 'Error:', err
                    return 1
            except OptionError, err:
                print >>sys.stderr, 'Error:', err
                return 1

    else:
        if '-g' in opts:
            code = sys.stdin.read()
            try:
                lexer = guess_lexer(code, **parsed_opts)
            except ClassNotFound:
                lexer = TextLexer(**parsed_opts)
        elif not lexer:
            print >>sys.stderr, 'Error: no lexer name given and reading ' + \
                                'from stdin (try using -g or -l <lexer>)'
            return 2
        else:
            code = sys.stdin.read()

    # No encoding given? Use latin1 if output file given,
    # stdin/stdout encoding otherwise.
    # (This is a compromise, I'm not too happy with it...)
    if 'encoding' not in parsed_opts and 'outencoding' not in parsed_opts:
        if outfn:
            # encoding pass-through
            fmter.encoding = 'latin1'
        else:
            if sys.version_info < (3,):
                # use terminal encoding; Python 3's terminals already do that
                lexer.encoding = getattr(sys.stdin, 'encoding',
                                         None) or 'ascii'
                fmter.encoding = getattr(sys.stdout, 'encoding',
                                         None) or 'ascii'
    elif not outfn and sys.version_info > (3,):
        # output to terminal with encoding -> use .buffer
        outfile = sys.stdout.buffer

    # ... and do it!
    try:
        # process filters
        for fname, fopts in F_opts:
            lexer.add_filter(fname, **fopts)
        highlight(code, lexer, fmter, outfile)
    except Exception, err:
        import traceback
        info = traceback.format_exception(*sys.exc_info())
        msg = info[-1].strip()
        if len(info) >= 3:
            # extract relevant file and position info
            msg += '\n   (f%s)' % info[-2].split('\n')[0].strip()[1:]
        print >>sys.stderr
        print >>sys.stderr, '*** Error while highlighting:'
        print >>sys.stderr, msg
        return 1

    return 0
<|endoftext|>


<commit_before>import ConfigParser
import shutil
import threading, time, os
from db import DB
from watchdog.observers import Observer
from watchdog.events import PatternMatchingEventHandler
from preprocessing import Preprocessor

class PostprocessingWorker(threading.Thread):
    """ Python script for Postprocessing worker... runs until cancelled or till max waiting time
    """

    pause_time = 2
    max_waiting_time = 60 * 60  # 60seconds * 60min = 1 hour in seconds
    base_path = ""
    pop_path = "population/"
    traces_path = "traces_afterVox/"
    debug = False
    db = None
    queue = []
    vox_preamble = 8  # number of lines that voxelyze adds before the actual output in a trace file
    config = ConfigParser.RawConfigParser()
    arena_x = 0
    arena_y = 0
    arena_type = ""
    end_time = 0
    timeTolerance = 0.0  # maximum mating time distance
    spaceTolerance = 0.01  # maximum mating distance radius
    pp = Preprocessor()
    indiv_max_age = 0
    indiv_infertile_span = 0.25

    def readConfig(self, config_path):
        self.config.read(config_path)
        self.exp_name = self.config.get('Experiment', 'name')
        self.path_prefix = self.config.get('Experiment', 'path_prefix')
        self.debug = self.config.get('Experiment', 'debug')
        self.base_path = os.path.expanduser(self.path_prefix + self.exp_name) + "/"
        self.pop_path = self.config.get('Postprocessing', 'pop_path')
        self.traces_path = self.config.get('Postprocessing', 'traces_path')
        self.vox_preamble = self.config.getint('Postprocessing', 'vox_preamble')
        self.pause_time = self.config.getint('Workers', 'pause_time')
        self.queue_length = self.config.getint('Workers', 'queue_len')
        self.max_waiting_time = self.config.getint('Workers', 'max_waiting_time')
        self.arena_x = self.config.getfloat('Arena', 'x')
        self.arena_y = self.config.getfloat('Arena', 'y')
        self.arena_type = self.config.get('Arena', 'type')
        self.end_time = self.config.getfloat('Experiment', 'end_time')
        self.indiv_max_age = self.config.getfloat('Population', 'indiv_max_age')

    def __init__(self, dbParams, config_path):
        threading.Thread.__init__(self)
        self.db = DB(dbParams[0], dbParams[1], dbParams[2], dbParams[3])
        self.readConfig(config_path)

        self.stopRequest = threading.Event()
        self.observer = Observer()

    def run(self):
        """ main thread function
        :return: None
        """
        waitCounter = 0
        startTime = time.time()

        obs_path = os.path.normpath(self.base_path + self.traces_path)
        self.observer.schedule(ChangeHandler(self), path=obs_path)
        print("PP: starting file observer on path:\n" + obs_path)
        self.observer.start()

        self.initialDirCheck(obs_path)

        while (not self.stopRequest.isSet() and waitCounter < self.max_waiting_time):
            if (len(self.queue) > 0):
                queue_partition = self.queue
                self.queue = self.queue[len(
                    queue_partition):]  # to make sure that in the short fraction of time no new file was added
                if (self.debug):
                    print("PP: found " + str(len(queue_partition)) + " todos")
                self.markAsVoxelyzed(queue_partition)
                self.adjustTraceFile(queue_partition)
                self.traceToDatabase(queue_partition)
                self.calculateOffspring(queue_partition)
                self.moveFiles(queue_partition)
                waitCounter = 0
            else:
                if (self.debug):
                    print("PP: found nothing")
                waitCounter += time.time() - startTime
                startTime = time.time()

            jobsRunning = self.db.getJobsWaitingCount()

            if (self.debug):
                print("PP: {n} jobs currently waiting in LISA queue...".format(n=jobsRunning))
                print("PP: sleeping now for " + str(self.pause_time) + "s")

            self.stopRequest.wait(self.pause_time)

        print ("PP: got exit signal... cleaning up")
        self.observer.join()

    def join(self, timeout=None):
        """ function to terminate the thread (softly)
        :param timeout: not implemented yet
        :return: None
        """
        if (self.debug):
            print("PP: got kill request for thread")
        self.stopRequest.set()
        self.observer.stop()
        super(PostprocessingWorker, self).join(timeout)

    def getIDfromTrace(self, file_path):
        path, filename = os.path.split(file_path)
        name_parts = filename.split(".")
        return name_parts[0]

    def initialDirCheck(self, path):
        """ upon start check if there are files in the target diretory, because the watcher only notices files being moved there while running
        :return: None
        """
        unprocessed = [ os.path.join(path,f) for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) ]
        for todo in unprocessed:
            self.addFile(todo)

    def markAsVoxelyzed(self, todos):
        """ mark all the individuals as voxelyzed, i.e. as successfully processed by Voxelyze
        :param todos: list of strings with trace file paths
        :return: None
        """
        for todo in todos:
            id = self.getIDfromTrace(todo)
            self.db.markAsVoxelyzed(id)
            self.db.setJobDone(id)

    def adjustTraceFile(self, todos):
        """ put the individuals into an arena, correct their coordinates, etc.
        :param todos: list of strings with the individual trace filepaths
        :return: None
        """

        for todo in todos:
            id = self.getIDfromTrace(todo)
            # get initial coordinates from DB
            indiv = self.db.getIndividual(id)
            first_trace = self.db.getFirstTrace(id)
            self.pp.addStartingPointArenaAndTime(todo, self.vox_preamble, self.arena_x, self.arena_y, self.arena_type,
                                                 first_trace["x"], first_trace["y"], indiv["born"], self.end_time)

    def traceToDatabase(self, todos):
        """ put the individuals into the database
        :param todos: list of strings with the individual trace filepaths
        :return: None
        """

        for todo in todos:
            id = self.getIDfromTrace(todo)
            with open(todo, 'r') as inputFile:
                traces = []

                fileAsList = inputFile.readlines()
                for i in range(0, len(fileAsList)):
                    traceLine = fileAsList[i].split()
                    traces.append([id, traceLine[1], traceLine[2], traceLine[3], traceLine[4]])
            if (self.debug):
                print("PP: adding {len} traces for individual {indiv} to DB".format(len=len(traces), indiv=id))
            self.db.addTraces(id, traces)

    def calculateOffspring(self, todos):
        """ yeah, well... generate offspring, calculate where the new individuals met friends on the way
        :param todos: list of strings with the individual IDs
        :return: None
        """
        for todo in todos:
            id = self.getIDfromTrace(todo)
            if (self.debug):
                print("PP: looking for mates for individual {indiv}...".format(indiv=id))
            mates = self.db.findMate(id, self.timeTolerance, self.spaceTolerance)
            i = 0
            while (len(mates) != 0):
                i+=1
                mate = mates[0]
                parent2 = {}
                parent2["id"] = mate["mate_id"]
                parent2["indiv_id"] = mate["mate_indiv_id"]
                parent2["ltime"] = mate["mate_ltime"]
                parent2["x"] = mate["mate_x"]
                parent2["y"] = mate["mate_y"]
                parent2["z"] = mate["mate_z"]
                self.db.makeBaby(mate, parent2, mate["ltime"], self.indiv_max_age*self.indiv_infertile_span)

                newStart = mate["id"]
                mates = self.db.findMate(id, self.timeTolerance, self.spaceTolerance, newStart)

            if (self.debug):
                print("PP: found {len} mating occurances for individual {indiv}".format(len=i, indiv=id))

        self.db.flush()

    def moveFiles(self, todos):
        """ once all preprocessing is done, move the files to their target destination
        :param todos: list of strings with the individual IDs
        :return: None
        """
        for indiv in todos:
            id = self.getIDfromTrace(indiv)
            shutil.move(indiv, self.base_path + self.pop_path + str(id) + ".trace")

    def addFile(self, path):
        self.queue.append(path)

class ChangeHandler(PatternMatchingEventHandler):
    patterns = ["*.trace"]
    pp_worker = None

    def __init__(self, pp_worker, patterns=None, ignore_patterns=None, ignore_directories=False, case_sensitive=False):
        self.pp_worker = pp_worker
        super(ChangeHandler, self).__init__(patterns, ignore_patterns, ignore_directories, case_sensitive)

    def process(self, event):
        """
        event.event_type
            'modified' | 'created' | 'moved' | 'deleted'
        event.is_directory
            True | False
        event.src_path
            path/to/observed/file
        """
        print ("PP: found new trace file:" + event.src_path)
        self.pp_worker.addFile(event.src_path)

    def on_created(self, event):
        self.process(event)
<commit_msg>fixed observer<commit_after>import ConfigParser
import shutil
import threading, time, os
from db import DB
from watchdog.observers import Observer
from watchdog.events import PatternMatchingEventHandler
from preprocessing import Preprocessor

class PostprocessingWorker(threading.Thread):
    """ Python script for Postprocessing worker... runs until cancelled or till max waiting time
    """

    pause_time = 2
    max_waiting_time = 60 * 60  # 60seconds * 60min = 1 hour in seconds
    base_path = ""
    pop_path = "population/"
    traces_path = "traces_afterVox/"
    debug = False
    db = None
    queue = []
    vox_preamble = 8  # number of lines that voxelyze adds before the actual output in a trace file
    config = ConfigParser.RawConfigParser()
    arena_x = 0
    arena_y = 0
    arena_type = ""
    end_time = 0
    timeTolerance = 0.0  # maximum mating time distance
    spaceTolerance = 0.01  # maximum mating distance radius
    pp = Preprocessor()
    indiv_max_age = 0
    indiv_infertile_span = 0.25

    def readConfig(self, config_path):
        self.config.read(config_path)
        self.exp_name = self.config.get('Experiment', 'name')
        self.path_prefix = self.config.get('Experiment', 'path_prefix')
        self.debug = self.config.get('Experiment', 'debug')
        self.base_path = os.path.expanduser(self.path_prefix + self.exp_name) + "/"
        self.pop_path = self.config.get('Postprocessing', 'pop_path')
        self.traces_path = self.config.get('Postprocessing', 'traces_path')
        self.vox_preamble = self.config.getint('Postprocessing', 'vox_preamble')
        self.pause_time = self.config.getint('Workers', 'pause_time')
        self.queue_length = self.config.getint('Workers', 'queue_len')
        self.max_waiting_time = self.config.getint('Workers', 'max_waiting_time')
        self.arena_x = self.config.getfloat('Arena', 'x')
        self.arena_y = self.config.getfloat('Arena', 'y')
        self.arena_type = self.config.get('Arena', 'type')
        self.end_time = self.config.getfloat('Experiment', 'end_time')
        self.indiv_max_age = self.config.getfloat('Population', 'indiv_max_age')

    def __init__(self, dbParams, config_path):
        threading.Thread.__init__(self)
        self.db = DB(dbParams[0], dbParams[1], dbParams[2], dbParams[3])
        self.readConfig(config_path)

        self.stopRequest = threading.Event()
        self.observer = Observer()

    def run(self):
        """ main thread function
        :return: None
        """
        waitCounter = 0
        startTime = time.time()

        obs_path = os.path.normpath(self.base_path + self.traces_path)
        self.observer.schedule(ChangeHandler(self), path=obs_path)
        print("PP: starting file observer on path:\n" + obs_path)
        self.observer.start()

        self.initialDirCheck(obs_path)

        while (not self.stopRequest.isSet() and waitCounter < self.max_waiting_time):
            if (len(self.queue) > 0):
                queue_partition = self.queue
                self.queue = self.queue[len(
                    queue_partition):]  # to make sure that in the short fraction of time no new file was added
                if (self.debug):
                    print("PP: found " + str(len(queue_partition)) + " todos")
                self.markAsVoxelyzed(queue_partition)
                self.adjustTraceFile(queue_partition)
                self.traceToDatabase(queue_partition)
                self.calculateOffspring(queue_partition)
                self.moveFiles(queue_partition)
                waitCounter = 0
            else:
                if (self.debug):
                    print("PP: found nothing")
                waitCounter += time.time() - startTime
                startTime = time.time()

            jobsRunning = self.db.getJobsWaitingCount()

            if (self.debug):
                print("PP: {n} jobs currently waiting in LISA queue...".format(n=jobsRunning))
                print("PP: sleeping now for " + str(self.pause_time) + "s")

            self.stopRequest.wait(self.pause_time)

        print ("PP: got exit signal... cleaning up")
        self.observer.join()

    def join(self, timeout=None):
        """ function to terminate the thread (softly)
        :param timeout: not implemented yet
        :return: None
        """
        if (self.debug):
            print("PP: got kill request for thread")
        self.stopRequest.set()
        self.observer.stop()
        super(PostprocessingWorker, self).join(timeout)

    def getIDfromTrace(self, file_path):
        path, filename = os.path.split(file_path)
        name_parts = filename.split(".")
        return name_parts[0]

    def initialDirCheck(self, path):
        """ upon start check if there are files in the target diretory, because the watcher only notices files being moved there while running
        :return: None
        """
        unprocessed = [ os.path.join(path,f) for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) ]
        for todo in unprocessed:
            self.addFile(todo)

    def markAsVoxelyzed(self, todos):
        """ mark all the individuals as voxelyzed, i.e. as successfully processed by Voxelyze
        :param todos: list of strings with trace file paths
        :return: None
        """
        for todo in todos:
            id = self.getIDfromTrace(todo)
            self.db.markAsVoxelyzed(id)
            self.db.setJobDone(id)

    def adjustTraceFile(self, todos):
        """ put the individuals into an arena, correct their coordinates, etc.
        :param todos: list of strings with the individual trace filepaths
        :return: None
        """

        for todo in todos:
            id = self.getIDfromTrace(todo)
            # get initial coordinates from DB
            indiv = self.db.getIndividual(id)
            first_trace = self.db.getFirstTrace(id)
            self.pp.addStartingPointArenaAndTime(todo, self.vox_preamble, self.arena_x, self.arena_y, self.arena_type,
                                                 first_trace["x"], first_trace["y"], indiv["born"], self.end_time)

    def traceToDatabase(self, todos):
        """ put the individuals into the database
        :param todos: list of strings with the individual trace filepaths
        :return: None
        """

        for todo in todos:
            id = self.getIDfromTrace(todo)
            with open(todo, 'r') as inputFile:
                traces = []

                fileAsList = inputFile.readlines()
                for i in range(0, len(fileAsList)):
                    traceLine = fileAsList[i].split()
                    traces.append([id, traceLine[1], traceLine[2], traceLine[3], traceLine[4]])
            if (self.debug):
                print("PP: adding {len} traces for individual {indiv} to DB".format(len=len(traces), indiv=id))
            self.db.addTraces(id, traces)

    def calculateOffspring(self, todos):
        """ yeah, well... generate offspring, calculate where the new individuals met friends on the way
        :param todos: list of strings with the individual IDs
        :return: None
        """
        for todo in todos:
            id = self.getIDfromTrace(todo)
            if (self.debug):
                print("PP: looking for mates for individual {indiv}...".format(indiv=id))
            mates = self.db.findMate(id, self.timeTolerance, self.spaceTolerance)
            i = 0
            while (len(mates) != 0):
                i+=1
                mate = mates[0]
                parent2 = {}
                parent2["id"] = mate["mate_id"]
                parent2["indiv_id"] = mate["mate_indiv_id"]
                parent2["ltime"] = mate["mate_ltime"]
                parent2["x"] = mate["mate_x"]
                parent2["y"] = mate["mate_y"]
                parent2["z"] = mate["mate_z"]
                self.db.makeBaby(mate, parent2, mate["ltime"], self.indiv_max_age*self.indiv_infertile_span)

                newStart = mate["id"]
                mates = self.db.findMate(id, self.timeTolerance, self.spaceTolerance, newStart)

            if (self.debug):
                print("PP: found {len} mating occurances for individual {indiv}".format(len=i, indiv=id))

        self.db.flush()

    def moveFiles(self, todos):
        """ once all preprocessing is done, move the files to their target destination
        :param todos: list of strings with the individual IDs
        :return: None
        """
        for indiv in todos:
            id = self.getIDfromTrace(indiv)
            shutil.move(indiv, self.base_path + self.pop_path + str(id) + ".trace")

    def addFile(self, path):
        self.queue.append(path)

class ChangeHandler(PatternMatchingEventHandler):
    patterns = ["*.trace"]
    pp_worker = None

    def __init__(self, pp_worker, patterns=None, ignore_patterns=None, ignore_directories=False, case_sensitive=False):
        self.pp_worker = pp_worker
        super(ChangeHandler, self).__init__(patterns, ignore_patterns, ignore_directories, case_sensitive)

    def process(self, event):
        """
        event.event_type
            'modified' | 'created' | 'moved' | 'deleted'
        event.is_directory
            True | False
        event.src_path
            path/to/observed/file
        """
        if (event.event_type == "moved"):
            path = event.dest_path
        else:
            path = event.src_path

        print ("PP: found new trace file:" + path)
        self.pp_worker.addFile(path)

    def on_created(self, event):
        self.process(event)
    def on_modified(self, event):
        self.process(event)
    def on_moved(self, event):
        self.process(event)
<|endoftext|>


<commit_before># Copyright 2011 James McCauley
#
# This file is part of POX.
#
# POX is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# POX is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with POX.  If not, see <http://www.gnu.org/licenses/>.

"""
Revent is a custom event library for POX.

The most common use case of revent is to inherit from EventMixin.

For example:

class Foo (EventMixin):
  def __init__(self):
   # This tells revent that we want to listen to events triggered by pox.core
   self.listenTo(pox.core)

  def _handle_ComponentRegistered(self, event):
    # The name of this method has a special meaning. Any method with a prefix
    # of '_handle_', and a suffix naming an EventType will automatically
    # be registered as an event handler.
    #
    # This method will now be called whenever pox.core triggers a
    # ComponentRegistered event.

    # All event handlers are passed an event object as a second parameter.
    component = event.component
    name = event.name
    print "I see you,", name, "!"

  # A second way to register handlers is to explicitly call self.addListener()
  # For example:
  def bar_handler(self, event):
    print "bar!", event

  # This has the same effect as defining a method called "_handle_UpEvent"
  self.addListener(UpEvent, bar_handler)
"""
import operator
# weakrefs are used for some event handlers.
#
# See: http://docs.python.org/library/weakref.html:
# "A weak reference to an object is not enough to keep the object alive: when the
# only remaining references to a referent are weak references, garbage collection
# is free to destroy the referent"
import weakref

showEventExceptions = False

nextEventID = 0
def generateEventID ():
  global nextEventID
  nextEventID += 1
  return nextEventID

def EventReturn (halt = False, remove = False):
  return (halt, remove)

EventHalt = EventReturn(halt=True)

EventRemove = EventReturn(remove=True)

EventHaltAndRemove = EventReturn(remove=True, halt=True)

class Event (object):
  def __init__ (self):
    self.halt = False
    self.source = None

  def _invoke (self, handler, *args, **kw):
    return handler(self, *args, **kw)

class EventMixin (object):
  """
  Mixin to be inherited from if the subclass is interested in handling events
  """
  # What does (_eventMixin_events == True) signify?
  _eventMixin_events = set()

  def _eventMixin_addEvent (self, eventType):
    self._eventMixin_init()
    if self._eventMixin_events == True:
      # Do nothing, all events already accepted!
      # print warning?
      return
    elif self._eventMixin_events == None:
      self._eventMixin_events = set()
    self._eventMixin_events.add(eventType)

  def __init__ (self):
    self._eventMixin_init()

  def _eventMixin_init (self):
    if not hasattr(self, "_eventMixin_events"):
      setattr(self, "_eventMixin_events", True)
    if not hasattr(self, "_eventMixin_handlers"):
      setattr(self, "_eventMixin_handlers", {})

  def raiseEventNoErrors (self, event, *args, **kw):
    """
    Raise an event, but squelch all exception thrown by handler.
    Also see raiseEvent()
    """
    #TODO: this should really keep subsequent events executing and print the
    #      specific handler that failed...
    try:
      return self.raiseEvent(event, *args, **kw)
    except:
      print "Event handler raised exception"
      if showEventExceptions:
        import traceback
        traceback.print_exc()
    return None

  def raiseEvent (self, event, *args, **kw):
    """
    Raises an event.
    If "event" is an event type, it will be initialized with args and kw, but only
    if there are actually listeners.
    Returns the event object, unless it was never created (because there were no
    listeners) in which case returns None.
    """
    self._eventMixin_init()

    classCall = False
    if isinstance(event, Event):
      eventType = event.__class__
      classCall = True
      if event.source is None: event.source = self
    elif issubclass(event, Event):
      # Check for early-out
      if event not in self._eventMixin_handlers:
        return None
      if len(self._eventMixin_handlers[event]) == 0:
        return None

      classCall = True
      eventType = event
      event = eventType(*args, **kw)
      args = ()
      kw = {}
      if not hasattr(event, 'source') or event.source is None: event.source = self
    #print "raise",event,eventType
    if self._eventMixin_events != True and eventType not in self._eventMixin_events:
      raise RuntimeError("Event " + str(eventType) + " not defined on object of type " + str(type(self)))

    # Create a copy so that it can be modified freely during event processing.
    # It might make sense to change this.
    handlers = self._eventMixin_handlers.get(eventType, [])
    for (priority, handler, once, eid) in handlers:
      if classCall:
        rv = event._invoke(handler, *args, **kw)
      else:
        rv = handler(event, *args, **kw)
      if once: self.removeListener(eid)
      if rv is None: continue
      if rv is False:
        self.removeListener(eid)
      if rv is True:
        break
      if type(rv) == tuple:
        if len(rv) >= 2 and rv[1] == True:
          self.removeListener(eid)
        if len(rv) >= 1 and rv[0]:
          break
        if len(rv) == 0:
          break
      #if classCall and hasattr(event, "halt") and event.halt:
      if classCall and event.halt:
        break
    return event

  def removeListeners (self, listeners):
    altered = False
    for l in listeners:
      altered = altered or self.removeListener(l)
    return altered

  def removeListener (self, handlerOrEID, eventType=None):
    """
    handlerOrEID : either a reference to a handler object, an integer (EID)
                  identifying the event type, or (eventType, EID) pair
    eventType : the type of event to remove the listener(s?) for
    """
    #print "Remove listener", handlerOrEID
    self._eventMixin_init()
    handler = handlerOrEID

    altered = False
    if type(handler) == tuple:
      # It's a type/eid pair
      if eventType == None: eventType = handler[0]
      handlers = self._eventMixin_handlers[eventType]
      l = len(handlers)
      self._eventMixin_handlers[eventType] = [x for x in handlers if x[3] != handler[1]]
      altered = altered or l != len(self._eventMixin_handlers[eventType])
    elif type(handler) == int:
      # It's an EID
      if eventType == None:
        for event in self._eventMixin_handlers:
          handlers = self._eventMixin_handlers[event]
          l = len(handlers)
          self._eventMixin_handlers[event] = [x for x in handlers if x[3] != handler]
          altered = altered or l != len(self._eventMixin_handlers[event])
      else:
        l = len(handlers)
        handlers = self._eventMixin_handlers[eventType]
        self._eventMixin_handlers[eventType] = [x for x in handlers if x[3] != handler]
        altered = altered or l != len(self._eventMixin_handlers[event])
    else:
      if eventType == None:
        for event in self._eventMixin_handlers:
          handlers = self._eventMixin_handlers[event]
          l = len(handlers)
          self._eventMixin_handlers[event] = [x for x in handlers if x[1] != handler]
          altered = altered or l != len(self._eventMixin_handlers[event])
      else:
        handlers = self._eventMixin_handlers[eventType]
        l = len(handlers)
        self._eventMixin_handlers[eventType] = [x for x in handlers if x[1] != handler]
        altered = altered or l != len(self._eventMixin_handlers[eventType])

    return altered

  def addListenerByName (self, *args, **kw):
    """
    Add a listener by name. An eventType argument must be present, which is used
    as the name. A handler argument must also be present.

    Also see addListener().
    """
    kw['byName'] = True
    return self.addListener(*args,**kw)

  def addListener (self, eventType, handler, once=False, weak=False, priority=None, byName=False):
    """
    eventType : event class object (e.g. ConnectionUp). If byName is True, should
                be a string (e.g. "ConnectionUp")
    handler : method object to be invoked on event triggers
    once : whether to stop invoking the handler after the first trigger
    weak : whether to allow the garbage collector to clean up handlers and sources
    priority : the order in which to call event handlers if there are multiple
               for an event type. TODO: priority value may be one of X, Y, or Z
    byName : whether eventType is an event Class or a string name

    Raises an exception if eventType is not in the source's _eventMixin_events list
    """
    self._eventMixin_init()
    if self._eventMixin_events != True and eventType not in self._eventMixin_events:
      # eventType wasn't found
      fail = True
      if byName:
        # if we were supposed to find the event by name, see if one of the event
        # names matches
        for e in self._eventMixin_events:
          if issubclass(e, Event):
            if e.__name__ == eventType:
              eventType = e
              fail = False
              break
      if fail:
        raise RuntimeError("Event " + str(eventType) + " not defined on object of type " + str(type(self)))
    if eventType not in self._eventMixin_handlers:
      # if no handler is already registered, initialize handler_list to []
      handler_list = self._eventMixin_handlers[eventType] = []
      self._eventMixin_handlers[eventType] = handler_list
    else:
      handler_list = self._eventMixin_handlers[eventType]

    eid = generateEventID()

    if weak: handler = CallProxy(self, handler, (eventType, eid))

    entry = (priority, handler, once, eid)
    assert entry not in handler_list # what might cause this to happen?
    handler_list.append(entry)
    if handler_list[0][0] != None: # what if a later element in the tuple has a priority?
      # If priority is specified, sort the event handlers
      handler_list.sort(reverse = True, key = operator.itemgetter(0), cmp =
             lambda a,b: (0 if a is None else a) - (0 if b is None else b) )

    return (eventType,eid)

  def listenTo (self, *args, **kv):
    """
    sink and source arguments must be present
    """
    return autoBindEvents(self, *args, **kv)

  def addListeners (self, sink, prefix='', weak=False):
    return autoBindEvents(sink, self, prefix, weak)

# Not part of the EventMixin class
# (My mental model of Python scoping is thoroughly broken)
def autoBindEvents (sink, source, prefix='', weak=False):
  """
  sink : the class listening to events (EventMixin)
  source : the class triggering events
  prefix : the prefix for the _handle methods. e.g., to listen to
           pox.topology, prefix would be '_pox_core'
  weak : whether
  """
  if len(prefix) > 0 and prefix[0] != '_': prefix = '_' + prefix
  # ignore source if it does not declare that it raises any the revent events
  if hasattr(source, '_eventMixin_events') == False:
    return []

  events = {}
  for e in source._eventMixin_events:
    if type(e) == str:
      events[e] = e
    else:
      events[e.__name__] = e

  listeners = []
  # for each method in sink
  for m in dir(sink):
    # get the method object
    a = getattr(sink, m)
    if callable(a):
      # if it has the revent prefix signature,
      if m.startswith("_handle" + prefix):
        m = m[8+len(prefix):]
        # and it is one of the events our source triggers
        if m in events:
          # append the listener
          listeners.append(source.addListener(events[m], a, weak))
          #print "autoBind: ",source,m,"to",sink

  return listeners

class CallProxy (object):
  """
  Custom proxy wrapper for /weak reference/ event handlers.
  Allows garbage collector to remove this handler, and the source
  object?
  """
  def __init__ (self, source, handler, removeData):
    """
    source :  object raising events
    handler : method to be called upon
    removeData : whether to remove data XXX
    """
    self.source = weakref.ref(source, self.forgetMe)
    self.obj = weakref.ref(handler.im_self, self.forgetMe)
    self.method = handler.im_func
    self.removeData = removeData
    self.name = str(handler)
  def forgetMe (self, o):
    """
    What is the `o` variable used for?
    """
    #print "Forgetting",self.removeData,self.method
    source = self.source()
    if source is not None:
      source.removeListener(self.removeData)
    self.obj = None
  def __call__ (self, *args, **kw):
    #print "weak call"
    if self.obj is None: return
    o = self.obj()
    if o is not None:
      return self.method(o, *args, **kw)
    print "callProxy object is gone!"
    raise RuntimeException("callProxy object is gone!")
  def __str__ (self):
    return "<CallProxy for " + self.name + ">"

"""
TODO
----
decorator for adding event classes to a class?
make mixin-able to existing classes
make mixin-able to existing objects
"""
<commit_msg>revent.py: Adding more comments<commit_after># Copyright 2011 James McCauley
#
# This file is part of POX.
#
# POX is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# POX is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with POX.  If not, see <http://www.gnu.org/licenses/>.

"""
Revent is a custom event library for POX.

The most common use case of revent is to inherit from EventMixin.

For example:

class Foo (EventMixin):
  def __init__(self):
   # This tells revent that we want to listen to events triggered by pox.core
   self.listenTo(pox.core)

  def _handle_ComponentRegistered(self, event):
    # The name of this method has a special meaning. Any method with a prefix
    # of '_handle_', and a suffix naming an EventType will automatically
    # be registered as an event handler.
    #
    # This method will now be called whenever pox.core triggers a
    # ComponentRegistered event.

    # All event handlers are passed an event object as a second parameter.
    component = event.component
    name = event.name
    print "I see you,", name, "!"

  # A second way to register handlers is to explicitly call self.addListener()
  # For example:
  def bar_handler(self, event):
    print "bar!", event

  # This has the same effect as defining a method called "_handle_UpEvent"
  self.addListener(UpEvent, bar_handler)
"""
import operator
# weakrefs are used for some event handlers.
#
# See: http://docs.python.org/library/weakref.html:
# "A weak reference to an object is not enough to keep the object alive: when the
# only remaining references to a referent are weak references, garbage collection
# is free to destroy the referent"
import weakref

showEventExceptions = False

nextEventID = 0
def generateEventID ():
  global nextEventID
  nextEventID += 1
  return nextEventID

def EventReturn (halt = False, remove = False):
  return (halt, remove)

EventHalt = EventReturn(halt=True)

EventRemove = EventReturn(remove=True)

EventHaltAndRemove = EventReturn(remove=True, halt=True)

class Event (object):
  def __init__ (self):
    self.halt = False
    self.source = None

  def _invoke (self, handler, *args, **kw):
    return handler(self, *args, **kw)

class EventMixin (object):
  """
  Mixin to be inherited from if the subclass is interested in handling events
  """
  # What does (_eventMixin_events == True) signify?
  _eventMixin_events = set()

  def _eventMixin_addEvent (self, eventType):
    self._eventMixin_init()
    if self._eventMixin_events == True:
      # Do nothing, all events already accepted!
      # print warning?
      return
    elif self._eventMixin_events == None:
      self._eventMixin_events = set()
    self._eventMixin_events.add(eventType)

  def __init__ (self):
    self._eventMixin_init()

  def _eventMixin_init (self):
    if not hasattr(self, "_eventMixin_events"):
      setattr(self, "_eventMixin_events", True)
    if not hasattr(self, "_eventMixin_handlers"):
      setattr(self, "_eventMixin_handlers", {})

  def raiseEventNoErrors (self, event, *args, **kw):
    """
    Raise an event, but squelch all exception thrown by handler.
    Also see raiseEvent()
    """
    #TODO: this should really keep subsequent events executing and print the
    #      specific handler that failed...
    try:
      return self.raiseEvent(event, *args, **kw)
    except:
      print "Event handler raised exception"
      if showEventExceptions:
        import traceback
        traceback.print_exc()
    return None

  def raiseEvent (self, event, *args, **kw):
    """
    Raises an event.
    If "event" is an event type, it will be initialized with args and kw, but only
    if there are actually listeners.
    Returns the event object, unless it was never created (because there were no
    listeners) in which case returns None.
    """
    self._eventMixin_init()

    classCall = False
    if isinstance(event, Event):
      eventType = event.__class__
      classCall = True
      if event.source is None: event.source = self
    elif issubclass(event, Event):
      # Check for early-out
      if event not in self._eventMixin_handlers:
        return None
      if len(self._eventMixin_handlers[event]) == 0:
        return None

      classCall = True
      eventType = event
      event = eventType(*args, **kw)
      args = ()
      kw = {}
      if not hasattr(event, 'source') or event.source is None: event.source = self
    #print "raise",event,eventType
    if self._eventMixin_events != True and eventType not in self._eventMixin_events:
      raise RuntimeError("Event " + str(eventType) + " not defined on object of type " + str(type(self)))

    # Create a copy so that it can be modified freely during event processing.
    # It might make sense to change this.
    handlers = self._eventMixin_handlers.get(eventType, [])
    for (priority, handler, once, eid) in handlers:
      if classCall:
        rv = event._invoke(handler, *args, **kw)
      else:
        rv = handler(event, *args, **kw)
      if once: self.removeListener(eid)
      if rv is None: continue
      if rv is False:
        self.removeListener(eid)
      if rv is True:
        break
      if type(rv) == tuple:
        if len(rv) >= 2 and rv[1] == True:
          self.removeListener(eid)
        if len(rv) >= 1 and rv[0]:
          break
        if len(rv) == 0:
          break
      #if classCall and hasattr(event, "halt") and event.halt:
      if classCall and event.halt:
        break
    return event

  def removeListeners (self, listeners):
    altered = False
    for l in listeners:
      altered = altered or self.removeListener(l)
    return altered

  def removeListener (self, handlerOrEID, eventType=None):
    """
    handlerOrEID : either a reference to a handler object, an integer (EID)
                  identifying the event type, or (eventType, EID) pair
    eventType : the type of event to remove the listener(s?) for
    """
    #print "Remove listener", handlerOrEID
    self._eventMixin_init()
    handler = handlerOrEID

    altered = False
    if type(handler) == tuple:
      # It's a type/eid pair
      if eventType == None: eventType = handler[0]
      handlers = self._eventMixin_handlers[eventType]
      l = len(handlers)
      self._eventMixin_handlers[eventType] = [x for x in handlers if x[3] != handler[1]]
      altered = altered or l != len(self._eventMixin_handlers[eventType])
    elif type(handler) == int:
      # It's an EID
      if eventType == None:
        for event in self._eventMixin_handlers:
          handlers = self._eventMixin_handlers[event]
          l = len(handlers)
          self._eventMixin_handlers[event] = [x for x in handlers if x[3] != handler]
          altered = altered or l != len(self._eventMixin_handlers[event])
      else:
        l = len(handlers)
        handlers = self._eventMixin_handlers[eventType]
        self._eventMixin_handlers[eventType] = [x for x in handlers if x[3] != handler]
        altered = altered or l != len(self._eventMixin_handlers[event])
    else:
      if eventType == None:
        for event in self._eventMixin_handlers:
          handlers = self._eventMixin_handlers[event]
          l = len(handlers)
          self._eventMixin_handlers[event] = [x for x in handlers if x[1] != handler]
          altered = altered or l != len(self._eventMixin_handlers[event])
      else:
        handlers = self._eventMixin_handlers[eventType]
        l = len(handlers)
        self._eventMixin_handlers[eventType] = [x for x in handlers if x[1] != handler]
        altered = altered or l != len(self._eventMixin_handlers[eventType])

    return altered

  def addListenerByName (self, *args, **kw):
    """
    Add a listener by name. An eventType argument must be present, which is used
    as the name. A handler argument must also be present.

    Also see addListener().
    """
    kw['byName'] = True
    return self.addListener(*args,**kw)

  def addListener (self, eventType, handler, once=False, weak=False, priority=None, byName=False):
    """
    Add an event handler for an event triggered by this object.

    eventType : event class object (e.g. ConnectionUp). If byName is True, should
                be a string (e.g. "ConnectionUp")
    handler : method object to be invoked on event triggers
    once : whether to stop invoking the handler after the first trigger
    weak : whether to allow the garbage collector to clean up handlers and sources
    priority : the order in which to call event handlers if there are multiple
               for an event type. TODO: priority value may be one of X, Y, or Z
    byName : whether eventType is an event Class or a string name

    Raises an exception if eventType is not in the source's _eventMixin_events list
    """
    self._eventMixin_init()
    if self._eventMixin_events != True and eventType not in self._eventMixin_events:
      # eventType wasn't found
      fail = True
      if byName:
        # if we were supposed to find the event by name, see if one of the event
        # names matches
        for e in self._eventMixin_events:
          if issubclass(e, Event):
            if e.__name__ == eventType:
              eventType = e
              fail = False
              break
      if fail:
        raise RuntimeError("Event " + str(eventType) + " not defined on object of type " + str(type(self)))
    if eventType not in self._eventMixin_handlers:
      # if no handler is already registered, initialize handler_list to []
      handler_list = self._eventMixin_handlers[eventType] = []
      self._eventMixin_handlers[eventType] = handler_list
    else:
      handler_list = self._eventMixin_handlers[eventType]

    eid = generateEventID()

    if weak: handler = CallProxy(self, handler, (eventType, eid))

    entry = (priority, handler, once, eid)
    assert entry not in handler_list # what might cause this to happen?
    handler_list.append(entry)
    if handler_list[0][0] != None: # what if a later element in the tuple has a priority?
      # If priority is specified, sort the event handlers
      handler_list.sort(reverse = True, key = operator.itemgetter(0), cmp =
             lambda a,b: (0 if a is None else a) - (0 if b is None else b) )

    return (eventType,eid)

  def listenTo (self, *args, **kv):
    """
    source argument must be present

    sink is set to self
    """
    return autoBindEvents(self, *args, **kv)

  def addListeners (self, sink, prefix='', weak=False):
    """
    Reflection foo: pick up all _handle methods defined by sink, and make sure
    they are executed when this object triggers the corresponding events.

    sink - the object to trigger listeners for

    source is set to self
    """
    return autoBindEvents(sink, self, prefix, weak)

# Not part of the EventMixin class
# (My mental model of Python scoping is thoroughly broken)
def autoBindEvents (sink, source, prefix='', weak=False):
  """
  sink : the class listening to events (EventMixin)
  source : the class triggering events
  prefix : the prefix for the _handle methods. e.g., to listen to
           pox.topology, prefix would be '_pox_core'
  weak : whether
  """
  if len(prefix) > 0 and prefix[0] != '_': prefix = '_' + prefix
  # ignore source if it does not declare that it raises any the revent events
  if hasattr(source, '_eventMixin_events') == False:
    return []

  events = {}
  for e in source._eventMixin_events:
    if type(e) == str:
      events[e] = e
    else:
      events[e.__name__] = e

  listeners = []
  # for each method in sink
  for m in dir(sink):
    # get the method object
    a = getattr(sink, m)
    if callable(a):
      # if it has the revent prefix signature,
      if m.startswith("_handle" + prefix):
        m = m[8+len(prefix):]
        # and it is one of the events our source triggers
        if m in events:
          # append the listener
          listeners.append(source.addListener(events[m], a, weak))
          #print "autoBind: ",source,m,"to",sink

  return listeners

class CallProxy (object):
  """
  Custom proxy wrapper for /weak reference/ event handlers.
  Allows garbage collector to remove this handler, and the source
  object?
  """
  def __init__ (self, source, handler, removeData):
    """
    source :  object raising events
    handler : method to be called upon
    removeData : whether to remove data XXX
    """
    self.source = weakref.ref(source, self.forgetMe)
    self.obj = weakref.ref(handler.im_self, self.forgetMe)
    self.method = handler.im_func
    self.removeData = removeData
    self.name = str(handler)
  def forgetMe (self, o):
    """
    What is the `o` variable used for?
    """
    #print "Forgetting",self.removeData,self.method
    source = self.source()
    if source is not None:
      source.removeListener(self.removeData)
    self.obj = None
  def __call__ (self, *args, **kw):
    #print "weak call"
    if self.obj is None: return
    o = self.obj()
    if o is not None:
      return self.method(o, *args, **kw)
    print "callProxy object is gone!"
    raise RuntimeException("callProxy object is gone!")
  def __str__ (self):
    return "<CallProxy for " + self.name + ">"

"""
TODO
----
decorator for adding event classes to a class?
make mixin-able to existing classes
make mixin-able to existing objects
"""
<|endoftext|>