<commit_before>"""
High-level user-level commands controlling the entire simulation.
$Id$
"""
__version__='$Revision$'

import pickle
import __main__

import topo

from topo.base.parameterizedobject import ParameterizedObject, Parameter
from topo.base.functionfamilies import OutputFn
from topo.base.sheet import Sheet
from topo.base.cf import CFSheet
from topo.base.projection import ProjectionSheet
from topo.sheets.generatorsheet import GeneratorSheet
from topo.misc.utils import get_states_of_classes_from_module

def save_input_generators():
    """Save a copy of the active_sim's current input_generators for all GeneratorSheets."""
    generator_sheets = topo.sim.objects(GeneratorSheet).values()
    for sheet in generator_sheets:
        sheet.push_input_generator()

def restore_input_generators():
    """Restore previously saved input_generators for all of topo.sim's GeneratorSheets."""
    generator_sheets = topo.sim.objects(GeneratorSheet).values()
    for sheet in generator_sheets:
        sheet.pop_input_generator()

### JABHACKALERT!  Should leave the state of all learning flags
### unchanged upon return; currently it's overwriting all of them.
def pattern_present(inputs={},duration=1.0,learning=False,overwrite_previous=False,apply_output_fn=True):
    """
    Present the specified test patterns for the specified duration.

    Given a set of input patterns (dictionary of
    GeneratorSheetName:PatternGenerator pairs), installs them into the
    specified GeneratorSheets, runs the simulation for the specified
    length of time, then restores the original patterns and the
    original simulation time.  Thus this input is not considered part
    of the regular simulation, and is usually for testing purposes.

    If a simulation is not provided, the active simulation, if one
    exists, is requested.

    If this process is interrupted by the user, the temporary patterns
    may still be installed on the retina.

    If overwrite_previous is true, the given inputs overwrite those
    previously defined.

    If learning is False, overwrites the existing values of Sheet.learning
    to disable learning, then reenables learning.
    """

    if not overwrite_previous:
        save_input_generators()

    ### JABALERT!  Should clean up how these are set on each
    ### sheet; it overwrites any old values.

    # turn off sheets' learning and output function learning
    #(e.g. in Homeostatic output functions) if learning=False
    if not learning:
        for each in topo.sim.objects(Sheet).values():
             each.learning = False
        for each in topo.sim.objects(ProjectionSheet).values():
            each.output_fn.learning = False

    if not apply_output_fn:
        for each in topo.sim.objects(Sheet).values():
             each.apply_output_fn = False

    gen_eps_list = topo.sim.objects(GeneratorSheet)

    # Register the inputs on each input sheet
    for each in inputs.keys():
        if gen_eps_list.has_key(each):
            gen_eps_list[each].set_input_generator(inputs[each])
        else:
            ParameterizedObject().warning('%s not a valid Sheet Name.' % each)

    topo.sim.event_push()
    topo.sim.run(duration)
    topo.sim.event_pop()

    # turn sheets' learning and output_fn learning back on if we turned it off before
    if not learning:
        for each in topo.sim.objects(Sheet).values():
            each.learning = True
        for each in topo.sim.objects(ProjectionSheet).values():
            each.output_fn.learning = True

    if not apply_output_fn:
        for each in topo.sim.objects(Sheet).values():
            each.apply_output_fn = True

    if not overwrite_previous:
        restore_input_generators()

def save_snapshot(snapshot_name):
    """
    Save a snapshot of the network's current state.

    Commands stored in topo.sim.startup_commands will be executed
    by the corresponding load_snapshot() function.

    Uses Python's 'pickle' module, so subject to the same limitations
    (see the pickle module's documentation) - with the notable
    exception of class attributes. Python does not pickle class
    attributes, but this function stores class attributes of any
    ParameterizedObject class that is declared within the topo
    package.

    (Subpackages of topo that are not part of the simulation are
    excluded from having their classes' attributes saved: plotting,
    tkgui, and tests.)
    """
    ### Warn that classes & functions defined in __main__ won't unpickle
    #
    import types
    for k,v in __main__.__dict__.items():
        # there's classes and functions...what else?
        if isinstance(v,type) or isinstance(v,types.FunctionType):
            if v.__module__ == "__main__":
                ParameterizedObject().warning("%s (type %s) has source in __main__; it will only be found on unpickling if the class is explicitly defined (e.g. by running the same script first) before unpickling."%(k,type(v)))

    ### Get ParameterizedObject class attributes
    #
    states_of_classes = {}

    # For now we just search topo, but it could be extended to all packages.
    # We exclude certain subpackages that contain classes that aren't part
    # of the simulation.
    exclude = ('plotting','tkgui','tests')
    get_states_of_classes_from_module(topo,states_of_classes,[],exclude)

    ### Set the release version for this simulation.
    #
    topo.sim.RELEASE = topo.release

    ### Now pickle the lot to a file
    #
    pickle.dump( (topo.sim.actual_sim,states_of_classes), open(snapshot_name,'wb'),2)

def load_snapshot(snapshot_name):
    """
    Load the simulation stored in snapshot_name.

    Unpickles the simulation, sets ParameterizedObject class
    attributes, and executes commands in topo.sim.startup_commands
    """
    sim,states_of_classes = pickle.load(open(snapshot_name,'rb'))

    topo.sim.change_sim(sim)

    ### Set class attributes
    #
    # (i.e. execute in __main__ the command "path.to.module.Class.x=y" for each class)
    for class_name,state in states_of_classes.items():

        # from "topo.base.parameter.Parameter", we want "topo.base.parameter"
        module_path = class_name[0:class_name.rindex('.')]
        exec 'import '+module_path in __main__.__dict__

        # now restore class Parameter values
        for p_name,p in state.items():
            __main__.__dict__['val'] = p
            try:
                exec 'setattr('+class_name+',"'+p_name+'",val)' in __main__.__dict__
            except:
                ParameterizedObject().warning('Problem restoring parameter %s=%s for class %s; name may have changed since the snapshot was created.' % (p_name,repr(p),class_name))

    ### Execute the startup commands
    #
    for cmd in topo.sim.startup_commands:
        exec cmd in __main__.__dict__

<commit_msg>Make snapshots be gzip'd.<commit_after>"""
High-level user-level commands controlling the entire simulation.
$Id$
"""
__version__='$Revision$'

import pickle
import __main__
import gzip

import topo

from topo.base.parameterizedobject import ParameterizedObject, Parameter
from topo.base.functionfamilies import OutputFn
from topo.base.sheet import Sheet
from topo.base.cf import CFSheet
from topo.base.projection import ProjectionSheet
from topo.sheets.generatorsheet import GeneratorSheet
from topo.misc.utils import get_states_of_classes_from_module

def save_input_generators():
    """Save a copy of the active_sim's current input_generators for all GeneratorSheets."""
    generator_sheets = topo.sim.objects(GeneratorSheet).values()
    for sheet in generator_sheets:
        sheet.push_input_generator()

def restore_input_generators():
    """Restore previously saved input_generators for all of topo.sim's GeneratorSheets."""
    generator_sheets = topo.sim.objects(GeneratorSheet).values()
    for sheet in generator_sheets:
        sheet.pop_input_generator()

### JABHACKALERT!  Should leave the state of all learning flags
### unchanged upon return; currently it's overwriting all of them.
def pattern_present(inputs={},duration=1.0,learning=False,overwrite_previous=False,apply_output_fn=True):
    """
    Present the specified test patterns for the specified duration.

    Given a set of input patterns (dictionary of
    GeneratorSheetName:PatternGenerator pairs), installs them into the
    specified GeneratorSheets, runs the simulation for the specified
    length of time, then restores the original patterns and the
    original simulation time.  Thus this input is not considered part
    of the regular simulation, and is usually for testing purposes.

    If a simulation is not provided, the active simulation, if one
    exists, is requested.

    If this process is interrupted by the user, the temporary patterns
    may still be installed on the retina.

    If overwrite_previous is true, the given inputs overwrite those
    previously defined.

    If learning is False, overwrites the existing values of Sheet.learning
    to disable learning, then reenables learning.
    """

    if not overwrite_previous:
        save_input_generators()

    ### JABALERT!  Should clean up how these are set on each
    ### sheet; it overwrites any old values.

    # turn off sheets' learning and output function learning
    #(e.g. in Homeostatic output functions) if learning=False
    if not learning:
        for each in topo.sim.objects(Sheet).values():
             each.learning = False
        for each in topo.sim.objects(ProjectionSheet).values():
            each.output_fn.learning = False

    if not apply_output_fn:
        for each in topo.sim.objects(Sheet).values():
             each.apply_output_fn = False

    gen_eps_list = topo.sim.objects(GeneratorSheet)

    # Register the inputs on each input sheet
    for each in inputs.keys():
        if gen_eps_list.has_key(each):
            gen_eps_list[each].set_input_generator(inputs[each])
        else:
            ParameterizedObject().warning('%s not a valid Sheet Name.' % each)

    topo.sim.event_push()
    topo.sim.run(duration)
    topo.sim.event_pop()

    # turn sheets' learning and output_fn learning back on if we turned it off before
    if not learning:
        for each in topo.sim.objects(Sheet).values():
            each.learning = True
        for each in topo.sim.objects(ProjectionSheet).values():
            each.output_fn.learning = True

    if not apply_output_fn:
        for each in topo.sim.objects(Sheet).values():
            each.apply_output_fn = True

    if not overwrite_previous:
        restore_input_generators()

def save_snapshot(snapshot_name):
    """
    Save a snapshot of the network's current state.

    Commands stored in topo.sim.startup_commands will be executed
    by the corresponding load_snapshot() function.

    Uses Python's 'pickle' module, so subject to the same limitations
    (see the pickle module's documentation) - with the notable
    exception of class attributes. Python does not pickle class
    attributes, but this function stores class attributes of any
    ParameterizedObject class that is declared within the topo
    package.

    (Subpackages of topo that are not part of the simulation are
    excluded from having their classes' attributes saved: plotting,
    tkgui, and tests.)
    """
    ### Warn that classes & functions defined in __main__ won't unpickle
    #
    import types
    for k,v in __main__.__dict__.items():
        # there's classes and functions...what else?
        if isinstance(v,type) or isinstance(v,types.FunctionType):
            if v.__module__ == "__main__":
                ParameterizedObject().warning("%s (type %s) has source in __main__; it will only be found on unpickling if the class is explicitly defined (e.g. by running the same script first) before unpickling."%(k,type(v)))

    ### Get ParameterizedObject class attributes
    #
    states_of_classes = {}

    # For now we just search topo, but it could be extended to all packages.
    # We exclude certain subpackages that contain classes that aren't part
    # of the simulation.
    exclude = ('plotting','tkgui','tests')
    get_states_of_classes_from_module(topo,states_of_classes,[],exclude)

    ### Set the release version for this simulation.
    #
    topo.sim.RELEASE = topo.release

    ### Now pickle the lot to a file
    #
    pickle.dump( (topo.sim.actual_sim,states_of_classes), gzip.open(snapshot_name,'wb'),2)

def load_snapshot(snapshot_name):
    """
    Load the simulation stored in snapshot_name.

    Unpickles the simulation, sets ParameterizedObject class
    attributes, and executes commands in topo.sim.startup_commands
    """
    sim,states_of_classes = pickle.load(gzip.open(snapshot_name,'rb'))

    topo.sim.change_sim(sim)

    ### Set class attributes
    #
    # (i.e. execute in __main__ the command "path.to.module.Class.x=y" for each class)
    for class_name,state in states_of_classes.items():

        # from "topo.base.parameter.Parameter", we want "topo.base.parameter"
        module_path = class_name[0:class_name.rindex('.')]
        exec 'import '+module_path in __main__.__dict__

        # now restore class Parameter values
        for p_name,p in state.items():
            __main__.__dict__['val'] = p
            try:
                exec 'setattr('+class_name+',"'+p_name+'",val)' in __main__.__dict__
            except:
                ParameterizedObject().warning('Problem restoring parameter %s=%s for class %s; name may have changed since the snapshot was created.' % (p_name,repr(p),class_name))

    ### Execute the startup commands
    #
    for cmd in topo.sim.startup_commands:
        exec cmd in __main__.__dict__

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Dang Mai <contact@dangmai.net>.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

"""Generates smart playlists based on beets queries.
"""

from __future__ import division, absolute_import, print_function

from beets.plugins import BeetsPlugin
from beets import ui
from beets.util import mkdirall, normpath, syspath, bytestring_path
from beets.library import Item, Album, parse_query_string
from beets.dbcore import OrQuery
from beets.dbcore.query import MultipleSort, ParsingError
import os
import six

class SmartPlaylistPlugin(BeetsPlugin):

    def __init__(self):
        super(SmartPlaylistPlugin, self).__init__()
        self.config.add({
            'relative_to': None,
            'playlist_dir': u'.',
            'auto': True,
            'playlists': []
        })

        self._matched_playlists = None
        self._unmatched_playlists = None

        if self.config['auto']:
            self.register_listener('database_change', self.db_change)

    def commands(self):
        spl_update = ui.Subcommand(
            'splupdate',
            help=u'update the smart playlists. Playlist names may be '
            u'passed as arguments.'
        )
        spl_update.func = self.update_cmd
        return [spl_update]

    def update_cmd(self, lib, opts, args):
        self.build_queries()
        if args:
            args = set(ui.decargs(args))
            for a in list(args):
                if not a.endswith(".m3u"):
                    args.add("{0}.m3u".format(a))

            playlists = set((name, q, a_q)
                            for name, q, a_q in self._unmatched_playlists
                            if name in args)
            if not playlists:
                raise ui.UserError(
                    u'No playlist matching any of {0} found'.format(
                        [name for name, _, _ in self._unmatched_playlists])
                )

            self._matched_playlists = playlists
            self._unmatched_playlists -= playlists
        else:
            self._matched_playlists = self._unmatched_playlists

        self.update_playlists(lib)

    def build_queries(self):
        """
        Instanciate queries for the playlists.

        Each playlist has 2 queries: one or items one for albums, each with a
        sort. We must also remember its name. _unmatched_playlists is a set of
        tuples (name, (q, q_sort), (album_q, album_q_sort)).

        sort may be any sort, or NullSort, or None. None and NullSort are
        equivalent and both eval to False.
        More precisely
        - it will be NullSort when a playlist query ('query' or 'album_query')
          is a single item or a list with 1 element
        - it will be None when there are multiple items i a query
        """
        self._unmatched_playlists = set()
        self._matched_playlists = set()

        for playlist in self.config['playlists'].get(list):
            if 'name' not in playlist:
                self._log.warning(u"playlist configuration is missing name")
                continue

            playlist_data = (playlist['name'],)
            try:
                for key, Model in (('query', Item), ('album_query', Album)):
                    qs = playlist.get(key)
                    if qs is None:
                        query_and_sort = None, None
                    elif isinstance(qs, six.string_types):
                        query_and_sort = parse_query_string(qs, Model)
                    elif len(qs) == 1:
                        query_and_sort = parse_query_string(qs[0], Model)
                    else:
                        # multiple queries and sorts
                        queries, sorts = zip(*(parse_query_string(q, Model)
                                               for q in qs))
                        query = OrQuery(queries)
                        final_sorts = []
                        for s in sorts:
                            if s:
                                if isinstance(s, MultipleSort):
                                    final_sorts += s.sorts
                                else:
                                    final_sorts.append(s)
                        if not final_sorts:
                            sort = None
                        elif len(final_sorts) == 1:
                            sort, = final_sorts
                        else:
                            sort = MultipleSort(final_sorts)
                        query_and_sort = query, sort

                    playlist_data += (query_and_sort,)

            except ParsingError as exc:
                self._log.warning(u"invalid query in playlist {}: {}",
                                  playlist['name'], exc)
                continue

            self._unmatched_playlists.add(playlist_data)

    def matches(self, model, query, album_query):
        if album_query and isinstance(model, Album):
            return album_query.match(model)
        if query and isinstance(model, Item):
            return query.match(model)
        return False

    def db_change(self, lib, model):
        if self._unmatched_playlists is None:
            self.build_queries()

        for playlist in self._unmatched_playlists:
            n, (q, _), (a_q, _) = playlist
            if self.matches(model, q, a_q):
                self._log.debug(
                    u"{0} will be updated because of {1}", n, model)
                self._matched_playlists.add(playlist)
                self.register_listener('cli_exit', self.update_playlists)

        self._unmatched_playlists -= self._matched_playlists

    def update_playlists(self, lib):
        self._log.info(u"Updating {0} smart playlists...",
                       len(self._matched_playlists))

        playlist_dir = self.config['playlist_dir'].as_filename()
        playlist_dir = bytestring_path(playlist_dir)
        relative_to = self.config['relative_to'].get()
        if relative_to:
            relative_to = normpath(relative_to)

        for playlist in self._matched_playlists:
            name, (query, q_sort), (album_query, a_q_sort) = playlist
            self._log.debug(u"Creating playlist {0}", name)
            items = []

            if query:
                items.extend(lib.items(query, q_sort))
            if album_query:
                for album in lib.albums(album_query, a_q_sort):
                    items.extend(album.items())

            m3us = {}
            # As we allow tags in the m3u names, we'll need to iterate through
            # the items and generate the correct m3u file names.
            for item in items:
                m3u_name = item.evaluate_template(name, True)
                if m3u_name not in m3us:
                    m3us[m3u_name] = []
                item_path = item.path
                if relative_to:
                    item_path = os.path.relpath(item.path, relative_to)
                if item_path not in m3us[m3u_name]:
                    m3us[m3u_name].append(item_path)
            # Now iterate through the m3us that we need to generate
            for m3u in m3us:
                m3u_path = normpath(os.path.join(playlist_dir,
                                    bytestring_path(m3u)))
                mkdirall(m3u_path)
                with open(syspath(m3u_path), 'wb') as f:
                    for path in m3us[m3u]:
                        f.write(path + b'\n')

        self._log.info(u"{0} playlists updated", len(self._matched_playlists))
<commit_msg>sanitize playlist names<commit_after># -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Dang Mai <contact@dangmai.net>.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

"""Generates smart playlists based on beets queries.
"""

from __future__ import division, absolute_import, print_function

from beets.plugins import BeetsPlugin
from beets import ui
from beets.util import mkdirall, normpath, sanitize_path, syspath, bytestring_path
from beets.library import Item, Album, parse_query_string
from beets.dbcore import OrQuery
from beets.dbcore.query import MultipleSort, ParsingError
import os
import six

class SmartPlaylistPlugin(BeetsPlugin):

    def __init__(self):
        super(SmartPlaylistPlugin, self).__init__()
        self.config.add({
            'relative_to': None,
            'playlist_dir': u'.',
            'auto': True,
            'playlists': []
        })

        self._matched_playlists = None
        self._unmatched_playlists = None

        if self.config['auto']:
            self.register_listener('database_change', self.db_change)

    def commands(self):
        spl_update = ui.Subcommand(
            'splupdate',
            help=u'update the smart playlists. Playlist names may be '
            u'passed as arguments.'
        )
        spl_update.func = self.update_cmd
        return [spl_update]

    def update_cmd(self, lib, opts, args):
        self.build_queries()
        if args:
            args = set(ui.decargs(args))
            for a in list(args):
                if not a.endswith(".m3u"):
                    args.add("{0}.m3u".format(a))

            playlists = set((name, q, a_q)
                            for name, q, a_q in self._unmatched_playlists
                            if name in args)
            if not playlists:
                raise ui.UserError(
                    u'No playlist matching any of {0} found'.format(
                        [name for name, _, _ in self._unmatched_playlists])
                )

            self._matched_playlists = playlists
            self._unmatched_playlists -= playlists
        else:
            self._matched_playlists = self._unmatched_playlists

        self.update_playlists(lib)

    def build_queries(self):
        """
        Instanciate queries for the playlists.

        Each playlist has 2 queries: one or items one for albums, each with a
        sort. We must also remember its name. _unmatched_playlists is a set of
        tuples (name, (q, q_sort), (album_q, album_q_sort)).

        sort may be any sort, or NullSort, or None. None and NullSort are
        equivalent and both eval to False.
        More precisely
        - it will be NullSort when a playlist query ('query' or 'album_query')
          is a single item or a list with 1 element
        - it will be None when there are multiple items i a query
        """
        self._unmatched_playlists = set()
        self._matched_playlists = set()

        for playlist in self.config['playlists'].get(list):
            if 'name' not in playlist:
                self._log.warning(u"playlist configuration is missing name")
                continue

            playlist_data = (playlist['name'],)
            try:
                for key, Model in (('query', Item), ('album_query', Album)):
                    qs = playlist.get(key)
                    if qs is None:
                        query_and_sort = None, None
                    elif isinstance(qs, six.string_types):
                        query_and_sort = parse_query_string(qs, Model)
                    elif len(qs) == 1:
                        query_and_sort = parse_query_string(qs[0], Model)
                    else:
                        # multiple queries and sorts
                        queries, sorts = zip(*(parse_query_string(q, Model)
                                               for q in qs))
                        query = OrQuery(queries)
                        final_sorts = []
                        for s in sorts:
                            if s:
                                if isinstance(s, MultipleSort):
                                    final_sorts += s.sorts
                                else:
                                    final_sorts.append(s)
                        if not final_sorts:
                            sort = None
                        elif len(final_sorts) == 1:
                            sort, = final_sorts
                        else:
                            sort = MultipleSort(final_sorts)
                        query_and_sort = query, sort

                    playlist_data += (query_and_sort,)

            except ParsingError as exc:
                self._log.warning(u"invalid query in playlist {}: {}",
                                  playlist['name'], exc)
                continue

            self._unmatched_playlists.add(playlist_data)

    def matches(self, model, query, album_query):
        if album_query and isinstance(model, Album):
            return album_query.match(model)
        if query and isinstance(model, Item):
            return query.match(model)
        return False

    def db_change(self, lib, model):
        if self._unmatched_playlists is None:
            self.build_queries()

        for playlist in self._unmatched_playlists:
            n, (q, _), (a_q, _) = playlist
            if self.matches(model, q, a_q):
                self._log.debug(
                    u"{0} will be updated because of {1}", n, model)
                self._matched_playlists.add(playlist)
                self.register_listener('cli_exit', self.update_playlists)

        self._unmatched_playlists -= self._matched_playlists

    def update_playlists(self, lib):
        self._log.info(u"Updating {0} smart playlists...",
                       len(self._matched_playlists))

        playlist_dir = self.config['playlist_dir'].as_filename()
        playlist_dir = bytestring_path(playlist_dir)
        relative_to = self.config['relative_to'].get()
        if relative_to:
            relative_to = normpath(relative_to)

        for playlist in self._matched_playlists:
            name, (query, q_sort), (album_query, a_q_sort) = playlist
            self._log.debug(u"Creating playlist {0}", name)
            items = []

            if query:
                items.extend(lib.items(query, q_sort))
            if album_query:
                for album in lib.albums(album_query, a_q_sort):
                    items.extend(album.items())

            m3us = {}
            # As we allow tags in the m3u names, we'll need to iterate through
            # the items and generate the correct m3u file names.
            for item in items:
                m3u_name = sanitize_path(item.evaluate_template(name, True), lib.replacements)
                if m3u_name not in m3us:
                    m3us[m3u_name] = []
                item_path = item.path
                if relative_to:
                    item_path = os.path.relpath(item.path, relative_to)
                if item_path not in m3us[m3u_name]:
                    m3us[m3u_name].append(item_path)
            # Now iterate through the m3us that we need to generate
            for m3u in m3us:
                m3u_path = normpath(os.path.join(playlist_dir,
                                    bytestring_path(m3u)))
                mkdirall(m3u_path)
                with open(syspath(m3u_path), 'wb') as f:
                    for path in m3us[m3u]:
                        f.write(path + b'\n')

        self._log.info(u"{0} playlists updated", len(self._matched_playlists))
<|endoftext|>


<commit_before># Copyright 2014, 2015 The ODL development group
#
# This file is part of ODL.
#
# ODL is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# ODL is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with ODL.  If not, see <http://www.gnu.org/licenses/>.

# Imports for common Python 2/3 codebase
from __future__ import print_function, division, absolute_import

from builtins import super
from future import standard_library
standard_library.install_aliases()

# ODL imports
from odl.space.fspace import FunctionSpace
from odl.set.sets import RealNumbers

__all__ = ('L2',)

class L2(FunctionSpace):
    """The space of square integrable functions on some domain."""

    def __init__(self, domain, field=RealNumbers()):
        super().__init__(domain, field)

    def _inner(self, v1, v2):
        """Inner product, not computable in continuous spaces."""
        raise NotImplementedError('inner product not computable in the'
                                  'non-discretized space {}.'.format(self))

    def __str__(self):
        if isinstance(self.field, RealNumbers):
            return 'L2({})'.format(self.domain)
        else:
            return 'L2({}, {})'.format(self.domain, self.field)

    def __repr__(self):
        if isinstance(self.field, RealNumbers):
            return 'L2({!r})'.format(self.domain)
        else:
            return 'L2({!r}, {!r})'.format(self.domain, self.field)

    class Vector(FunctionSpace.Vector):
        """Representation of an `L2` element."""

if __name__ == '__main__':
    from doctest import testmod, NORMALIZE_WHITESPACE
    testmod(optionflags=NORMALIZE_WHITESPACE)
<commit_msg>doc of default now 0 warnings<commit_after># Copyright 2014, 2015 The ODL development group
#
# This file is part of ODL.
#
# ODL is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# ODL is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with ODL.  If not, see <http://www.gnu.org/licenses/>.

# Imports for common Python 2/3 codebase
from __future__ import print_function, division, absolute_import

from builtins import super
from future import standard_library
standard_library.install_aliases()

# ODL imports
from odl.space.fspace import FunctionSpace
from odl.set.sets import RealNumbers

__all__ = ('L2',)

class L2(FunctionSpace):
    """The space of square integrable functions on some domain."""

    def __init__(self, domain, field=RealNumbers()):
        super().__init__(domain, field)

    def _inner(self, v1, v2):
        """Inner product, not computable in continuous spaces."""
        raise NotImplementedError('inner product not computable in the'
                                  'non-discretized space {}.'.format(self))

    def __str__(self):
        if isinstance(self.field, RealNumbers):
            return 'L2({})'.format(self.domain)
        else:
            return 'L2({}, {})'.format(self.domain, self.field)

    def __repr__(self):
        if isinstance(self.field, RealNumbers):
            return 'L2({!r})'.format(self.domain)
        else:
            return 'L2({!r}, {!r})'.format(self.domain, self.field)

    class Vector(FunctionSpace.Vector):
        """Representation of an :class:`L2` element."""

if __name__ == '__main__':
    from doctest import testmod, NORMALIZE_WHITESPACE
    testmod(optionflags=NORMALIZE_WHITESPACE)
<|endoftext|>


<commit_before># genson_parser.py
#
# Based on json parser code from Paul McGuire, 2007
#
#
genson_bnf = """
object
    { members }
    {}
key
    string
    key_tuple
key_tuple
    ( key_list )
key_list
    string
    key_list, string
members
    key : value
    members , key : value
value_tuple
    ( elements )
    ()
array
    [ elements ]
    []
elements
    value
    elements , value
generator
    kw ()
    kw (args)
    kw (kwargs)
    kw (args, kwargs)
args
    elements
kwargs
    kw = value
    kwargs , kw = value
value
    string
    number
    object
    array
    true
    false
    value_tuple
    null
"""

from pyparsing import *
from parameter_generators import *

# a simple helper functions
def generator(name, gen_args=[], gen_kwargs={}):

    generator_class = generators.get(name, None)
    if generator_class is None:
        raise Exception('Unknown generator class: %s' % name)

    if type(gen_kwargs) is ParseResults:
        gen_kwargs = gen_kwargs.asList()
    gen_kwargs = dict(gen_kwargs)

    g = generator_class(*gen_args, **gen_kwargs)

    return g

TRUE = Keyword("true").setParseAction( replaceWith(True) )
FALSE = Keyword("false").setParseAction( replaceWith(False) )
NULL = Keyword("null").setParseAction( replaceWith(None) )

json_string = dblQuotedString.setParseAction( removeQuotes )
json_number = Combine( Optional('-') + ( '0' | Word('123456789',nums) ) +
                    Optional( '.' + Word(nums) ) +
                    Optional( Word('eE',exact=1) + Word(nums+'+-',nums) ) )

genson_key_tuple = Forward()
genson_key = ( json_string | genson_key_tuple )
genson_key_tuple << Suppress('(') + delimitedList( genson_key ) + \
                         Suppress(')')
genson_key_tuple.setParseAction(tuple)

json_object = Forward()
json_value = Forward()
json_elements = delimitedList( json_value )
json_array = Group(Suppress('[') + Optional(json_elements) + Suppress(']') )

genson_value_tuple = Suppress('(') + json_elements + Suppress(')')
genson_value_tuple.setParseAction(tuple)

genson_kwargs = Group(delimitedList( Word(alphas) + Suppress("=") + \
                              json_value ))
genson_generator =  Word(alphas)("name") + \
                    Suppress('(') + \
                    Optional(json_elements)("args") + \
                    Optional(Suppress(',')) +\
                    Optional(genson_kwargs)("kwargs") + \
                    Suppress(')')
genson_generator.setParseAction(lambda x: generator(x.name, x.args, x.kwargs))

genson_grid_shorthand = Suppress("<") + \
                       json_elements("args") + \
                       Suppress(">")
genson_grid_shorthand.setParseAction(lambda x: generator("grid", x.args))

json_value << ( genson_value_tuple | genson_generator | \
                genson_grid_shorthand | \
                json_string | json_number | json_object | \
                json_array | TRUE | FALSE | NULL )

member_def = Group( genson_key + Suppress(':') + json_value )
json_members = delimitedList( member_def )
json_object << Dict( Suppress('{') + Optional(json_members) + Suppress('}') )

json_comment = cppStyleComment
json_object.ignore( json_comment )

def clean_dict(x):
    x_list = x.asList()
    return dict(x_list)

json_object.setParseAction(clean_dict)

def convert_numbers(s,l,toks):
    n = toks[0]
    try:
        return int(n)
    except ValueError, ve:
        return float(n)

json_number.setParseAction( convert_numbers )

class GENSONParser:
    def __init__(self):
        self.grammar = json_object

    def parse_string(self, genson_string):
        result = self.grammar.parseString(genson_string)
        return result.asList()[0]

<commit_msg>Small modification of setParseAction(tuple) calls to allow this code to run on my system<commit_after># genson_parser.py
#
# Based on json parser code from Paul McGuire, 2007
#
#
genson_bnf = """
object
    { members }
    {}
key
    string
    key_tuple
key_tuple
    ( key_list )
key_list
    string
    key_list, string
members
    key : value
    members , key : value
value_tuple
    ( elements )
    ()
array
    [ elements ]
    []
elements
    value
    elements , value
generator
    kw ()
    kw (args)
    kw (kwargs)
    kw (args, kwargs)
args
    elements
kwargs
    kw = value
    kwargs , kw = value
value
    string
    number
    object
    array
    true
    false
    value_tuple
    null
"""

from pyparsing import *
from parameter_generators import *

def make_tuple(x):
    return tuple(x)

# a simple helper functions
def generator(name, gen_args=[], gen_kwargs={}):

    generator_class = generators.get(name, None)
    if generator_class is None:
        raise Exception('Unknown generator class: %s' % name)

    if type(gen_kwargs) is ParseResults:
        gen_kwargs = gen_kwargs.asList()
    gen_kwargs = dict(gen_kwargs)

    g = generator_class(*gen_args, **gen_kwargs)

    return g

TRUE = Keyword("true").setParseAction( replaceWith(True) )
FALSE = Keyword("false").setParseAction( replaceWith(False) )
NULL = Keyword("null").setParseAction( replaceWith(None) )

json_string = dblQuotedString.setParseAction( removeQuotes )
json_number = Combine( Optional('-') + ( '0' | Word('123456789',nums) ) +
                    Optional( '.' + Word(nums) ) +
                    Optional( Word('eE',exact=1) + Word(nums+'+-',nums) ) )

genson_key_tuple = Forward()
genson_key = ( json_string | genson_key_tuple )
genson_key_tuple << Suppress('(') + delimitedList( genson_key ) + \
                         Suppress(')')
# genson_key_tuple.setParseAction(tuple)
genson_key_tuple.setParseAction(make_tuple)

json_object = Forward()
json_value = Forward()
json_elements = delimitedList( json_value )
json_array = Group(Suppress('[') + Optional(json_elements) + Suppress(']') )

genson_value_tuple = Suppress('(') + json_elements + Suppress(')')
# genson_value_tuple.setParseAction(tuple)
genson_value_tuple.setParseAction(make_tuple)

genson_kwargs = Group(delimitedList( Word(alphas) + Suppress("=") + \
                              json_value ))
genson_generator =  Word(alphas)("name") + \
                    Suppress('(') + \
                    Optional(json_elements)("args") + \
                    Optional(Suppress(',')) +\
                    Optional(genson_kwargs)("kwargs") + \
                    Suppress(')')
genson_generator.setParseAction(lambda x: generator(x.name, x.args, x.kwargs))

genson_grid_shorthand = Suppress("<") + \
                       json_elements("args") + \
                       Suppress(">")
genson_grid_shorthand.setParseAction(lambda x: generator("grid", x.args))

json_value << ( genson_value_tuple | genson_generator | \
                genson_grid_shorthand | \
                json_string | json_number | json_object | \
                json_array | TRUE | FALSE | NULL )

member_def = Group( genson_key + Suppress(':') + json_value )
json_members = delimitedList( member_def )
json_object << Dict( Suppress('{') + Optional(json_members) + Suppress('}') )

json_comment = cppStyleComment
json_object.ignore( json_comment )

def clean_dict(x):
    x_list = x.asList()
    return dict(x_list)

json_object.setParseAction(clean_dict)

def convert_numbers(s,l,toks):
    n = toks[0]
    try:
        return int(n)
    except ValueError, ve:
        return float(n)

json_number.setParseAction( convert_numbers )

class GENSONParser:
    def __init__(self):
        self.grammar = json_object

    def parse_string(self, genson_string):
        result = self.grammar.parseString(genson_string)
        return result.asList()[0]

<|endoftext|>


<commit_before>"""
swat_controller

based on http://www.irongeek.com/i.php?page=security/security-and-software-defined-networking-sdn-openflow

ARP poisoning resistant: passive, active, internal node, external node

the controller statically maps all switches with the ideal configuration (proactive) saving time in the init
process,

the component is event-driven and detection and mitigation code runs with higher priority than normal code.
it will be easy to develop new detection and mitigation modules and swap them into the component.

normal task mgmt is based on l2_learning provided component.

eg: if attacker detected then tell the switch to send all packet from that switch to a dedicated IDS port
eg2: if attacker detected then exclude the node from the network
eg3: reverse MITM attack using the attacker as a target and a hidden node as is target
"""

from pox.core import core
import pox.openflow.libopenflow_01 as of
from pox.lib.recoco import Timer
from pox.lib.util import dpidToStr
from pox.lib.addresses import IPAddr, EthAddr
from pox.lib.revent import Event, EventMixin, EventHalt

from pprint import pformat

log = core.getLogger()

# TODO:
#       check periodically the consistency of the static mapping quering the switches.
#       add nicira self-learning capability to auto-proove consistency of the mapping
#       blacklist arp spoofer

class ArpPoison(Event):

    """ArpPoison event"""

    def __init__(self):
        Event.__init__(self)

class AntiArpPosoning(object):

    """
    Class able to detect datapath ArpPoisoning

    one-time/switch ConnectionUp event is handled by _init_static function
    outside this class.
    """

    def __init__(self, event, ip_to_mac, ips_port, flood_port ):
        """
        blocking handlers: self._detect_arp_poison
        """
        self.event = event
        self.connection = event.connection  # convenient reference
        self.connection._eventMixin_events.add(ArpPoison)
        self.ip_to_mac = ip_to_mac
        self.ips_port = ips_port
        self.flood_port = flood_port

        # debug attributes
        # aap_log = pformat(self.__dict__, indent=4)
        # log.debug("self.__dict__: %s" % aap_log)
        # connection_log = pformat(event.connection._eventMixin_events, indent=4)
        # log.debug("event.connection: %s" % connection_log)

        self.connection.addListenerByName("PacketIn", self._detect_arp_poison, priority=1, once=False)
        self.connection.addListenerByName("PacketIn", self._handle_PacketIn, priority=0, once=False)
        self.connection.addListenerByName("ArpPoison", self._handle_ArpPoison, priority=1, once=False)

    def drop(self, event):
        """
        Drop a packet
        """
        pass

    def flood(self, event):
        """
        Flood a packet
        """
        pass

    def arp_detector_1(self, event):
        """
        Try to detect arp poisoning

        :returns: flag
        """
        arp_poisoning = False

        return arp_poisoning

    def arp_detector_2(self, event):
        """
        Try to detect arp poisoning

        :returns: flag
        """
        arp_poisoning = False

        return arp_poisoning

    def _detect_arp_poison(self, event):
        """
        if event contains arp poisoning raise
        and ArpPoison event and block lower priority
        event handling chain.

        eg: PacketIn won't trigger _handle_PacketIn
        """

        arp_poisoning = self.arp_detector_1(event)
        # arp_poisoning = self.arp_detector_2(event)

        if arp_poisoning:
            self.connection.raiseEvent(ArpPoison)
            return EventHalt
        else:
            pass

    def ap_handler_1(self, event):
        """
        redirect attacker traffic to IPS
        update all other switches
        """
        pass

    def ap_handler_2(self, event):
        """
        kick-out the attacker
        update all other switches
        """
        pass

    def _handle_ArpPoison(self, event):
        """
        Use it to update other switches
        """
        log.info("_handle_ArpPoison: %d" % self.event.dpid)

        pass
        # ap_handler_1(event)
        # ap_handler_2(event)

    def _handle_ConnectionDown(self, event):
        """
        TODO
        """
        pass

    def _handle_PacketIn(self, event):
        """
        Std PacketIn handling
        """
        log.info("_handle_PacketIn: %d" % self.event.dpid)

        # if multicast:
        #     flood()
        # elif already_mapped:
        #     add_flow(source, destination)
        #     add_flow(destination, source)
        # else:
        #     save_new_mapping
        #     flood()

    def _handle_FlowRemoved(self, event):
        """
        TODO
        """
        pass

def swat_map1():
    """
    six plcs and hmi.
    """

    ip_to_mac = {
            '192.168.1.10':  '00:1D:9C:C7:B0:70',
            '192.168.1.20':  '00:1D:9C:C8:BC:46',
            '192.168.1.30':  '00:1D:9C:C8:BD:F2',
            '192.168.1.40':  '00:1D:9C:C7:FA:2C',
            '192.168.1.50':  '00:1D:9C:C8:BC:2F',
            '192.168.1.60':  '00:1D:9C:C7:FA:2D',
            '192.168.1.100': '00:1D:9C:C6:72:E8',
            }

    return ip_to_mac

def _init_static(event):
    """
    Create a static mapping (proactive)

    init switch flooding and IPS ports

    init AntiArpPosoning datapath interface

    """
    log.info("_init_static: %d" % event.dpid)

    ip_to_mac = swat_map1()

    flood_port = of.OFPP_FLOOD
    # flood_port = of.OFPP_ALL

    ips_port = 4000  # used to redirect suspect traffic
    # timeout = 10  # sec, still NOT used

    AntiArpPosoning(event, ip_to_mac, ips_port, flood_port)

def launch(par=False):
    """
    _init_static_mapping listens to each ConnectionUp event.
    """

    core.openflow.addListenerByName("ConnectionUp", _init_static, priority=5)
<commit_msg>Add flood to swat_controller<commit_after>"""
swat_controller

based on http://www.irongeek.com/i.php?page=security/security-and-software-defined-networking-sdn-openflow

ARP poisoning resistant: passive, active, internal node, external node

the controller statically maps all switches with the ideal configuration (proactive) saving time in the init
process,

the component is event-driven and detection and mitigation code runs with higher priority than normal code.
it will be easy to develop new detection and mitigation modules and swap them into the component.

normal task mgmt is based on l2_learning provided component.

eg: if attacker detected then tell the switch to send all packet from that switch to a dedicated IDS port
eg2: if attacker detected then exclude the node from the network
eg3: reverse MITM attack using the attacker as a target and a hidden node as is target
"""

from pox.core import core
import pox.openflow.libopenflow_01 as of
from pox.lib.recoco import Timer
from pox.lib.util import dpidToStr
from pox.lib.addresses import IPAddr, EthAddr
from pox.lib.revent import Event, EventMixin, EventHalt

from pprint import pformat

log = core.getLogger()

# TODO:
#       check periodically the consistency of the static mapping quering the switches.
#       add nicira self-learning capability to auto-proove consistency of the mapping
#       blacklist arp spoofer
#       add switch-based init mac_to_port mapping? AntiArpPoison store a mac_to_port map
#       and send flow modification on connection up and remove flows on connection down

class ArpPoison(Event):

    """ArpPoison event"""

    def __init__(self):
        Event.__init__(self)

class AntiArpPoison(object):

    """
    Class able to detect datapath ArpPoisoning

    one-time/switch ConnectionUp event is handled by _init_static function
    outside this class.
    """

    def __init__(self, event, ip_to_mac, ips_port, flood_port ):
        """
        blocking handlers: self._detect_arp_poison blocks PacketIn handling
        """

        self.connection = event.connection  # convenient reference
        self.connection._eventMixin_events.add(ArpPoison)
        self.ip_to_mac = ip_to_mac
        self.ips_port = ips_port
        self.flood_port = flood_port

        # debug attributes
        # aap_log = pformat(self.__dict__, indent=4)
        # log.debug("self.__dict__: %s" % aap_log)
        # connection_log = pformat(event.connection._eventMixin_events, indent=4)
        # log.debug("event.connection: %s" % connection_log)

        self.connection.addListenerByName("ConnectionUp", self._handle_ConnectionUp, priority=1, once=False)
        self.connection.addListenerByName("PacketIn", self._detect_arp_poison, priority=1, once=False)
        self.connection.addListenerByName("PacketIn", self._handle_PacketIn, priority=0, once=False)
        self.connection.addListenerByName("ArpPoison", self._handle_ArpPoison, priority=1, once=False)

    def drop(self, event):
        """
        Drop a packet
        """
        pass

    def flood(self, event):
        """
        Flood a packet
        """
        flood_port = self.flood_port
        packet = event.parsed
        log.debug("flood: dataptath %i: %s -> %s" % (event.dpid, packet.src, packet.dst))

        msg = of.ofp_packet_out()
        action = of.ofp_action_output(port=flood_port)
        msg.actions.append(action)

        msg.data = event.ofp  # PacketOut payload in the same as PacketIn
        log.debug(str(msg.in_port))
        msg.in_port = event.port
        log.debug(str(msg.in_port))

        self.connection.send(msg)

    def arp_detector_1(self, event):
        """
        Try to detect arp poisoning

        :returns: flag
        """
        arp_poisoning = False

        return arp_poisoning

    def arp_detector_2(self, event):
        """
        Try to detect arp poisoning

        :returns: flag
        """
        arp_poisoning = False

        return arp_poisoning

    def _detect_arp_poison(self, event):
        """
        if event contains arp poisoning raise
        and ArpPoison event and block lower priority
        event handling chain.

        eg: PacketIn won't trigger _handle_PacketIn
        """

        arp_poisoning = self.arp_detector_1(event)
        # arp_poisoning = self.arp_detector_2(event)

        if arp_poisoning:
            self.connection.raiseEvent(ArpPoison)
            return EventHalt
        else:
            pass

    def ap_handler_1(self, event):
        """
        redirect attacker traffic to IPS
        update all other switches
        """
        pass

    def ap_handler_2(self, event):
        """
        kick-out the attacker
        update all other switches
        """
        pass

    def _handle_ArpPoison(self, event):
        """
        Use it to update other switches
        """
        log.info("_handle_ArpPoison: %d" % event.dpid)

        # ap_handler_1(event)
        # ap_handler_2(event)

    def _handle_ConnectionUp(self, event):
        """
        TODO: send flowmods to statically init the switch
        """
        log.info("_handle_ConnectionUp: %d" % event.dpid)

    def _handle_ConnectionDown(self, event):
        """
        TODO: remove all flow from the switch
        """
        log.info("_handle_ConnectionDown: %d" % event.dpid)

    def _handle_PacketIn(self, event):
        """
        Std PacketIn handling
        """
        log.info("_handle_PacketIn: %d" % event.dpid)

        self.flood(event)
        # if multicast: # dl_dst = ff:ff:ff:ff:ff:ff
        #     flood()
        # elif already_mapped:
        #     add_flow(source, destination)
        #     add_flow(destination, source)
        # else:
        #     save_new_mapping
        #     flood()

    def _handle_FlowRemoved(self, event):
        """
        TODO
        """
        pass

def swat_map1():
    """
    six plcs and hmi.
    """

    ip_to_mac = {
            '192.168.1.10':  '00:1D:9C:C7:B0:70',
            '192.168.1.20':  '00:1D:9C:C8:BC:46',
            '192.168.1.30':  '00:1D:9C:C8:BD:F2',
            '192.168.1.40':  '00:1D:9C:C7:FA:2C',
            '192.168.1.50':  '00:1D:9C:C8:BC:2F',
            '192.168.1.60':  '00:1D:9C:C7:FA:2D',
            '192.168.1.100': '00:1D:9C:C6:72:E8',
            }

    return ip_to_mac

def _init_static(event):
    """
    Create a static mapping (proactive)

    init switch flooding and IPS ports

    init AntiArpPoison datapath interface

    """
    log.info("_init_static: %d" % event.dpid)

    ip_to_mac = swat_map1()

    flood_port = of.OFPP_FLOOD
    # flood_port = of.OFPP_ALL

    ips_port = 4000  # used to redirect suspect traffic
    # timeout = 10  # sec, still NOT used

    AntiArpPoison(event, ip_to_mac, ips_port, flood_port)

def launch(par=False):
    """
    _init_static_mapping listens to each ConnectionUp event.
    """

    core.openflow.addListenerByName("ConnectionUp", _init_static, priority=5)
<|endoftext|>


<commit_before>#!/bin/env python

import subprocess
import json

def changes():
    s = subprocess.getoutput("ssh openbmc.gerrit gerrit query " +
                             "--format json --all-reviewers " +
                             "--current-patch-set -- " +
                             "'age:1d status:open -is:draft " +
                             "label:Code-Review>=-1 " +
                             "-project:openbmc/openbmc-test-automation'")

    changes = list(map(json.loads, s.splitlines()))
    del changes[-1]

    return changes

for c in changes():
    print(c['subject'])

<commit_msg>Initial delay reason assessment<commit_after>#!/bin/env python

import subprocess
import json

def changes():
    s = subprocess.getoutput("ssh openbmc.gerrit gerrit query " +
                             "--format json --all-reviewers " +
                             "--current-patch-set -- " +
                             "'age:1d status:open -is:draft " +
                             "label:Code-Review>=-1 " +
                             "-project:openbmc/openbmc-test-automation'")

    changes = list(map(json.loads, s.splitlines()))
    del changes[-1]

    return changes

username_map = {
    'jenkins-openbmc': "Jenkins",
    'williamspatrick': "@iawillia",
    'bradbishop': "@bradleyb"
}

def map_username(user, name):
    return username_map.get(user, "[%s: %s]" % (user, name))

def map_approvals(approvals):
    mapped = {}
    for a in approvals:
        approval_type = a['type']
        approval_owner = map_username(a['by']['username'], a['by']['name'])
        approval_score = int(a['value'])

        if approval_type not in mapped:
            mapped[approval_type] = {}

        mapped[approval_type][approval_owner] = approval_score

    return mapped

def map_reviewers(reviewers, owner):
    mapped = []
    for r in reviewers:
        reviewer_user = r['username']
        reviewer_name = r['name']

        if reviewer_user == 'jenkins-openbmc':
            continue

        reviewer_username = map_username(r['username'], r['name'])

        if reviewer_username == owner:
            continue

        mapped.append(reviewer_username)

    return mapped

def reason(change):
    subject = change['subject']
    owner = map_username(change['owner']['username'], change['owner']['name'])
    reviewers = map_reviewers(change['allReviewers'], owner)
    approvals = map_approvals(change['currentPatchSet']['approvals'])

#    print("----")
#    print(subject)
#    print(owner)
#    print(approvals)
#    print(reviewers)
#    print("----")

    if len(reviewers) < 2:
        return "%s has added insufficient reviewers." % owner

    if ('Verified' in approvals):
        verified = approvals['Verified']
        scores = list(filter(lambda x: verified[x] < 0, verified))
        if len(scores):
            return "%s should resolve verification failure." % owner

    if ('Code-Review' not in approvals):
        return "Missing code review by %s." % (", ".join(reviewers))

    reviewed = approvals['Code-Review']
    rejected_by = list(filter(lambda x: reviewed[x] < 0, reviewed))
    if len(rejected_by):
        return "%s should resolve code review comments." % owner

    reviewed_by = list(filter(lambda x: reviewed[x] > 0, reviewed))
    if len(reviewed_by) < 2:
        return "Missing code review by %s." % \
               (", ".join(set(reviewers) - set(reviewed_by)))

    if ('Verified' not in approvals):
        return "May be missing Jenkins verification."

    approved_by = list(filter(lambda x: reviewed[x] == 2, reviewed))
    if len(approved_by):
        return "Ready for merge by %s." % (", ".join(approved_by))
    else:
        return "Awaiting merge review."

for c in changes():
    print(c['subject'])
    print(c['url'])
    print(reason(c))
    print("----")

<|endoftext|>


<commit_before>"""
Commands that were in the Makefile for running various tests. Roughly
converted to a Python script so we can run on all platforms. Work in
progress.

Buildbot shows how to run the tests, but here are some examples:

unit:
./topographica -p 'targets=["unit"]' topo/tests/runtests.py

default set:
./topographica topo/tests/runtests.py

all:
./topographica -p 'targets=["all"]' topo/tests/runtests.py

Ideally, the 'unit' target (i.e. unit tests) would be all that people
need to run. Developers, for checking they haven't messed obvious
stuff up. Users, for checking a Topographica installation functions ok
on their system. The other, slower targets would only need to be run
by buildbot (on behalf of everyone).

Unfortunately, the unit tests do not currently cover enough! It's an
important project to improve them. Until then, the default is to run
more of the tests (but not all, since that takes too long).
"""

# CEBALERT: need to fix the issue with global_params reporting name=X
# is unused.

# Comments from the Makefile that I haven't processed yet:
#
# (1)
# CB: Beyond 14 dp, the results of the current tests do not match on
# ppc64 and i686 (using linux).  In the future, decimal=14 might have
# to be reduced (if the tests change, or to accommodate other
# processors/platforms).
#

import glob
import os
import sys
import tempfile
import commands

import param
from topo.misc.commandline import global_params as p

p.add(
    targets = param.List(default=[],doc="Leave empty for default set, ['all'] for all tests except speedtests, ['speed'] for speed tests, or else list the targets required."),

    coverage = param.Boolean(default=False),

    timing = param.Boolean(default=False),

    testdp = param.Number(default=7),

    testdp_unopt = param.Number(default=5)

    )

if p.coverage:
    coverage_cmd = "bin/coverage run --rcfile=doc/buildbot/coveragerc -a -p"
else:
    coverage_cmd = ""

if p.timing:
    timing_cmd = "/usr/bin/time"
else:
    timing_cmd = ""

# http://stackoverflow.com/questions/377017/test-if-executable-exists-in-python
def which(program):
    import os
    def is_exe(fpath):
        return os.path.exists(fpath) and os.access(fpath, os.X_OK)

    fpath, fname = os.path.split(program)
    if fpath:
        if is_exe(program):
            return program
    else:
        for path in os.environ["PATH"].split(os.pathsep):
            exe_file = os.path.join(path, program)
            if is_exe(exe_file):
                return exe_file
    return ""

xvfb = which('xvfb-run')
if not xvfb:
    print "xvfb-run not found; any GUI components that are run will display windows"
else:
    xvfb = xvfb + " -a"

if len(p.targets)==0:
    # DEFAULT
    p.targets = ['unit','traintests','snapshots','gui','maptests'] # maptests wouldn't be default except it's caught platform different problems before (there aren't enough unit tests!)

elif p.targets == ['allsnapshottests']:
    # CEBALERT: should just combine these tests anyway
    p.targets = ['snapshots','pickle']

# ->params ?
tests_dir = param.resolve_path("topo/tests",path_to_file=False)
scripts_dir = param.resolve_path(".",path_to_file=False) ### XXX
topographica_script = xvfb + " " + timing_cmd + coverage_cmd + " " + sys.argv[0] + " " +  " "

def _runc(cmd):
    print cmd
    return os.system(cmd)

import topo.misc.keyedlist
target = topo.misc.keyedlist.KeyedList()
speedtarget = topo.misc.keyedlist.KeyedList()

# CEBALERT: need to pick which scripts to include for traintests and
# speedtests and startupspeedtests (see test_script.py).
#
# From the Makefile:
# SCRIPTS= ^hierarchical.ty ^lissom_or.ty ^lissom_oo_or.ty ^som_retinotopy.ty ^sullivan_neurocomputing04.ty ^lissom.ty ^lissom_fsa.ty ^gcal.ty ^lissom_whisker_barrels.ty
# CEB: tests on these scripts temporarily suspended (SF.net #2053538)
# ^lissom_oo_or_homeostatic.ty ^lissom_oo_or_homeostatic_tracked.ty
# ^lissom_or_noshrinking.ty  - only matches to 4 dp with IMPORT_WEAVE=0
# Now I'm using this list for train-tests:

# CEBALERT: this list should be defined in one place.
#from setup import TRAINSCRIPTS
TRAINSCRIPTS = [
    "examples/hierarchical.ty",
    "models/lissom_or.ty",
    "models/lissom_oo_or.ty",
    "examples/som_retinotopy.ty",
    "models/sullivan_neurocomputing04.ty",
    "models/lissom.ty",
#    "models/lissom_fsa.ty",  # CEBALERT: disabled for now (needs special case - look_at=fsa not v1, see below)
    "examples/gcal.ty"
    ]

# (and a different list for speedtests - see test_script.py).
#
# Also: special cases from the Makefile to consider restoring:
#
##topo/tests/lissom.ty_DATA:
##	./topographica -c 'from topo.tests.test_script import generate_data; generate_data(script="examples/lissom.ty",data_filename="tests/lissom.ty_DATA",run_for=[1,99,150],look_at="V1",cortex_density=8,retina_density=6,lgn_density=6,dims=["or","od","dr","dy","cr","sf"])'
##
##topo/tests/lissom_fsa.ty_DATA:
##	./topographica -c 'from topo.tests.test_script import generate_data; generate_data(script="examples/lissom_fsa.ty",data_filename="tests/lissom_fsa.ty_DATA",run_for=[1,99,150],look_at="FSA",cortex_density=8,retina_density=24,lgn_density=24)'
##
##topo/tests/lissom_whisker_barrels.ty_DATA:
##	./topographica -c 'from topo.tests.test_script import generate_data; generate_data(script="examples/lissom_whisker_barrels.ty",data_filename="tests/lissom_whisker_barrels.ty_DATA",run_for=[1,99,150],look_at="S1")'

target['traintests'] = []
for script in TRAINSCRIPTS:
    script_path = os.path.join(scripts_dir,script)
    target['traintests'].append(topographica_script +  ''' -c "from topo.tests.test_script import test_script; test_script(script=%(script_path)s,decimal=%(dp)s)"'''%dict(script_path=repr(script_path),dp=p.testdp))

# CEBALERT: should use the code above but just with the changes for running without weave.
target['unopttraintests'] = []
for script in TRAINSCRIPTS:
    script_path = os.path.join(scripts_dir,script)
    target['unopttraintests'].append(topographica_script + " -c 'import_weave=False'" +  " -c 'from topo.tests.test_script import test_script; test_script(script=\"%(script_path)s\",decimal=%(dp)s)'"%dict(script_path=script_path,dp=p.testdp_unopt))

speedtarget['speedtests'] = []
SPEEDSCRIPTS = TRAINSCRIPTS
SPEEDSCRIPTS.remove("examples/hierarchical.ty") # CEBALERT: remove problematic example (doesn't work for some densities)
for script in SPEEDSCRIPTS:
    script_path = os.path.join(scripts_dir,script)
    speedtarget['speedtests'].append(topographica_script +  ''' -c "from topo.tests.test_script import compare_speed_data;compare_speed_data(script=%(script_path)s)"'''%dict(script_path=repr(script_path)))

STARTUPSPEEDSCRIPTS = ["models/lissom.ty","examples/gcal.ty"]

speedtarget['startupspeedtests'] = []
for script in STARTUPSPEEDSCRIPTS:
    script_path = os.path.join(scripts_dir,script)
    speedtarget['startupspeedtests'].append(topographica_script +  ''' -c "from topo.tests.test_script import compare_startup_speed_data;compare_startup_speed_data(script=%(script_path)s)"'''%dict(script_path=repr(script_path)))

##### snapshot-tests
target['snapshots'] = []

# snapshot-compatibility-tests:
snapshot_path = os.path.join(tests_dir,"lissom_oo_or_od_dr_cr_dy_sf_000010.00.typ")
target['snapshots'].append(topographica_script + ''' -c "from topo.command import load_snapshot; load_snapshot(%(snapshot_path)s)" -c "topo.sim.run(1)"'''%dict(snapshot_path=repr(snapshot_path)))

# Test that simulations give the same results whether run straight
# through or run part way, saved, reloaded, and run on to the same
# point.
# simulation-snapshot-tests:
target['snapshots'].append(topographica_script + ''' -c "from topo.tests.test_script import compare_with_and_without_snapshot_NoSnapshot as A; A()"''')

target['snapshots'].append(topographica_script + ''' -c "from topo.tests.test_script import compare_with_and_without_snapshot_CreateSnapshot as B; B()"''')

target['snapshots'].append(topographica_script + ''' -c "from topo.tests.test_script import compare_with_and_without_snapshot_LoadSnapshot as C; C()"''')

# CEBALERT: remove ~/topographica/*PICKLETEST* ?

target['pickle'] = []
# pickle-all-classes:
target['pickle'].append(topographica_script + ''' -c "from topo.tests.test_script import pickle_unpickle_everything; errs = pickle_unpickle_everything(); import sys; sys.exit(errs)"''')

# unpickle-compatibility-tests:
pickle_path = os.path.join(tests_dir,"instances-r11275.pickle")
target['pickle'].append(topographica_script + '''-l -c "from topo.tests.test_script import pickle_unpickle_everything; errs = pickle_unpickle_everything(existing_pickles=%(pickle_path)s); import sys; sys.exit(errs)"'''%dict(pickle_path=repr(pickle_path)))

# CEBALERT: hack that this will always be created even when test not being run
# DSALERT: also it's not portable; python has a tempfile module that should
# easily be able to replace this
tmpd = commands.getoutput("mktemp -d")
#script-repr-tests:
target['scriptrepr']=[]
script = os.path.join(scripts_dir,"examples/hierarchical.ty")
target['scriptrepr'].append(topographica_script + " %(script)s -a -c \"import param;param.normalize_path.prefix='%(tmpd)s'\" -c \"save_script_repr('script_repr_test.ty')\""%dict(tmpd=tmpd,script=script))

script_repr_test_path = os.path.join(tmpd,"script_repr_test.ty")
target['scriptrepr'].append(topographica_script + " " + script_repr_test_path)
target['scriptrepr'].append(topographica_script + "-c \"import shutil;shutil.rmtree('%s')\""%tmpd)

### GUI tests
target['gui'] = []
# basic-gui-tests:
target['gui'].append(topographica_script + ' -g -c "from topo.tests.gui_tests import run_basic; nerr=run_basic();topo.guimain.quit_topographica(check=False,exit_status=nerr)"')

# detailed-gui-tests:
target['gui'].append(topographica_script + ' -g -c "from topo.tests.gui_tests import run_detailed; nerr=run_detailed(); topo.guimain.quit_topographica(check=False,exit_status=nerr)"')

target['batch'] = []
target['batch'].append(topographica_script + ' -c "from topo.tests.test_script import test_runbatch; test_runbatch()"')

target['unit'] = []
target['unit'].append(topographica_script + ' -c "import topo.tests; t=topo.tests.run(); import sys; sys.exit(len(t.failures+t.errors))"')

# CEBALERT: should use lissom.ty and test more map types
# pass a list of plotgroup names to test() instead of plotgroups_to_test to restrict the tests
target['maptests'] = []
script = os.path.join(scripts_dir,"models/lissom_oo_or.ty")
target['maptests'].append(topographica_script + ' -c "cortex_density=8" %s -c "topo.sim.run(100);from topo.tests.test_map_measurement import *; test(plotgroups_to_test)"'%script)

def start():
    errors = []

    print "Running: %s"%p.targets
    print

    # CEBALERT: rename one of target or targets!

    if p.targets==['all']:
        targets = target.keys()
    elif p.targets==['speed']:
        targets = speedtarget.keys()
    else:
        targets = p.targets

    target.update(speedtarget)

    for name in targets:
        print "*** " + name
        for cmd in target[name]:
            if _runc(cmd) > 0:
                if name not in errors:
                    errors.append(name)

    print
    print "="*60
    print
    print "runtests.start(): targets with errors: %s"%len(errors)
    if len(errors)>0:
        print errors
        sys.exit(1)
    else:
        sys.exit(0)

if __name__=="__main__":
    start()

<commit_msg>Allow 'allsnapshottests' shortcut to work with other targets.<commit_after>"""
Commands that were in the Makefile for running various tests. Roughly
converted to a Python script so we can run on all platforms. Work in
progress.

Buildbot shows how to run the tests, but here are some examples:

unit:
./topographica -p 'targets=["unit"]' topo/tests/runtests.py

default set:
./topographica topo/tests/runtests.py

all:
./topographica -p 'targets=["all"]' topo/tests/runtests.py

Ideally, the 'unit' target (i.e. unit tests) would be all that people
need to run. Developers, for checking they haven't messed obvious
stuff up. Users, for checking a Topographica installation functions ok
on their system. The other, slower targets would only need to be run
by buildbot (on behalf of everyone).

Unfortunately, the unit tests do not currently cover enough! It's an
important project to improve them. Until then, the default is to run
more of the tests (but not all, since that takes too long).
"""

# CEBALERT: need to fix the issue with global_params reporting name=X
# is unused.

# Comments from the Makefile that I haven't processed yet:
#
# (1)
# CB: Beyond 14 dp, the results of the current tests do not match on
# ppc64 and i686 (using linux).  In the future, decimal=14 might have
# to be reduced (if the tests change, or to accommodate other
# processors/platforms).
#

import glob
import os
import sys
import tempfile
import commands

import param
from topo.misc.commandline import global_params as p

p.add(
    targets = param.List(default=[],doc="Leave empty for default set, ['all'] for all tests except speedtests, ['speed'] for speed tests, or else list the targets required."),

    coverage = param.Boolean(default=False),

    timing = param.Boolean(default=False),

    testdp = param.Number(default=7),

    testdp_unopt = param.Number(default=5)

    )

if p.coverage:
    coverage_cmd = "bin/coverage run --rcfile=doc/buildbot/coveragerc -a -p"
else:
    coverage_cmd = ""

if p.timing:
    timing_cmd = "/usr/bin/time"
else:
    timing_cmd = ""

# http://stackoverflow.com/questions/377017/test-if-executable-exists-in-python
def which(program):
    import os
    def is_exe(fpath):
        return os.path.exists(fpath) and os.access(fpath, os.X_OK)

    fpath, fname = os.path.split(program)
    if fpath:
        if is_exe(program):
            return program
    else:
        for path in os.environ["PATH"].split(os.pathsep):
            exe_file = os.path.join(path, program)
            if is_exe(exe_file):
                return exe_file
    return ""

xvfb = which('xvfb-run')
if not xvfb:
    print "xvfb-run not found; any GUI components that are run will display windows"
else:
    xvfb = xvfb + " -a"

if len(p.targets)==0:
    # DEFAULT
    p.targets = ['unit','traintests','snapshots','gui','maptests'] # maptests wouldn't be default except it's caught platform different problems before (there aren't enough unit tests!)

# Allow allsnapshottests as shortcut for snapshots and pickle.
# CEBALERT: should just combine these tests anyway.
if "allsnapshottests" in p.targets:
    index = p.targets.index("allsnapshottests")
    p.targets[index] = "snapshots"
    p.targets.insert(index,"pickle")

# ->params ?
tests_dir = param.resolve_path("topo/tests",path_to_file=False)
scripts_dir = param.resolve_path(".",path_to_file=False) ### XXX
topographica_script = xvfb + " " + timing_cmd + coverage_cmd + " " + sys.argv[0] + " " +  " "

def _runc(cmd):
    print cmd
    return os.system(cmd)

import topo.misc.keyedlist
target = topo.misc.keyedlist.KeyedList()
speedtarget = topo.misc.keyedlist.KeyedList()

# CEBALERT: need to pick which scripts to include for traintests and
# speedtests and startupspeedtests (see test_script.py).
#
# From the Makefile:
# SCRIPTS= ^hierarchical.ty ^lissom_or.ty ^lissom_oo_or.ty ^som_retinotopy.ty ^sullivan_neurocomputing04.ty ^lissom.ty ^lissom_fsa.ty ^gcal.ty ^lissom_whisker_barrels.ty
# CEB: tests on these scripts temporarily suspended (SF.net #2053538)
# ^lissom_oo_or_homeostatic.ty ^lissom_oo_or_homeostatic_tracked.ty
# ^lissom_or_noshrinking.ty  - only matches to 4 dp with IMPORT_WEAVE=0
# Now I'm using this list for train-tests:

# CEBALERT: this list should be defined in one place.
#from setup import TRAINSCRIPTS
TRAINSCRIPTS = [
    "examples/hierarchical.ty",
    "models/lissom_or.ty",
    "models/lissom_oo_or.ty",
    "examples/som_retinotopy.ty",
    "models/sullivan_neurocomputing04.ty",
    "models/lissom.ty",
#    "models/lissom_fsa.ty",  # CEBALERT: disabled for now (needs special case - look_at=fsa not v1, see below)
    "examples/gcal.ty"
    ]

# (and a different list for speedtests - see test_script.py).
#
# Also: special cases from the Makefile to consider restoring:
#
##topo/tests/lissom.ty_DATA:
##	./topographica -c 'from topo.tests.test_script import generate_data; generate_data(script="examples/lissom.ty",data_filename="tests/lissom.ty_DATA",run_for=[1,99,150],look_at="V1",cortex_density=8,retina_density=6,lgn_density=6,dims=["or","od","dr","dy","cr","sf"])'
##
##topo/tests/lissom_fsa.ty_DATA:
##	./topographica -c 'from topo.tests.test_script import generate_data; generate_data(script="examples/lissom_fsa.ty",data_filename="tests/lissom_fsa.ty_DATA",run_for=[1,99,150],look_at="FSA",cortex_density=8,retina_density=24,lgn_density=24)'
##
##topo/tests/lissom_whisker_barrels.ty_DATA:
##	./topographica -c 'from topo.tests.test_script import generate_data; generate_data(script="examples/lissom_whisker_barrels.ty",data_filename="tests/lissom_whisker_barrels.ty_DATA",run_for=[1,99,150],look_at="S1")'

target['traintests'] = []
for script in TRAINSCRIPTS:
    script_path = os.path.join(scripts_dir,script)
    target['traintests'].append(topographica_script +  ''' -c "from topo.tests.test_script import test_script; test_script(script=%(script_path)s,decimal=%(dp)s)"'''%dict(script_path=repr(script_path),dp=p.testdp))

# CEBALERT: should use the code above but just with the changes for running without weave.
target['unopttraintests'] = []
for script in TRAINSCRIPTS:
    script_path = os.path.join(scripts_dir,script)
    target['unopttraintests'].append(topographica_script + " -c 'import_weave=False'" +  " -c 'from topo.tests.test_script import test_script; test_script(script=\"%(script_path)s\",decimal=%(dp)s)'"%dict(script_path=script_path,dp=p.testdp_unopt))

speedtarget['speedtests'] = []
SPEEDSCRIPTS = TRAINSCRIPTS
SPEEDSCRIPTS.remove("examples/hierarchical.ty") # CEBALERT: remove problematic example (doesn't work for some densities)
for script in SPEEDSCRIPTS:
    script_path = os.path.join(scripts_dir,script)
    speedtarget['speedtests'].append(topographica_script +  ''' -c "from topo.tests.test_script import compare_speed_data;compare_speed_data(script=%(script_path)s)"'''%dict(script_path=repr(script_path)))

STARTUPSPEEDSCRIPTS = ["models/lissom.ty","examples/gcal.ty"]

speedtarget['startupspeedtests'] = []
for script in STARTUPSPEEDSCRIPTS:
    script_path = os.path.join(scripts_dir,script)
    speedtarget['startupspeedtests'].append(topographica_script +  ''' -c "from topo.tests.test_script import compare_startup_speed_data;compare_startup_speed_data(script=%(script_path)s)"'''%dict(script_path=repr(script_path)))

##### snapshot-tests
target['snapshots'] = []

# snapshot-compatibility-tests:
snapshot_path = os.path.join(tests_dir,"lissom_oo_or_od_dr_cr_dy_sf_000010.00.typ")
target['snapshots'].append(topographica_script + ''' -c "from topo.command import load_snapshot; load_snapshot(%(snapshot_path)s)" -c "topo.sim.run(1)"'''%dict(snapshot_path=repr(snapshot_path)))

# Test that simulations give the same results whether run straight
# through or run part way, saved, reloaded, and run on to the same
# point.
# simulation-snapshot-tests:
target['snapshots'].append(topographica_script + ''' -c "from topo.tests.test_script import compare_with_and_without_snapshot_NoSnapshot as A; A()"''')

target['snapshots'].append(topographica_script + ''' -c "from topo.tests.test_script import compare_with_and_without_snapshot_CreateSnapshot as B; B()"''')

target['snapshots'].append(topographica_script + ''' -c "from topo.tests.test_script import compare_with_and_without_snapshot_LoadSnapshot as C; C()"''')

# CEBALERT: remove ~/topographica/*PICKLETEST* ?

target['pickle'] = []
# pickle-all-classes:
target['pickle'].append(topographica_script + ''' -c "from topo.tests.test_script import pickle_unpickle_everything; errs = pickle_unpickle_everything(); import sys; sys.exit(errs)"''')

# unpickle-compatibility-tests:
pickle_path = os.path.join(tests_dir,"instances-r11275.pickle")
target['pickle'].append(topographica_script + '''-l -c "from topo.tests.test_script import pickle_unpickle_everything; errs = pickle_unpickle_everything(existing_pickles=%(pickle_path)s); import sys; sys.exit(errs)"'''%dict(pickle_path=repr(pickle_path)))

# CEBALERT: hack that this will always be created even when test not being run
# DSALERT: also it's not portable; python has a tempfile module that should
# easily be able to replace this
tmpd = commands.getoutput("mktemp -d")
#script-repr-tests:
target['scriptrepr']=[]
script = os.path.join(scripts_dir,"examples/hierarchical.ty")
target['scriptrepr'].append(topographica_script + " %(script)s -a -c \"import param;param.normalize_path.prefix='%(tmpd)s'\" -c \"save_script_repr('script_repr_test.ty')\""%dict(tmpd=tmpd,script=script))

script_repr_test_path = os.path.join(tmpd,"script_repr_test.ty")
target['scriptrepr'].append(topographica_script + " " + script_repr_test_path)
target['scriptrepr'].append(topographica_script + "-c \"import shutil;shutil.rmtree('%s')\""%tmpd)

### GUI tests
target['gui'] = []
# basic-gui-tests:
target['gui'].append(topographica_script + ' -g -c "from topo.tests.gui_tests import run_basic; nerr=run_basic();topo.guimain.quit_topographica(check=False,exit_status=nerr)"')

# detailed-gui-tests:
target['gui'].append(topographica_script + ' -g -c "from topo.tests.gui_tests import run_detailed; nerr=run_detailed(); topo.guimain.quit_topographica(check=False,exit_status=nerr)"')

target['batch'] = []
target['batch'].append(topographica_script + ' -c "from topo.tests.test_script import test_runbatch; test_runbatch()"')

target['unit'] = []
target['unit'].append(topographica_script + ' -c "import topo.tests; t=topo.tests.run(); import sys; sys.exit(len(t.failures+t.errors))"')

# CEBALERT: should use lissom.ty and test more map types
# pass a list of plotgroup names to test() instead of plotgroups_to_test to restrict the tests
target['maptests'] = []
script = os.path.join(scripts_dir,"models/lissom_oo_or.ty")
target['maptests'].append(topographica_script + ' -c "cortex_density=8" %s -c "topo.sim.run(100);from topo.tests.test_map_measurement import *; test(plotgroups_to_test)"'%script)

def start():
    errors = []

    print "Running: %s"%p.targets
    print

    # CEBALERT: rename one of target or targets!

    if p.targets==['all']:
        targets = target.keys()
    elif p.targets==['speed']:
        targets = speedtarget.keys()
    else:
        targets = p.targets

    target.update(speedtarget)

    for name in targets:
        print "*** " + name
        for cmd in target[name]:
            if _runc(cmd) > 0:
                if name not in errors:
                    errors.append(name)

    print
    print "="*60
    print
    print "runtests.start(): targets with errors: %s"%len(errors)
    if len(errors)>0:
        print errors
        sys.exit(1)
    else:
        sys.exit(0)

if __name__=="__main__":
    start()

<|endoftext|>


<commit_before># This file is part of BenchExec, a framework for reliable benchmarking:
# https://github.com/sosy-lab/benchexec
#
# SPDX-FileCopyrightText: 2007-2020 Dirk Beyer <https://www.sosy-lab.org>
#
# SPDX-License-Identifier: Apache-2.0

import os
from benchexec.tools.sv_benchmarks_util import get_data_model_from_task, ILP32, LP64
import benchexec.tools.template
import benchexec.result as result

class Tool(benchexec.tools.template.BaseTool2):
    """
    This class serves as tool adaptor for FuSeBMC (https://github.com/kaled-alshmrany/FuSeBMC)
    """

    REQUIRED_PATHS_TESTCOMP20 = ["esbmc", "esbmc-wrapper.py", "my_instrument"]
    REQUIRED_PATHS_TESTCOMP21 = [
        "esbmc",
        "fusebmc.py",
        "FuSeBMC_inustrment/FuSeBMC_inustrment",
        "fusebmc_output",
        "map2check-fuzzer",
    ]
    REQUIRED_PATHS_TESTCOMP22 = [
        "FuSeBMC_engines",
        "FuSeBMC_FuzzerLib",
        "FuSeBMC_GoalTracerLib",
        "FuSeBMC_inustrument/FuSeBMC_inustrment",
        "fusebmc_output",
        "FuSeBMC_SeedGenLib",
        "FuSeBMC_TCGen",
        "fusebmc_util",
        "fusebmc.py",
        "fusebmc_getinput_from_testcase.py",
        "fusebmc_seed_gen.py",
    ]
    def executable(self, tool_locator):
        return tool_locator.find_executable("fusebmc.py")

    def working_directory(self, executable):
        executableDir = os.path.dirname(executable)
        return executableDir

    def version(self, executable):
        return self._version_from_tool(executable, "-v")

    def name(self):
        return "FuSeBMC"

    def cmdline(self, executable, options, task, rlimits):
        data_model_param = get_data_model_from_task(task, {ILP32: "32", LP64: "64"})
        if data_model_param and "--arch" not in options:
            options += ["--arch", data_model_param]
        return (
            [executable]
            + ["-p", task.property_file]
            + options
            + [task.single_input_file]
        )

    def program_files(self, executable):
        """
        Determine the file paths to be adopted
        """
        if self._version == 21:
            paths = self.REQUIRED_PATHS_TESTCOMP21
        elif self._version > 21:
            paths = self.REQUIRED_PATHS_TESTCOMP22
        return self._program_files_from_executable(executable, paths)

    def determine_result(self, run):
        status = result.RESULT_UNKNOWN

        if run.output.any_line_contains("DONE"):
            status = result.RESULT_DONE

        if status == result.RESULT_UNKNOWN:
            if run.was_timeout:
                status = "TIMEOUT"
            elif not run.output.any_line_contains("Unknown"):
                status = "ERROR"

        return status
<commit_msg>Update fusebmc.py<commit_after># This file is part of BenchExec, a framework for reliable benchmarking:
# https://github.com/sosy-lab/benchexec
#
# SPDX-FileCopyrightText: 2007-2020 Dirk Beyer <https://www.sosy-lab.org>
#
# SPDX-License-Identifier: Apache-2.0

import os
from benchexec.tools.sv_benchmarks_util import get_data_model_from_task, ILP32, LP64
import benchexec.tools.template
import benchexec.result as result

class Tool(benchexec.tools.template.BaseTool2):
    """
    This class serves as tool adaptor for FuSeBMC (https://github.com/kaled-alshmrany/FuSeBMC)
    """

    def executable(self, tool_locator):
        return tool_locator.find_executable("fusebmc.py")

    def working_directory(self, executable):
        executableDir = os.path.dirname(executable)
        return executableDir

    def version(self, executable):
        return self._version_from_tool(executable, "-v")

    def name(self):
        return "FuSeBMC"

    def cmdline(self, executable, options, task, rlimits):
        data_model_param = get_data_model_from_task(task, {ILP32: "32", LP64: "64"})
        if data_model_param and "--arch" not in options:
            options += ["--arch", data_model_param]
        return (
            [executable]
            + ["-p", task.property_file]
            + options
            + [task.single_input_file]
        )

    def determine_result(self, run):
        status = result.RESULT_UNKNOWN

        if run.output.any_line_contains("DONE"):
            status = result.RESULT_DONE

        if status == result.RESULT_UNKNOWN:
            if run.was_timeout:
                status = "TIMEOUT"
            elif not run.output.any_line_contains("Unknown"):
                status = "ERROR"

        return status
<|endoftext|>