<commit_before>from __future__ import print_function

import logging

from simphony.core.cuba import CUBA
from simphony.cuds.abc_modeling_engine import ABCModelingEngine
from simphony_mayavi.sources.api import EngineSource
from traits.api import HasTraits, Instance, ListStr, Button, Str, Float, Int
from traitsui.api import View, VGroup, HGroup, Item, EnumEditor

# This class is intended to be used as a standalone as well as an evisage
# plugin for mayavi.  For the latter, mayavi imports cannot be put at the
# module level or it would cause import cycles

class EngineSourceManager(HasTraits):
    ''' A basic tool for visualising CUDS datasets in a modeling
    engine as well as performing basic operations on the engine

    Examples
    ---------
    >>> from simphony.engine import lammps   # LAMMPS
    >>> # Setup the engine so it is ready to run
    >>> dem = lammps.LammpsWrapper(...)
    >>> # Setup for visualization
    >>> from simphony_mayavi.plugins.api import EngineSourceManager
    >>> manager = EngineSourceManager(dem)
    >>> manager.show_config()   # GUI

    One can still perform all functions of this manager without
    opening up a GUI ::

    >>> from simphony_mayavi.plugins.api import EngineSourceManager
    >>> manager = EngineSourceManager(dem)
    >>> manager.datasets   # display all available datasets
    ["particles1", "mesh1"]
    >>> # Show the particles1 CUDS
    >>> manager.add_dataset_to_scene("particles")
    >>> # Run the engine 10 times and animate the results
    >>> manager.animate(10)
    >>> # Adjust time step of the engine
    >>> manager.time_step = 0.1
    >>> # Adjust the number of time steps for each run
    >>> manager.number_of_time_steps = 100
    '''
    # Set by Envisage when this class is used as a mayavi plugin
    window = Instance("pyface.workbench.api.WorkbenchWindow")

    # ----------------------------------------------
    # Basic Info of the modeling engine
    # ----------------------------------------------
    modeling_engine = Instance(ABCModelingEngine, allow_none=False)
    datasets = ListStr
    _dataset = Str

    # -------------------------------------------------------
    # Basic time step parameters to interact with the engine
    # -------------------------------------------------------
    time_step = Float(allow_none=False)
    number_of_time_steps = Float(allow_none=False)

    # ----------------------------------------------
    # only relevant to UI
    # ----------------------------------------------
    _number_of_runs = Int(1)
    _add_to_scene = Button("Add to scene")
    _animate = Button("Animate")
    _animator = None
    _animate_delay = 20

    view = View(
        VGroup(
            HGroup(Item("_dataset", label="Dataset",
                        editor=EnumEditor(name="datasets")),
                   Item("_add_to_scene", show_label=False)),

            VGroup(Item(label="Engine Settings", emphasized=True),
                   Item("_"),
                   Item("time_step"),
                   Item("number_of_time_steps"),
                   HGroup(Item(name="_number_of_runs",
                               label="Runs for"),
                          Item(label="time(s)"))),

            HGroup(Item("_animate", show_label=False))
        )
    )

    # ------------------------------------------------
    # Public property
    # ------------------------------------------------
    @property
    def datasets(self):
        """ Datasets (list) in the modeling engines """
        return self.modeling_engine.get_dataset_names()

    # ----------------------------------------------------
    # Initialization
    # ----------------------------------------------------
    def __init__(self, modeling_engine, as_plugin=False):
        # SimPhoNy Modeling Engine wrapper
        self.modeling_engine = modeling_engine

        # default dataset
        if self.datasets:
            self._dataset = self.datasets[0]

        # For interacting with the engine
        if CUBA.TIME_STEP in self.modeling_engine.CM:
            self.time_step = self.modeling_engine.CM[CUBA.TIME_STEP]
        else:
            logging.warning("TIME_STEP is not found.")

        if CUBA.NUMBER_OF_TIME_STEPS in self.modeling_engine.CM:
            value = self.modeling_engine.CM[CUBA.NUMBER_OF_TIME_STEPS]
            self.number_of_time_steps = value
        else:
            logging.warning("NUMBER_OF_TIME_STEPS is not found.")

        if as_plugin:
            # as Mayavi plugin
            from mayavi.plugins.script import Script
            self.mayavi_engine = self.window.get_service(Script)
        else:
            # default standalone
            from mayavi import mlab
            self.mayavi_engine = mlab.get_engine()

    # -------------------------------------------------
    # Functions relevant to the UI
    # -------------------------------------------------
    def __add_to_scene_fired(self):
        self.add_dataset_to_scene(self._dataset)

    def __animate_fired(self):
        if self._animator:
            self._animate_delay = self._animator.delay
        if self._animator and not self._animator.ui.destroyed:
            self._animator.close()
        self._animator = self.animate(self._number_of_runs,
                                      delay=self._animate_delay)

    # ----------------------------------------------------------
    # Trait handlers
    # ----------------------------------------------------------
    def _time_step_changed(self):
        self.modeling_engine.CM[CUBA.TIME_STEP] = self.time_step

    def _number_of_time_steps_changed(self):
        value = self.number_of_time_steps
        self.modeling_engine.CM[CUBA.NUMBER_OF_TIME_STEPS] = value

    # -----------------------------------------------------------
    # Public methods
    # -----------------------------------------------------------
    def animate(self, number_of_runs, delay=20, ui=True):
        ''' Run the modeling engine, update and thus animate the scene
        at the end of each run

        Parameters
        ----------
        number_of_runs : int
           the number of times the engine.run() is called
        delay : int
           delay between each run
        ui : bool
           whether an UI is shown

        Returns
        -------
        animator : mayavi.tools.animator.Animator
        '''
        from mayavi.mlab import animate

        @animate(delay=delay, ui=ui)
        def anim():
            for _ in xrange(number_of_runs):
                self.modeling_engine.run()
                self.update_current_sources()
                self.mayavi_engine.current_scene.render()
                yield()
        return anim()

    def update_current_sources(self):
        ''' Update all the sources in the current scene '''
        for source in self.get_current_sources():
            source.update()

    def add_dataset_to_scene(self, name, module=None):
        from simphony_mayavi.modules.default_module import default_module

        source = EngineSource(engine=self.modeling_engine)
        source.dataset = name
        self.mayavi_engine.add_source(source)

        # add module to the source
        if module:
            self.mayavi_engine.add_module(module)
        else:
            self.mayavi_engine.add_module(default_module(source._vtk_cuds))

    def get_current_sources(self):
        ''' Return a list of sources in the current scene

        Returns
        -------
        sources : list of EngineSource
        '''
        return self.mayavi_engine.current_scene.children

    def show_config(self):
        ''' Show the UI of this manager
        '''
        self.configure_traits(kind="live")
<commit_msg>added update_all_scenes features<commit_after>from __future__ import print_function

import logging

from simphony.core.cuba import CUBA
from simphony.cuds.abc_modeling_engine import ABCModelingEngine
from simphony_mayavi.sources.api import EngineSource
from traits.api import (HasTraits, Instance, ListStr, Button, Str, Float, Int,
                        Bool)
from traitsui.api import View, VGroup, HGroup, Item, EnumEditor, message

# This class is intended to be used as a standalone as well as an evisage
# plugin for mayavi.  For the latter, mayavi imports cannot be put at the
# module level or it would cause import cycles

class EngineSourceManager(HasTraits):
    ''' A basic tool for visualising CUDS datasets from a modeling
    engine as well as performing basic operations on the engine

    Examples
    ---------
    >>> from simphony.engine import lammps   # LAMMPS
    >>> # Setup the engine so it is ready to run
    >>> dem = lammps.LammpsWrapper(...)
    >>> # Setup for visualization
    >>> from simphony_mayavi.plugins.api import EngineSourceManager
    >>> manager = EngineSourceManager(dem)
    >>> manager.show_config()   # GUI

    One can still perform all functions of this manager without
    opening up a GUI ::

    >>> from simphony_mayavi.plugins.api import EngineSourceManager
    >>> manager = EngineSourceManager(dem)
    >>> manager.datasets   # display all available datasets
    ["particles1", "mesh1"]
    >>> # Show the particles1 CUDS
    >>> manager.add_dataset_to_scene("particles")
    >>> # Run the engine 10 times and animate the results
    >>> manager.animate(10)
    >>> # Adjust time step of the engine
    >>> manager.time_step = 0.1
    >>> # Adjust the number of time steps for each run
    >>> manager.number_of_time_steps = 100
    '''
    # Set by Envisage when this class is used as a mayavi plugin
    window = Instance("pyface.workbench.api.WorkbenchWindow")

    # ----------------------------------------------
    # Basic Info of the modeling engine
    # ----------------------------------------------
    modeling_engine = Instance(ABCModelingEngine, allow_none=False)
    datasets = ListStr
    _dataset = Str

    # -------------------------------------------------------
    # Basic time step parameters to interact with the engine
    # -------------------------------------------------------
    time_step = Float(allow_none=False)
    number_of_time_steps = Float(allow_none=False)

    # ----------------------------------------------
    # only relevant to UI
    # ----------------------------------------------
    _number_of_runs = Int(1)
    _add_to_scene = Button("Add to scene")
    _animate = Button("Animate")
    _animator = None
    _animate_delay = 20
    _update_all_scenes = Bool(False)

    view = View(
        VGroup(
            HGroup(Item("_dataset", label="Dataset",
                        editor=EnumEditor(name="datasets")),
                   Item("_add_to_scene", show_label=False)),

            Item(label="Engine Settings", emphasized=True),
            Item("_"),
            Item("time_step"),
            Item("number_of_time_steps"),
            HGroup(Item(name="_number_of_runs",
                        label="Runs for"),
                   Item(label="time(s)")),

            HGroup(Item("_animate", show_label=False),
                   Item(name="_update_all_scenes",
                        label="Update all scenes"))))

    # ------------------------------------------------
    # Public property
    # ------------------------------------------------
    @property
    def datasets(self):
        """ Datasets (list) in the modeling engines """
        return self.modeling_engine.get_dataset_names()

    # ----------------------------------------------------
    # Initialization
    # ----------------------------------------------------
    def __init__(self, modeling_engine, as_plugin=False):
        # SimPhoNy Modeling Engine wrapper
        self.modeling_engine = modeling_engine

        # default dataset
        if self.datasets:
            self._dataset = self.datasets[0]

        # For interacting with the engine
        if CUBA.TIME_STEP in self.modeling_engine.CM:
            self.time_step = self.modeling_engine.CM[CUBA.TIME_STEP]
        else:
            logging.warning("TIME_STEP is not found.")

        if CUBA.NUMBER_OF_TIME_STEPS in self.modeling_engine.CM:
            value = self.modeling_engine.CM[CUBA.NUMBER_OF_TIME_STEPS]
            self.number_of_time_steps = value
        else:
            logging.warning("NUMBER_OF_TIME_STEPS is not found.")

        if as_plugin:
            # as Mayavi plugin
            from mayavi.plugins.script import Script
            self.mayavi_engine = self.window.get_service(Script)
        else:
            # default standalone
            from mayavi import mlab
            self.mayavi_engine = mlab.get_engine()

    # -------------------------------------------------
    # Functions relevant to the UI
    # -------------------------------------------------
    def __add_to_scene_fired(self):
        self.add_dataset_to_scene(self._dataset)

    def __animate_fired(self):
        self.animate(self._number_of_runs,
                     update_all_scenes=self._update_all_scenes)

    # ----------------------------------------------------------
    # Trait handlers
    # ----------------------------------------------------------
    def _time_step_changed(self):
        self.modeling_engine.CM[CUBA.TIME_STEP] = self.time_step

    def _number_of_time_steps_changed(self):
        value = self.number_of_time_steps
        self.modeling_engine.CM[CUBA.NUMBER_OF_TIME_STEPS] = value

    # -----------------------------------------------------------
    # Public methods
    # -----------------------------------------------------------
    def animate(self, number_of_runs, delay=None, ui=True,
                update_all_scenes=False):
        ''' Run the modeling engine, update and thus animate the scene
        at the end of each run

        Parameters
        ----------
        number_of_runs : int
           the number of times the engine.run() is called
        delay : int
           delay between each run.
           If None, use last setting or Mayavi's default: 500
        ui : bool
           whether an UI is shown
        update_all_scenes : bool
           whether all scenes are updated
        '''
        from mayavi.mlab import animate

        # remember the last delay being set
        if delay is None:
            if self._animator:
                delay = self._animator.delay
            else:
                delay = 500

        # close the old window and start a new one
        # FIXME: there should be a better way
        if self._animator and not self._animator.ui.destroyed:
            self._animator.close()

        if update_all_scenes:
            get_sources_func = self.get_all_sources
        else:
            get_sources_func = self.get_current_sources

        try:
            sources = get_sources_func()
        except AttributeError:
            text = "Nothing in scene.  Engine is not run."
            message(text)
            raise RuntimeError(text)

        @animate(delay=delay, ui=ui)
        def anim():
            for _ in xrange(number_of_runs):
                self.modeling_engine.run()
                self.update_sources(sources)
                self.mayavi_engine.current_scene.render()
                yield()
        self._animator = anim()

    def update_sources(self, sources):
        ''' Update multiple sources'''
        for source in sources:
            source.update()

    def add_dataset_to_scene(self, name, module=None):
        from simphony_mayavi.modules.default_module import default_module

        source = EngineSource(engine=self.modeling_engine)
        source.dataset = name
        self.mayavi_engine.add_source(source)

        # add module to the source
        if module:
            self.mayavi_engine.add_module(module)
        else:
            self.mayavi_engine.add_module(default_module(source._vtk_cuds))

    def get_current_sources(self):
        ''' Return the current scene's sources that are belong
        to this manager's modeling engine

        Returns
        -------
        sources : set of EngineSource
        '''
        sources = self.mayavi_engine.current_scene.children
        return {source for source in sources
                if source.engine == self.modeling_engine}

    def get_all_sources(self):
        ''' Return sources from all the scenes and that the sources
        are belong to this manager's modeling engine

        Returns
        -------
        sources : set of EngineSource
        '''
        sources = {source
                   for scene in self.mayavi_engine.scenes
                   for source in scene.children
                   if source.engine == self.modeling_engine}
        return sources

    def show_config(self):
        ''' Show the UI of this manager
        '''
        self.configure_traits(kind="live")
<|endoftext|>


<commit_before>from collections import OrderedDict
from copy import copy

from devlib import (LinuxTarget, AndroidTarget, LocalLinuxTarget,
                    Platform, Juno, TC2, Gem5SimulationPlatform,
                    AdbConnection, SshConnection, LocalConnection,
                    Gem5Connection)

from wa.framework import pluginloader
from wa.framework.exception import PluginLoaderError
from wa.framework.plugin import Plugin, Parameter
from wa.framework.target.assistant import LinuxAssistant, AndroidAssistant
from wa.utils.types import list_of_strings, list_of_ints
from wa.utils.misc import isiterable

def get_target_descriptions(loader=pluginloader):
    targets = {}
    for cls in loader.list_target_descriptors():
        descriptor = cls()
        for desc in descriptor.get_descriptions():
            if desc.name in targets:
                msg = 'Duplicate target "{}" returned by {} and {}'
                prev_dtor = targets[desc.name].source
                raise PluginLoaderError(msg.format(desc.name, prev_dtor.name,
                                                   descriptor.name))
            targets[desc.name] = desc
    return targets.values()

def instantiate_target(tdesc, params, connect=None, extra_platform_params=None):
    target_params = {p.name: p for p in tdesc.target_params}
    platform_params = {p.name: p for p in tdesc.platform_params}
    conn_params = {p.name: p for p in tdesc.conn_params}
    assistant_params = {p.name: p for p in tdesc.assistant_params}

    tp, pp, cp = {}, {}, {}

    for supported_params, new_params in (target_params, tp), (platform_params, pp), (conn_params, cp):
        for name, value in supported_params.iteritems():
            if value.default:
                new_params[name] = value.default

    for name, value in params.iteritems():
        if name in target_params:
            tp[name] = value
        elif name in platform_params:
            pp[name] = value
        elif name in conn_params:
            cp[name] = value
        elif name in assistant_params:
            pass
        else:
            msg = 'Unexpected parameter for {}: {}'
            raise ValueError(msg.format(tdesc.name, name))

    for pname, pval in (extra_platform_params or {}).iteritems():
        if pname in pp:
            raise RuntimeError('Platform parameter clash: {}'.format(pname))
        pp[pname] = pval

    tp['platform'] = (tdesc.platform or Platform)(**pp)
    if cp:
        tp['connection_settings'] = cp
    if tdesc.connection:
        tp['conn_cls'] = tdesc.connection
    if connect is not None:
        tp['connect'] = connect

    return tdesc.target(**tp)

def instantiate_assistant(tdesc, params, target):
    assistant_params = {}
    for param in tdesc.assistant_params:
        if param.name in params:
            assistant_params[param.name] = params[param.name]
        elif param.default:
            assistant_params[param.name] = param.default
    return tdesc.assistant(target, **assistant_params)

class TargetDescription(object):

    def __init__(self, name, source, description=None, target=None, platform=None,
                 conn=None, assistant=None, target_params=None, platform_params=None,
                 conn_params=None, assistant_params=None):
        self.name = name
        self.source = source
        self.description = description
        self.target = target
        self.platform = platform
        self.connection = conn
        self.assistant = assistant
        self._set('target_params', target_params)
        self._set('platform_params', platform_params)
        self._set('conn_params', conn_params)
        self._set('assistant_params', assistant_params)

    def get_default_config(self):
        param_attrs = ['target_params', 'platform_params',
                       'conn_params', 'assistant_params']
        config = {}
        for pattr in param_attrs:
            for n, p in getattr(self, pattr).itervalues():
                config[n] = p.default
        return config

    def _set(self, attr, vals):
        if vals is None:
            vals = {}
        elif isiterable(vals):
            if not hasattr(vals, 'iteritems'):
                vals = {v.name: v for v in vals}
        else:
            msg = '{} must be iterable; got "{}"'
            raise ValueError(msg.format(attr, vals))
        setattr(self, attr, vals)

class TargetDescriptor(Plugin):

    kind = 'target_descriptor'

    def get_descriptions(self):
        return []

COMMON_TARGET_PARAMS = [
    Parameter('working_directory', kind=str,
              description='''
              On-target working directory that will be used by WA. This
              directory must be writable by the user WA logs in as without
              the need for privilege elevation.
              '''),
    Parameter('executables_directory', kind=str,
              description='''
              On-target directory where WA will install its executable
              binaries.  This location must allow execution. This location does
              *not* need to be writable by unprivileged users or rooted devices
              (WA will install with elevated privileges as necessary).
              '''),
    Parameter('modules', kind=list_of_strings,
              description='''
              A list of additional modules to be installed for the target.

              ``devlib`` implements functionality for particular subsystems as
              modules.  A number of "default" modules (e.g. for cpufreq
              subsystem) are loaded automatically, unless explicitly disabled.
              If additional modules need to be loaded, they may be specified
              using this parameter.

              Please see ``devlib`` documentation for information on the available
              modules.
              '''),
    Parameter('load_default_modules', kind=bool, default=True,
              description='''
              A number of modules (e.g. for working with the cpufreq subsystem) are
              loaded by default when a Target is instantiated. Setting this to
              ``True`` would suppress that, ensuring that only the base Target
              interface is initialized.

              You may want to set this to ``False`` if there is a problem with one
              or more default modules on your platform (e.g. your device is
              unrooted and cpufreq is not accessible to unprivileged users), or
              if ``Target`` initialization is taking too long for your platform.
              '''),
]

COMMON_PLATFORM_PARAMS = [
    Parameter('core_names', kind=list_of_strings,
              description='''
              List of names of CPU cores in the order that they appear to the
              kernel. If not specified, it will be inferred from the platform.
              '''),
    Parameter('core_clusters', kind=list_of_ints,
              description='''
              Cluster mapping corresponding to the cores in ``core_names``.
              Cluster indexing starts at ``0``.  If not specified, this will be
              inferred from ``core_names`` -- consecutive cores with the same
              name will be assumed to share a cluster.
              '''),
    Parameter('big_core', kind=str,
              description='''
              The name of the big cores in a big.LITTLE system. If not
              specified, this will be inferred, either from the name (if one of
              the names in ``core_names`` matches known big cores), or by
              assuming that the last cluster is big.
              '''),
    Parameter('model', kind=str,
              description='''
              Hardware model of the platform. If not specified, an attempt will
              be made to read it from target.
              '''),
    Parameter('modules', kind=list_of_strings,
              description='''
              An additional list of modules to be loaded into the target.
              '''),
]

VEXPRESS_PLATFORM_PARAMS = [
    Parameter('serial_port', kind=str,
              description='''
              The serial device/port on the host for the initial connection to
              the target (used for early boot, flashing, etc).
              '''),
    Parameter('baudrate', kind=int,
              description='''
              Baud rate for the serial connection.
              '''),
    Parameter('vemsd_mount', kind=str,
              description='''
              VExpress MicroSD card mount location. This is a MicroSD card in
              the VExpress device that is mounted on the host via USB. The card
              contains configuration files for the platform and firmware and
              kernel images to be flashed.
              '''),
    Parameter('bootloader', kind=str,
              allowed_values=['uefi', 'uefi-shell', 'u-boot', 'bootmon'],
              description='''
              Selects the bootloader mechanism used by the board. Depending on
              firmware version, a number of possible boot mechanisms may be use.

              Please see ``devlib`` documentation for descriptions.
              '''),
    Parameter('hard_reset_method', kind=str,
              allowed_values=['dtr', 'reboottxt'],
              description='''
              There are a couple of ways to reset VersatileExpress board if the
              software running on the board becomes unresponsive. Both require
              configuration to be enabled (please see ``devlib`` documentation).

              ``dtr``: toggle the DTR line on the serial connection
              ``reboottxt``: create ``reboot.txt`` in the root of the VEMSD mount.

              '''),
]

GEM5_PLATFORM_PARAMS = [
    Parameter('gem5_bin', kind=str, mandatory=True,
              description='''
              Path to the gem5 binary
              '''),
    Parameter('gem5_args', kind=str, mandatory=True,
              description='''
              Arguments to be passed to the gem5 binary
              '''),
    Parameter('gem5_virtio', kind=str, mandatory=True,
              description='''
              VirtIO device setup arguments to be passed to gem5. VirtIO is used
              to transfer files between the simulation and the host.
              '''),
    Parameter('name', kind=str, default='gem5',
              description='''
              The name for the gem5 "device".
              '''),
]

CONNECTION_PARAMS = {
    AdbConnection: [
        Parameter('device', kind=str,
                description="""
                ADB device name
                """),
        Parameter('adb_server', kind=str,
                description="""
                ADB server to connect to.
                """),
    ],
    SshConnection: [
        Parameter('host', kind=str, mandatory=True,
                description="""
                Host name or IP address of the target.
                """),
        Parameter('username', kind=str, mandatory=True,
                description="""
                User name to connect with
                """),
        Parameter('password', kind=str,
                description="""
                Password to use.
                """),
        Parameter('keyfile', kind=str,
                description="""
                Key file to use
                """),
        Parameter('port', kind=int,
                description="""
                The port SSH server is listening on on the target.
                """),
        Parameter('telnet', kind=bool, default=False,
                description="""
                If set to ``True``, a Telnet connection, rather than
                SSH will be used.
                """),
        Parameter('password_prompt', kind=str,
                description="""
                Password prompt to expect
                """),
        Parameter('original_prompt', kind=str,
                description="""
                Original shell prompt to expect.
                """),
        Parameter('sudo_cmd', kind=str,
                default="sudo -- sh -c '{}'",
                description="""
                Sudo command to use. Must have ``"{}"``` specified
                somewher in the string it indicate where the command
                to be run via sudo is to go.
                """),
    ],
    Gem5Connection: [
        Parameter('host', kind=str, mandatory=False,
                description="""
                Host name or IP address of the target.
                """),
        Parameter('username', kind=str, default='root',
                description="""
                User name to connect to gem5 simulation.
                """),
        Parameter('password', kind=str,
                description="""
                Password to use.
                """),
        Parameter('port', kind=int,
                description="""
                The port SSH server is listening on on the target.
                """),
        Parameter('password_prompt', kind=str,
                description="""
                Password prompt to expect
                """),
        Parameter('original_prompt', kind=str,
                description="""
                Original shell prompt to expect.
                """),
    ],
    LocalConnection: [
        Parameter('password', kind=str,
                description="""
                Password to use for sudo. if not specified, the user will
                be prompted during intialization.
                """),
        Parameter('keep_password', kind=bool, default=True,
                description="""
                If ``True`` (the default), the password will be cached in
                memory after it is first obtained from the user, so that the
                user would not be prompted for it again.
                """),
        Parameter('unrooted', kind=bool, default=False,
                description="""
                Indicate that the target should be considered unrooted; do not
                attempt sudo or ask the user for their password.
                """),
    ],
}

# name --> ((target_class, conn_class), params_list, defaults, assistant_class)
TARGETS = {
    'linux': ((LinuxTarget, SshConnection), COMMON_TARGET_PARAMS, None),
    'android': ((AndroidTarget, AdbConnection), COMMON_TARGET_PARAMS +
                [Parameter('package_data_directory', kind=str, default='/data/data',
                           description='''
                           Directory containing Android data
                           '''),
                ], None),
    'local': ((LocalLinuxTarget, LocalConnection), COMMON_TARGET_PARAMS, None),
}

# name --> assistant
ASSISTANTS = {
    'linux': LinuxAssistant,
    'android': AndroidAssistant,
    'local': LinuxAssistant,
}

# name --> ((platform_class, conn_class), params_list, defaults)
# Note: normally, connection is defined by the Target name, but
#       platforms may choose to override it
PLATFORMS = {
    'generic': ((Platform, None), COMMON_PLATFORM_PARAMS, None),
    'juno': ((Juno, None), COMMON_PLATFORM_PARAMS + VEXPRESS_PLATFORM_PARAMS,
            {
                 'vemsd_mount': '/media/JUNO',
                 'baudrate': 115200,
                 'bootloader': 'u-boot',
                 'hard_reset_method': 'dtr',
            }),
    'tc2': ((TC2, None), COMMON_PLATFORM_PARAMS + VEXPRESS_PLATFORM_PARAMS,
            {
                 'vemsd_mount': '/media/VEMSD',
                 'baudrate': 38400,
                 'bootloader': 'bootmon',
                 'hard_reset_method': 'reboottxt',
            }),
    'gem5': ((Gem5SimulationPlatform, Gem5Connection), GEM5_PLATFORM_PARAMS, None),
}

class DefaultTargetDescriptor(TargetDescriptor):

    name = 'devlib_targets'

    description = """
    The default target descriptor that provides descriptions in the form
    <platform>_<target>.

    These map directly onto ``Target``\ s and ``Platform``\ s supplied by ``devlib``.

    """

    def get_descriptions(self):
        result = []
        for target_name, target_tuple in TARGETS.iteritems():
            (target, conn), target_params = self._get_item(target_tuple)
            assistant = ASSISTANTS[target_name]
            conn_params =  CONNECTION_PARAMS[conn]
            for platform_name, platform_tuple in PLATFORMS.iteritems():
                (platform, plat_conn), platform_params = self._get_item(platform_tuple)
                name = '{}_{}'.format(platform_name, target_name)
                td = TargetDescription(name, self)
                td.target = target
                td.platform = platform
                td.assistant = assistant
                td.target_params = target_params
                td.platform_params = platform_params
                td.assistant_params = assistant.parameters

                if plat_conn:
                    td.conn = plat_conn
                    td.conn_params =  CONNECTION_PARAMS[plat_conn]
                else:
                    td.conn = conn
                    td.conn_params = conn_params

                result.append(td)
        return result

    def _get_item(self, item_tuple):
        cls, params, defaults = item_tuple
        if not defaults:
            return cls, params

        param_map = OrderedDict((p.name, copy(p)) for p in params)
        for name, value in defaults.iteritems():
            if name not in param_map:
                raise ValueError('Unexpected default "{}"'.format(name))
            param_map[name].default = value
        return cls, param_map.values()
<commit_msg>framework/target: handle aliases<commit_after>from collections import OrderedDict
from copy import copy

from devlib import (LinuxTarget, AndroidTarget, LocalLinuxTarget,
                    Platform, Juno, TC2, Gem5SimulationPlatform,
                    AdbConnection, SshConnection, LocalConnection,
                    Gem5Connection)

from wa.framework import pluginloader
from wa.framework.configuration.core import get_config_point_map
from wa.framework.exception import PluginLoaderError
from wa.framework.plugin import Plugin, Parameter
from wa.framework.target.assistant import LinuxAssistant, AndroidAssistant
from wa.utils.types import list_of_strings, list_of_ints
from wa.utils.misc import isiterable

def get_target_descriptions(loader=pluginloader):
    targets = {}
    for cls in loader.list_target_descriptors():
        descriptor = cls()
        for desc in descriptor.get_descriptions():
            if desc.name in targets:
                msg = 'Duplicate target "{}" returned by {} and {}'
                prev_dtor = targets[desc.name].source
                raise PluginLoaderError(msg.format(desc.name, prev_dtor.name,
                                                   descriptor.name))
            targets[desc.name] = desc
    return targets.values()

def instantiate_target(tdesc, params, connect=None, extra_platform_params=None):
    target_params = get_config_point_map(tdesc.target_params)
    platform_params = get_config_point_map(tdesc.platform_params)
    conn_params = get_config_point_map(tdesc.conn_params)
    assistant_params = get_config_point_map(tdesc.assistant_params)

    tp, pp, cp = {}, {}, {}

    for supported_params, new_params in (target_params, tp), (platform_params, pp), (conn_params, cp):
        for name, value in supported_params.iteritems():
            if value.default and name == value.name:
                new_params[name] = value.default

    for name, value in params.iteritems():
        if name in target_params:
            tp[name] = value
        elif name in platform_params:
            pp[name] = value
        elif name in conn_params:
            cp[name] = value
        elif name in assistant_params:
            pass
        else:
            msg = 'Unexpected parameter for {}: {}'
            raise ValueError(msg.format(tdesc.name, name))

    for pname, pval in (extra_platform_params or {}).iteritems():
        if pname in pp:
            raise RuntimeError('Platform parameter clash: {}'.format(pname))
        pp[pname] = pval

    tp['platform'] = (tdesc.platform or Platform)(**pp)
    if cp:
        tp['connection_settings'] = cp
    if tdesc.connection:
        tp['conn_cls'] = tdesc.connection
    if connect is not None:
        tp['connect'] = connect

    return tdesc.target(**tp)

def instantiate_assistant(tdesc, params, target):
    assistant_params = {}
    for param in tdesc.assistant_params:
        if param.name in params:
            assistant_params[param.name] = params[param.name]
        elif param.default:
            assistant_params[param.name] = param.default
    return tdesc.assistant(target, **assistant_params)

class TargetDescription(object):

    def __init__(self, name, source, description=None, target=None, platform=None,
                 conn=None, assistant=None, target_params=None, platform_params=None,
                 conn_params=None, assistant_params=None):
        self.name = name
        self.source = source
        self.description = description
        self.target = target
        self.platform = platform
        self.connection = conn
        self.assistant = assistant
        self._set('target_params', target_params)
        self._set('platform_params', platform_params)
        self._set('conn_params', conn_params)
        self._set('assistant_params', assistant_params)

    def get_default_config(self):
        param_attrs = ['target_params', 'platform_params',
                       'conn_params', 'assistant_params']
        config = {}
        for pattr in param_attrs:
            for n, p in getattr(self, pattr).itervalues():
                config[n] = p.default
        return config

    def _set(self, attr, vals):
        if vals is None:
            vals = {}
        elif isiterable(vals):
            if not hasattr(vals, 'iteritems'):
                vals = {v.name: v for v in vals}
        else:
            msg = '{} must be iterable; got "{}"'
            raise ValueError(msg.format(attr, vals))
        setattr(self, attr, vals)

class TargetDescriptor(Plugin):

    kind = 'target_descriptor'

    def get_descriptions(self):
        return []

COMMON_TARGET_PARAMS = [
    Parameter('working_directory', kind=str,
              description='''
              On-target working directory that will be used by WA. This
              directory must be writable by the user WA logs in as without
              the need for privilege elevation.
              '''),
    Parameter('executables_directory', kind=str,
              description='''
              On-target directory where WA will install its executable
              binaries.  This location must allow execution. This location does
              *not* need to be writable by unprivileged users or rooted devices
              (WA will install with elevated privileges as necessary).
              '''),
    Parameter('modules', kind=list_of_strings,
              description='''
              A list of additional modules to be installed for the target.

              ``devlib`` implements functionality for particular subsystems as
              modules.  A number of "default" modules (e.g. for cpufreq
              subsystem) are loaded automatically, unless explicitly disabled.
              If additional modules need to be loaded, they may be specified
              using this parameter.

              Please see ``devlib`` documentation for information on the available
              modules.
              '''),
    Parameter('load_default_modules', kind=bool, default=True,
              description='''
              A number of modules (e.g. for working with the cpufreq subsystem) are
              loaded by default when a Target is instantiated. Setting this to
              ``True`` would suppress that, ensuring that only the base Target
              interface is initialized.

              You may want to set this to ``False`` if there is a problem with one
              or more default modules on your platform (e.g. your device is
              unrooted and cpufreq is not accessible to unprivileged users), or
              if ``Target`` initialization is taking too long for your platform.
              '''),
]

COMMON_PLATFORM_PARAMS = [
    Parameter('core_names', kind=list_of_strings,
              description='''
              List of names of CPU cores in the order that they appear to the
              kernel. If not specified, it will be inferred from the platform.
              '''),
    Parameter('core_clusters', kind=list_of_ints,
              description='''
              Cluster mapping corresponding to the cores in ``core_names``.
              Cluster indexing starts at ``0``.  If not specified, this will be
              inferred from ``core_names`` -- consecutive cores with the same
              name will be assumed to share a cluster.
              '''),
    Parameter('big_core', kind=str,
              description='''
              The name of the big cores in a big.LITTLE system. If not
              specified, this will be inferred, either from the name (if one of
              the names in ``core_names`` matches known big cores), or by
              assuming that the last cluster is big.
              '''),
    Parameter('model', kind=str,
              description='''
              Hardware model of the platform. If not specified, an attempt will
              be made to read it from target.
              '''),
    Parameter('modules', kind=list_of_strings,
              description='''
              An additional list of modules to be loaded into the target.
              '''),
]

VEXPRESS_PLATFORM_PARAMS = [
    Parameter('serial_port', kind=str,
              description='''
              The serial device/port on the host for the initial connection to
              the target (used for early boot, flashing, etc).
              '''),
    Parameter('baudrate', kind=int,
              description='''
              Baud rate for the serial connection.
              '''),
    Parameter('vemsd_mount', kind=str,
              description='''
              VExpress MicroSD card mount location. This is a MicroSD card in
              the VExpress device that is mounted on the host via USB. The card
              contains configuration files for the platform and firmware and
              kernel images to be flashed.
              '''),
    Parameter('bootloader', kind=str,
              allowed_values=['uefi', 'uefi-shell', 'u-boot', 'bootmon'],
              description='''
              Selects the bootloader mechanism used by the board. Depending on
              firmware version, a number of possible boot mechanisms may be use.

              Please see ``devlib`` documentation for descriptions.
              '''),
    Parameter('hard_reset_method', kind=str,
              allowed_values=['dtr', 'reboottxt'],
              description='''
              There are a couple of ways to reset VersatileExpress board if the
              software running on the board becomes unresponsive. Both require
              configuration to be enabled (please see ``devlib`` documentation).

              ``dtr``: toggle the DTR line on the serial connection
              ``reboottxt``: create ``reboot.txt`` in the root of the VEMSD mount.

              '''),
]

GEM5_PLATFORM_PARAMS = [
    Parameter('gem5_bin', kind=str, mandatory=True,
              description='''
              Path to the gem5 binary
              '''),
    Parameter('gem5_args', kind=str, mandatory=True,
              description='''
              Arguments to be passed to the gem5 binary
              '''),
    Parameter('gem5_virtio', kind=str, mandatory=True,
              description='''
              VirtIO device setup arguments to be passed to gem5. VirtIO is used
              to transfer files between the simulation and the host.
              '''),
    Parameter('name', kind=str, default='gem5',
              description='''
              The name for the gem5 "device".
              '''),
]

CONNECTION_PARAMS = {
    AdbConnection: [
        Parameter('device', kind=str,
                description="""
                ADB device name
                """),
        Parameter('adb_server', kind=str,
                description="""
                ADB server to connect to.
                """),
    ],
    SshConnection: [
        Parameter('host', kind=str, mandatory=True,
                description="""
                Host name or IP address of the target.
                """),
        Parameter('username', kind=str, mandatory=True,
                description="""
                User name to connect with
                """),
        Parameter('password', kind=str,
                description="""
                Password to use.
                """),
        Parameter('keyfile', kind=str,
                description="""
                Key file to use
                """),
        Parameter('port', kind=int,
                description="""
                The port SSH server is listening on on the target.
                """),
        Parameter('telnet', kind=bool, default=False,
                description="""
                If set to ``True``, a Telnet connection, rather than
                SSH will be used.
                """),
        Parameter('password_prompt', kind=str,
                description="""
                Password prompt to expect
                """),
        Parameter('original_prompt', kind=str,
                description="""
                Original shell prompt to expect.
                """),
        Parameter('sudo_cmd', kind=str,
                default="sudo -- sh -c '{}'",
                description="""
                Sudo command to use. Must have ``"{}"``` specified
                somewher in the string it indicate where the command
                to be run via sudo is to go.
                """),
    ],
    Gem5Connection: [
        Parameter('host', kind=str, mandatory=False,
                description="""
                Host name or IP address of the target.
                """),
        Parameter('username', kind=str, default='root',
                description="""
                User name to connect to gem5 simulation.
                """),
        Parameter('password', kind=str,
                description="""
                Password to use.
                """),
        Parameter('port', kind=int,
                description="""
                The port SSH server is listening on on the target.
                """),
        Parameter('password_prompt', kind=str,
                description="""
                Password prompt to expect
                """),
        Parameter('original_prompt', kind=str,
                description="""
                Original shell prompt to expect.
                """),
    ],
    LocalConnection: [
        Parameter('password', kind=str,
                description="""
                Password to use for sudo. if not specified, the user will
                be prompted during intialization.
                """),
        Parameter('keep_password', kind=bool, default=True,
                description="""
                If ``True`` (the default), the password will be cached in
                memory after it is first obtained from the user, so that the
                user would not be prompted for it again.
                """),
        Parameter('unrooted', kind=bool, default=False,
                description="""
                Indicate that the target should be considered unrooted; do not
                attempt sudo or ask the user for their password.
                """),
    ],
}

# name --> ((target_class, conn_class), params_list, defaults, assistant_class)
TARGETS = {
    'linux': ((LinuxTarget, SshConnection), COMMON_TARGET_PARAMS, None),
    'android': ((AndroidTarget, AdbConnection), COMMON_TARGET_PARAMS +
                [Parameter('package_data_directory', kind=str, default='/data/data',
                           description='''
                           Directory containing Android data
                           '''),
                ], None),
    'local': ((LocalLinuxTarget, LocalConnection), COMMON_TARGET_PARAMS, None),
}

# name --> assistant
ASSISTANTS = {
    'linux': LinuxAssistant,
    'android': AndroidAssistant,
    'local': LinuxAssistant,
}

# name --> ((platform_class, conn_class), params_list, defaults)
# Note: normally, connection is defined by the Target name, but
#       platforms may choose to override it
PLATFORMS = {
    'generic': ((Platform, None), COMMON_PLATFORM_PARAMS, None),
    'juno': ((Juno, None), COMMON_PLATFORM_PARAMS + VEXPRESS_PLATFORM_PARAMS,
            {
                 'vemsd_mount': '/media/JUNO',
                 'baudrate': 115200,
                 'bootloader': 'u-boot',
                 'hard_reset_method': 'dtr',
            }),
    'tc2': ((TC2, None), COMMON_PLATFORM_PARAMS + VEXPRESS_PLATFORM_PARAMS,
            {
                 'vemsd_mount': '/media/VEMSD',
                 'baudrate': 38400,
                 'bootloader': 'bootmon',
                 'hard_reset_method': 'reboottxt',
            }),
    'gem5': ((Gem5SimulationPlatform, Gem5Connection), GEM5_PLATFORM_PARAMS, None),
}

class DefaultTargetDescriptor(TargetDescriptor):

    name = 'devlib_targets'

    description = """
    The default target descriptor that provides descriptions in the form
    <platform>_<target>.

    These map directly onto ``Target``\ s and ``Platform``\ s supplied by ``devlib``.

    """

    def get_descriptions(self):
        result = []
        for target_name, target_tuple in TARGETS.iteritems():
            (target, conn), target_params = self._get_item(target_tuple)
            assistant = ASSISTANTS[target_name]
            conn_params =  CONNECTION_PARAMS[conn]
            for platform_name, platform_tuple in PLATFORMS.iteritems():
                (platform, plat_conn), platform_params = self._get_item(platform_tuple)
                name = '{}_{}'.format(platform_name, target_name)
                td = TargetDescription(name, self)
                td.target = target
                td.platform = platform
                td.assistant = assistant
                td.target_params = target_params
                td.platform_params = platform_params
                td.assistant_params = assistant.parameters

                if plat_conn:
                    td.conn = plat_conn
                    td.conn_params =  CONNECTION_PARAMS[plat_conn]
                else:
                    td.conn = conn
                    td.conn_params = conn_params

                result.append(td)
        return result

    def _get_item(self, item_tuple):
        cls, params, defaults = item_tuple
        if not defaults:
            return cls, params

        param_map = OrderedDict((p.name, copy(p)) for p in params)
        for name, value in defaults.iteritems():
            if name not in param_map:
                raise ValueError('Unexpected default "{}"'.format(name))
            param_map[name].default = value
        return cls, param_map.values()
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: iso-8859-1 -*-
# A rule-based style and grammar checker
# Copyright (C) 2002,2003 Daniel Naber <daniel.naber@t-online.de>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or (at
# your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
# USA

import codecs
import getopt
import os
import pre as re
import socket
import string
import sys
import time

import Tagger
import Chunker
import Rules
import SentenceSplitter

class TextChecker:
	"""A rule-based style and grammar checker."""

	entities = { 	"amp" : "&",
					"pound": "P",		# fixme: use ""
					"eacute": "e",
					"aacute": "a",
					"bquo": "\"",
					"equo": "\"",
					"ecirc": "e",
					"quot": "'",
					#"deg": u"",
					"dollar": "$",
					"egrave": "e",
					"percnt": "&",
					"ndash": "-",
					"mdash": "--",
					"hellip": "...",
					"lsqb": "[",
					"rsqb": "]",
					"uuml": u"",
					"auml": u"",
					"ouml": u"",
					"Uuml": u"",
					"Auml": u"",
					"Ouml": u""
				}

	def __init__(self, grammar, falsefriends, words, \
		builtin, textlanguage, mothertongue, max_sentence_length):
		# Which rules are activated (a lists of IDs):
		self.grammar = grammar
		self.falsefriends = falsefriends
		self.words = words
		self.builtin = builtin
		self.textlanguage = textlanguage
		self.mothertongue = mothertongue
		self.max_sentence_length = max_sentence_length
		self.tagger = Tagger.Tagger()
		self.chunker = Chunker.Chunker()
		rules = Chunker.Rules()
		self.chunker.setRules(rules)
		self.tagger.bindData()
		self.rules = Rules.Rules(self.max_sentence_length, self.grammar,\
			self.words, self.builtin, self.falsefriends, \
			textlanguage, mothertongue)
		self.bnc_paras = 0
		self.bnc_sentences = 0
		return

	def checkFile(self, filename):
		"""Check a text file and return the results as an XML formatted list
		of possible errors."""
		f = open(filename)
		text = f.read()
		f.close()
		(rule_matches, result, tagged_words) = self.check(text)
		return (rule_matches, result, tagged_words)

	def check(self, text):
		"""Check a text string and return the results as an XML formatted list
		of possible errors."""
		splitter = SentenceSplitter.SentenceSplitter()
		sentences = splitter.split(text)
		#print sentences
		#tx = time.time()
		rule_matches = []
		char_counter = 0
		all_tagged_words = []
		for sentence in sentences:
			tagged_words = self.tagger.tagText(sentence)
			#print tagged_words
			chunks = self.chunker.chunk(tagged_words)
			#print "CHUNKS: %s" % chunks
			tagged_words.insert(0, ('', None, 'SENT_START'))
			tagged_words.append(('', None, 'SENT_END'))
			all_tagged_words.extend(tagged_words)
			#print "time1: %.2fsec" % (time.time()-tx)
			#tx = time.time()
			for rule in self.rules.rules:
				matches = rule.match(tagged_words, chunks, char_counter)
				rule_matches.extend(matches)
				#print "time2: %.2fsec" % (time.time()-tx)
				#tx = time.time()
			for triple in sentence:
				char_counter = char_counter + len(triple[0])

		if not self.builtin or "WHITESPACE" in self.builtin:
			whitespace_rule = Rules.WhitespaceRule()
			rule_matches.extend(whitespace_rule.match(all_tagged_words))

		#print "###%s
" % str(all_tagged_words)
		rule_match_list = []
		for rule_match in rule_matches:
			rule_match_list.append(rule_match.toXML())
		xml_part = "<errors>\n%s\n</errors>\n" % string.join(rule_match_list, "\n")
		#print "time3: %.2fsec" % (time.time()-tx)
		# TODO: optionally return tagged text
		#print "2=>%.2f" % (time.time()-tx)
		return (rule_matches, xml_part, all_tagged_words)

	def cleanEntities(self, s):
		"""Replace only the most common BNC entities with their
		ASCII respresentation."""
		for key in self.entities:
			s = re.compile("&%s;?" % key).sub("%s" % self.entities[key], s)
		return s

	def checkBNCFiles(self, directory, checker):
		"""Recursively load all files from a directory, extract
		all paragraphs and feed them to the style and grammar checker
		one by one."""
		para_regex = re.compile("
(.*?)
", re.DOTALL)
		sentence_regex = re.compile("<s n=\d+>", re.DOTALL)
		xml_regex = re.compile("<.*?>", re.DOTALL)
		whitespace_regex = re.compile("\s+", re.DOTALL)
		files = []
		filemode = 0
		if os.path.isfile(directory):		# call with a filename is okay
			files = [directory]
			filemode = 1
		else:
			files = os.listdir(directory)
		for file in files:
			filename = None
			if filemode:
				filename = file
			else:
				filename = os.path.join(directory, file)
			if os.path.isdir(filename):
				#print filename
				self.checkBNCFiles(filename, checker)
			elif os.path.isfile(filename) and filename.find(".") != -1:
				print >> sys.stderr, "Ignoring %s" % filename
			elif os.path.isfile(filename):
				print >> sys.stderr, "FILE=%s" % filename
				f = open(filename, 'r')
				s = f.read()
				f.close()
				s = unicode(s, 'iso-8859-1')
				s_matches = sentence_regex.findall(s)
				self.bnc_sentences = self.bnc_sentences + len(s_matches)
				matches = para_regex.findall(s)
				for match in matches:
					#print len(match)
					self.bnc_paras = self.bnc_paras + 1
					s = xml_regex.sub("", match)
					s = whitespace_regex.sub(" ", s)
					s = self.cleanEntities(s)
					s = s.strip()
					(rule_matches, result, tagged_words) = checker.check(s)
					if len(rule_matches) == 0:
						pass
						#print >> sys.stderr, "No errors found."
					else:
						for rule_match in rule_matches:
							s_mark = "%s***%s" % (s[:rule_match.from_pos], s[rule_match.from_pos:])
							print "%s:\n\n%s" % (filename, s_mark.encode('utf8'), result.encode('utf8'))
							#print "%s:\n\n%s" % (filename, result.encode('utf8'))
		return

def usage():
	print "Usage: TextChecker.py [OPTION] <filename>"
	print "  -h, --help               Show this help"
	print "  -c, --check              Check directory with BNC files in SGML format"
	print "  -g, --grammar=...        Use only these grammar rules"
	print "  -f, --falsefriends=...   Use only these false friend rules"
	print "  -w, --words=...          Use only these style/word rules"
	print "  -b, --builtin=...        Use only these builtin rules"
	print "  -m, --mothertongue=...   Your native language"
	print "  -t, --textlanguage=...   The text's language"
	print "  -l, --sentencelength=... Maximum sentence length"
	return

def main():
	options = None
	rest = None
	try:
		(options, rest) = getopt.getopt(sys.argv[1:], 'hcg:f:w:b:m:t:l:', \
			['help', 'check', 'grammar=', 'falsefriends=', 'words=', \
			'builtin=', 'mothertongue=', 'textlanguage=', 'sentencelength='])
	except getopt.GetoptError,e :
		print >> sys.stderr, "Error: ", e
		usage()
		sys.exit(2)
	grammar = None
	falsefriends = None
	words = None
	builtin = None
	textlanguage = mothertongue = None
	max_sentence_length = None

	for o, a in options:
		if o in ("-g", "--grammar"):
			grammar = a.split(",")
		elif o in ("-f", "--falsefriends"):
			falsefriends = a.split(",")
		elif o in ("-w", "--words"):
			words = a.split(",")
		elif o in ("-b", "--builtin"):
			builtin = a.split(",")
		elif o in ("-m", "--mothertongue"):
			mothertongue = a
		elif o in ("-t", "--textlanguage"):
			textlanguage = a
		elif o in ("-l", "--sentencelength"):
			max_sentence_length = a

	for o, a in options:
		if o in ("-h", "--help"):
			usage()
			sys.exit(0)
		elif o in ("-c", "--check"):
			checker = TextChecker(grammar, falsefriends, words, \
				builtin, textlanguage, mothertongue, max_sentence_length)
			for filename in rest:
				checker.checkBNCFiles(filename, checker)
			print >> sys.stderr, "Checked %d sentences in %d paragraphs." % \
				(checker.bnc_sentences, checker.bnc_paras)
			sys.exit(0)

	if len(rest) == 1:
		checker = TextChecker(grammar, falsefriends, words, builtin, \
			textlanguage, mothertongue, max_sentence_length)
		(rule_matches, result, tagged_words) = checker.checkFile(rest[0])
		if not result:
			print >> sys.stderr, "No errors found."
		else:
			print result.encode('latin1')
	else:
		usage()
		sys.exit(1)
	return

if __name__ == "__main__":
    main()
<commit_msg>ugly hack to avoid type error, needs better fix<commit_after>#!/usr/bin/python
# -*- coding: iso-8859-1 -*-
# A rule-based style and grammar checker
# Copyright (C) 2002,2003 Daniel Naber <daniel.naber@t-online.de>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or (at
# your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
# USA

import codecs
import getopt
import os
import pre as re
import socket
import string
import sys
import time

import Tagger
import Chunker
import Rules
import SentenceSplitter

class TextChecker:
	"""A rule-based style and grammar checker."""

	entities = { 	"amp" : "&",
					"pound": "P",		# fixme: use ""
					"eacute": "e",
					"aacute": "a",
					"bquo": "\"",
					"equo": "\"",
					"ecirc": "e",
					"quot": "'",
					#"deg": u"",
					"dollar": "$",
					"egrave": "e",
					"percnt": "&",
					"ndash": "-",
					"mdash": "--",
					"hellip": "...",
					"lsqb": "[",
					"rsqb": "]",
					"uuml": "u",	#fixme: use
					"auml": "a",	# see above!
					"ouml": "o",
					"Uuml": "U",
					"Auml": "A",
					"Ouml": "O"
				}

	def __init__(self, grammar, falsefriends, words, \
		builtin, textlanguage, mothertongue, max_sentence_length):
		# Which rules are activated (a lists of IDs):
		self.grammar = grammar
		self.falsefriends = falsefriends
		self.words = words
		self.builtin = builtin
		self.textlanguage = textlanguage
		self.mothertongue = mothertongue
		self.max_sentence_length = max_sentence_length
		self.tagger = Tagger.Tagger()
		self.chunker = Chunker.Chunker()
		rules = Chunker.Rules()
		self.chunker.setRules(rules)
		self.tagger.bindData()
		self.rules = Rules.Rules(self.max_sentence_length, self.grammar,\
			self.words, self.builtin, self.falsefriends, \
			textlanguage, mothertongue)
		self.bnc_paras = 0
		self.bnc_sentences = 0
		return

	def checkFile(self, filename):
		"""Check a text file and return the results as an XML formatted list
		of possible errors."""
		f = open(filename)
		text = f.read()
		f.close()
		(rule_matches, result, tagged_words) = self.check(text)
		return (rule_matches, result, tagged_words)

	def check(self, text):
		"""Check a text string and return the results as an XML formatted list
		of possible errors."""
		splitter = SentenceSplitter.SentenceSplitter()
		sentences = splitter.split(text)
		#print sentences
		#tx = time.time()
		rule_matches = []
		char_counter = 0
		all_tagged_words = []
		for sentence in sentences:
			tagged_words = self.tagger.tagText(sentence)
			#print tagged_words
			chunks = self.chunker.chunk(tagged_words)
			#print "CHUNKS: %s" % chunks
			tagged_words.insert(0, ('', None, 'SENT_START'))
			tagged_words.append(('', None, 'SENT_END'))
			all_tagged_words.extend(tagged_words)
			#print "time1: %.2fsec" % (time.time()-tx)
			#tx = time.time()
			for rule in self.rules.rules:
				matches = rule.match(tagged_words, chunks, char_counter)
				rule_matches.extend(matches)
				#print "time2: %.2fsec" % (time.time()-tx)
				#tx = time.time()
			for triple in sentence:
				char_counter = char_counter + len(triple[0])

		if not self.builtin or "WHITESPACE" in self.builtin:
			whitespace_rule = Rules.WhitespaceRule()
			rule_matches.extend(whitespace_rule.match(all_tagged_words))

		#print "###%s
" % str(all_tagged_words)
		rule_match_list = []
		for rule_match in rule_matches:
			rule_match_list.append(rule_match.toXML())
		xml_part = "<errors>\n%s\n</errors>\n" % string.join(rule_match_list, "\n")
		#print "time3: %.2fsec" % (time.time()-tx)
		# TODO: optionally return tagged text
		#print "2=>%.2f" % (time.time()-tx)
		return (rule_matches, xml_part, all_tagged_words)

	def cleanEntities(self, s):
		"""Replace only the most common BNC entities with their
		ASCII respresentation."""
		try:
			for key in self.entities:
				s = re.compile("&%s;?" % key).sub("%s" % self.entities[key], s)
		except TypeError:
			# FIXME: what to do here?!
			print >> sys.stderr, "TypeError: '%s'" % s
		return s

	def checkBNCFiles(self, directory, checker):
		"""Recursively load all files from a directory, extract
		all paragraphs and feed them to the style and grammar checker
		one by one."""
		para_regex = re.compile("
(.*?)
", re.DOTALL)
		sentence_regex = re.compile("<s n=\d+>", re.DOTALL)
		xml_regex = re.compile("<.*?>", re.DOTALL)
		whitespace_regex = re.compile("\s+", re.DOTALL)
		files = []
		filemode = 0
		if os.path.isfile(directory):		# call with a filename is okay
			files = [directory]
			filemode = 1
		else:
			files = os.listdir(directory)
		for file in files:
			filename = None
			if filemode:
				filename = file
			else:
				filename = os.path.join(directory, file)
			if os.path.isdir(filename):
				#print filename
				self.checkBNCFiles(filename, checker)
			elif os.path.isfile(filename) and filename.find(".") != -1:
				print >> sys.stderr, "Ignoring %s" % filename
			elif os.path.isfile(filename):
				print >> sys.stderr, "FILE=%s" % filename
				f = open(filename, 'r')
				s = f.read()
				f.close()
				s = unicode(s, 'iso-8859-1')
				s_matches = sentence_regex.findall(s)
				self.bnc_sentences = self.bnc_sentences + len(s_matches)
				matches = para_regex.findall(s)
				for match in matches:
					#print len(match)
					self.bnc_paras = self.bnc_paras + 1
					s = xml_regex.sub("", match)
					s = whitespace_regex.sub(" ", s)
					s = self.cleanEntities(s)
					s = s.strip()
					(rule_matches, result, tagged_words) = checker.check(s)
					if len(rule_matches) == 0:
						pass
						#print >> sys.stderr, "No errors found."
					else:
						for rule_match in rule_matches:
							s_mark = "%s***%s" % (s[:rule_match.from_pos], s[rule_match.from_pos:])
							print "%s:\n\n%s" % (filename, s_mark.encode('utf8'), result.encode('utf8'))
							#print "%s:\n\n%s" % (filename, result.encode('utf8'))
		return

def usage():
	print "Usage: TextChecker.py [OPTION] <filename>"
	print "  -h, --help               Show this help"
	print "  -c, --check              Check directory with BNC files in SGML format"
	print "  -g, --grammar=...        Use only these grammar rules"
	print "  -f, --falsefriends=...   Use only these false friend rules"
	print "  -w, --words=...          Use only these style/word rules"
	print "  -b, --builtin=...        Use only these builtin rules"
	print "  -m, --mothertongue=...   Your native language"
	print "  -t, --textlanguage=...   The text's language"
	print "  -l, --sentencelength=... Maximum sentence length"
	return

def main():
	options = None
	rest = None
	try:
		(options, rest) = getopt.getopt(sys.argv[1:], 'hcg:f:w:b:m:t:l:', \
			['help', 'check', 'grammar=', 'falsefriends=', 'words=', \
			'builtin=', 'mothertongue=', 'textlanguage=', 'sentencelength='])
	except getopt.GetoptError,e :
		print >> sys.stderr, "Error: ", e
		usage()
		sys.exit(2)
	grammar = None
	falsefriends = None
	words = None
	builtin = None
	textlanguage = mothertongue = None
	max_sentence_length = None

	for o, a in options:
		if o in ("-g", "--grammar"):
			grammar = a.split(",")
		elif o in ("-f", "--falsefriends"):
			falsefriends = a.split(",")
		elif o in ("-w", "--words"):
			words = a.split(",")
		elif o in ("-b", "--builtin"):
			builtin = a.split(",")
		elif o in ("-m", "--mothertongue"):
			mothertongue = a
		elif o in ("-t", "--textlanguage"):
			textlanguage = a
		elif o in ("-l", "--sentencelength"):
			max_sentence_length = a

	for o, a in options:
		if o in ("-h", "--help"):
			usage()
			sys.exit(0)
		elif o in ("-c", "--check"):
			checker = TextChecker(grammar, falsefriends, words, \
				builtin, textlanguage, mothertongue, max_sentence_length)
			for filename in rest:
				checker.checkBNCFiles(filename, checker)
			print >> sys.stderr, "Checked %d sentences in %d paragraphs." % \
				(checker.bnc_sentences, checker.bnc_paras)
			sys.exit(0)

	if len(rest) == 1:
		checker = TextChecker(grammar, falsefriends, words, builtin, \
			textlanguage, mothertongue, max_sentence_length)
		(rule_matches, result, tagged_words) = checker.checkFile(rest[0])
		if not result:
			print >> sys.stderr, "No errors found."
		else:
			print result.encode('latin1')
	else:
		usage()
		sys.exit(1)
	return

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Mon Aug 31 10:17:09 2015

@author: mje
"""
import mne
from mne.minimum_norm import (apply_inverse_epochs, read_inverse_operator)
import numpy as np

from my_settings import *

# Using the same inverse operator when inspecting single trials Vs. evoked
snr = 1.0  # Standard assumption for average data but using it for single trial
lambda2 = 1.0 / snr ** 2
method = "dSPM"  # use dSPM method (could also be MNE or sLORETA)

for subject in subjects:
    # Load data
    inverse_operator = read_inverse_operator(mne_folder +
                                             "%s-inv.fif" % subject)
    epochs = mne.read_epochs(epochs_folder +
                             "%s_filtered_ica_mc_tsss-epo.fif" % subject)

    for cond in epochs.event_id.keys():
        stcs = apply_inverse_epochs(epochs[cond], inverse_operator, lambda2,
                                    method, pick_ori="normal")
        exec("stcs_%s = stcs" % cond)

lbl_ctl_left = []
lbl_ent_left = []
lbl_ctl_right = []
lbl_ent_right = []

for j in range(len(subjects[:1])):
    src = mne.read_source_spaces(mne_folder + "%s-oct6-src.fif" % subjects[j])
    labels = mne.read_labels_from_annot(subjects[j], parc='PALS_B12_Brodmann',
                                        regexp="Bro",
                                        subjects_dir=subjects_dir)
    labels_occ = [labels[6]]

    lbl_ent_left.append(mne.extract_label_time_course(ent_left[j],
                                                      labels=labels_occ,
                                                      src=src,
                                                      mode="pca_flip"))

    lbl_ctl_left.append(mne.extract_label_time_course(ctl_left[j],
                                                      labels=labels_occ,
                                                      src=src,
                                                      mode="pca_flip"))

    lbl_ent_right.append(mne.extract_label_time_course(ent_right[j],
                                                       labels=labels_occ,
                                                       src=src,
                                                       mode="pca_flip"))

    lbl_ctl_right.append(mne.extract_label_time_course(ctl_right[j],
                                                       labels=labels_occ,
                                                       src=src,
                                                       mode="pca_flip"))

lbl_ent_left = np.squeeze(np.asarray(lbl_ent_left))
lbl_ctl_left = np.squeeze(np.asarray(lbl_ctl_left))
lbl_ent_right = np.squeeze(np.asarray(lbl_ent_right))
lbl_ctl_right = np.squeeze(np.asarray(lbl_ctl_right))

def ITC_over_time(data, faverage=True):
    """Calculate the ITC over time.

    Parameters
    ----------
    data : numpy array
        It should be trials x channels x frequencies x times.
    faverage : bool
        If true the average is returned, If false each frequency is returned.

    Returns
    -------
    result : numpy array
        The result is a numpy array with the length equal to the number of
        trials.
    """
    result = np.empty([data.shape[1], data.shape[0]])

    for freq in range(result.shape[0]):
        for i in range(result.shape[1]):
            result[freq, i] =\
                np.abs(np.mean(np.exp(1j * (np.angle(data[i, freq, :])))))

    if faverage:
        result = result.mean(axis=0).squeeze()

    return result<commit_msg>added bits for pipeline<commit_after># -*- coding: utf-8 -*-
"""
Created on Mon Aug 31 10:17:09 2015

@author: mje
"""
import mne
from mne.minimum_norm import (apply_inverse_epochs, read_inverse_operator)
from mne.time_frequency import cwt_morlet
import numpy as np

from my_settings import *

# Using the same inverse operator when inspecting single trials Vs. evoked
snr = 1.0  # Standard assumption for average data but using it for single trial
lambda2 = 1.0 / snr ** 2
method = "dSPM"  # use dSPM method (could also be MNE or sLORETA)

for subject in subjects[:1]:
    # Load data
    inverse_operator = read_inverse_operator(mne_folder +
                                             "%s-inv.fif" % subject)
    epochs = mne.read_epochs(epochs_folder +
                             "%s_ds_filtered_ica_mc_tsss-epo.fif" % subject)
    epochs.resample(250, n_jobs=4)

    for cond in epochs.event_id.keys():
        stcs = apply_inverse_epochs(epochs[cond], inverse_operator, lambda2,
                                    method, pick_ori="normal")
        exec("stcs_%s = stcs" % cond)

# morhping

lbl_ctl_left = []
lbl_ent_left = []
lbl_ctl_right = []
lbl_ent_right = []

src = mne.read_source_spaces(mne_folder + "%s-oct6-src.fif" % "0004")
labels = mne.read_labels_from_annot("0004", parc='PALS_B12_Brodmann',
                                    regexp="Bro",
                                    subjects_dir=subjects_dir)
labels_occ = [labels[6]]

lbl_left = []

for j in range(len(stcs_ent_left)):
    lbl_left.append(mne.extract_label_time_course(stcs_ent_left[j],
                                                      labels=labels_occ,
                                                      src=src,
                                                      mode="mean_flip"))

for j in range(len(stcs_ctl_left)):
    lbl_left.append(mne.extract_label_time_course(stcs_ctl_left[j],
                                                      labels=labels_occ,
                                                      src=src,
                                                      mode="mean_flip"))

for j in range(len(stcs_ent_right)):
    lbl_left.append(mne.extract_label_time_course(stcs_ent_right[j],
                                                       labels=labels_occ,
                                                       src=src,
                                                       mode="mean_flip"))

for j in range(len(stcs_ctl_right)):
    lbl_ctl_right.append(mne.extract_label_time_course(stcs_ctl_right[j],
                                                       labels=labels_occ,
                                                       src=src,
                                                       mode="mean_flip"))

lbl_ent_left = np.squeeze(np.asarray(lbl_ent_left))
lbl_ctl_left = np.squeeze(np.asarray(lbl_ctl_left))
lbl_ent_right = np.squeeze(np.asarray(lbl_ent_right))
lbl_ctl_right = np.squeeze(np.asarray(lbl_ctl_right))

def ITC_over_trials(data, faverage=True):
    """Calculate the ITC over time.

    Parameters
    ----------
    data : numpy array
        It should be trials x channels x frequencies x times.
    faverage : bool
        If true the average is returned, If false each frequency is returned.

    Returns
    -------
    result : numpy array
        The result is a numpy array with the length equal to the number of
        trials.
    """
    result = np.empty([data.shape[1], data.shape[-1]])

    for freq in range(result.shape[0]):
        for i in range(result.shape[1]):
            result[freq, i] =\
                np.abs(np.mean(np.exp(1j * (np.angle(data[:, freq, i])))))

    if faverage:
        result = result.mean(axis=0).squeeze()

    return result

lbl_left = np.squeeze(np.asarray(lbl_left))

freqs = np.arange(8, 13, 1)
n_cycle = freqs / 3.

tfr_ent_left  = cwt_morlet(lbl_left, epochs.info["sfreq"], freqs,
                           use_fft=True, n_cycles=n_cycle)

tfr_clt_left  = cwt_morlet(lbl_left, epochs.info["sfreq"], freqs,
                           use_fft=True, n_cycles=n_cycle)

tfr_ent_right  = cwt_morlet(lbl_left, epochs.info["sfreq"], freqs,
                            use_fft=True, n_cycles=n_cycle)

tfr_clt_right = cwt_morlet(lbl_ctl_right, epochs.info["sfreq"], freqs,
                           use_fft=True, n_cycles=n_cycle)<|endoftext|>