<commit_before>__author__ = 'Frederik Diehl'

from apsis.models.experiment import Experiment
from abc import ABCMeta, abstractmethod
import Queue
import sys
from time import sleep
import signal
import multiprocessing
import Queue

class Optimizer(object):
    """
    This defines a basic Optimizer interface.

    An optimizer contains an _experiment, which represents the last known state
     of the experiment. It can be updated using update, while the next
     candidates can be generated with get_next_candidates, which represent the
     actual work of the optimizer.
    Additionally, it contains an exit function, which can be used to cleanly
    exit the optimizer. Since the optimizer may well implement multicore or
    distributed architectures, it is necessary to cleanly exit.

    Parameters
    ----------
    SUPPORTED_PARAM_TYPES : list
        A list of the supported parameters for this optimizer. Not all
        parameters may be supported by any optimizer.
    _experiment : Experiment
        The current state of the experiment. Is used as a base for the the
        optimization.
    """
    __metaclass__ = ABCMeta

    SUPPORTED_PARAM_TYPES = []

    _experiment = None

    def __init__(self, experiment, optimizer_params):
        """
        Initializes the optimizer.

        Parameters
        ----------
        experiment : Experiment
            The experiment representing the current state of the execution.
        optimizer_params : dict, optional
            Dictionary of the optimizer parameters. If None, some standard
            parameters will be assumed.

        Raises
        ------
        ValueError
            Iff the experiment is not supported.
        """
        if not self._is_experiment_supported(experiment):
            raise ValueError("Experiment contains unsupported parameters. "
                             "Optimizer %s supports %s, experiment parameters "
                             "are %s." %(self.__class__.__name__,
                                         self.SUPPORTED_PARAM_TYPES,
                                         experiment.parameter_definitions))
        self._experiment = experiment

    def update(self, experiment):
        """
        Updates the experiment.

        Implementation note: This function (for the base class) only sets
        the new _experiment. Subclasses can call it to ensure correct setting
        of the experiment parameter.

        Parameters
        ----------
        experiment : Experiment
            The experiment representing the current state of the execution.

        Raises
        ------
        ValueError
            Iff the experiment is not supported.
        """
        if self._is_experiment_supported(experiment):
            raise ValueError("Experiment contains unsupported parameters.")
        self._experiment = experiment

    @abstractmethod
    def get_next_candidates(self, num_candidates=1):
        """
        Returns a number of candidates.

        Parameters
        ----------
        num_candidates : strictly positive int, optional
            The maximum number of candidates returned. Note that there may
            be less than that in the list.

        Returns
        -------
        candidates : list of candidates
            The list of candidates to evaluate next. May be None (which
            represents no current candidates being available), and may have
            between one and num_candidates candidates.
        """
        pass

    def exit(self):
        """
        Cleanly exits this optimizer.

        Note that, since optimizers may be of a multiprocessing or
        distributed nature, this function is important for stopping.

        Internal Note: This function, which will be inherited, does nothing. If
         an optimizer does not require another any other behaviour, it will not
         be necessary to redefine it.
        """
        pass

    def _is_experiment_supported(self, experiment):
        """
        Tests whether all parameter types in experiment are supported by this
        optimizer.

        Parameters
        ----------
        experiment : Experiment
            The experiment to test.

        Returns
        -------
        supported : bool
            False iff one or more of experiment's parameter definitions are not
            supported.
        """
        for name, pd in experiment.parameter_definitions.iteritems():
            if not self._is_supported_param_type(pd):
                return False
        return True

    def _is_supported_param_type(self, param):
        """
        Tests whether a certain parameter is supported by the optimizer.

        Parameters
        ----------
        param :
            The parameter to be tested

        Result
        ------
        is_supported : bool
            True iff param is supported by this optimizer.
        """
        if isinstance(self.SUPPORTED_PARAM_TYPES, list):
            for sup in self.SUPPORTED_PARAM_TYPES:
                if isinstance(param, sup):
                    return True

        return False

class QueueBasedOptimizer(Optimizer):
    """
    This implements a queue-based optimizer.

    A queue-based optimizer is an abstraction of another optimizer. It works by
     putting the other optimizer into another process, then communicating
     with it via queues and a QueueBackend instance. This means easy
     deployability without having to change code.

    Internally, the QueueBackend puts new candidates onto the
    optimizer_out_queue, keeping it at min_candidates, until it receives a
    new update. In that case, all current candidates in the out_queue are
    deleted.

    Parameters
    ----------
    _optimizer_in_queue : Queue
        The queue with which you can send data (experiments) to the optimizer.
    _optimizer_out_queue : Queue
        The queue on which you can receive data.
    """
    _optimizer_in_queue = None
    _optimizer_out_queue = None

    _optimizer_process = None

    def __init__(self, optimizer_class, experiment, optimizer_params=None):
        """
        Initializes a new QueueBasedOptimizer class.

        Parameters
        ----------
        optimizer_class : an Optimizer subclass
            The class of optimizer this should abstract from. The optimizer is
            then initialized here.
        experiment : Experiment
            The experiment representing the current state of the execution.
        optimizer_params : dict, optional
            Dictionary of the optimizer parameters. If None, some standard
            parameters will be assumed.
            Supports the parameter "min_candidates", which sets the number
            of candidates that should be kept ready. Default is 5.
            Supports the parameter "update_time", which sets the minimum time
            in seconds between checking for updates. Default is 0.1s
        """
        self._optimizer_in_queue = multiprocessing.Queue()
        self._optimizer_out_queue = multiprocessing.Queue()

        optimizer = QueueBackend(optimizer_class, optimizer_params, experiment,
                                 self._optimizer_out_queue,
                                 self._optimizer_in_queue)
        optimizer.start()

        super(QueueBasedOptimizer, self).__init__(experiment, optimizer_params)

    def get_next_candidates(self, num_candidates=1):
        next_candidates = []
        try:
            for i in range(num_candidates):
                new_candidate = self._optimizer_out_queue.get_nowait()
                next_candidates.append(new_candidate)
        except Queue.Empty:
            pass
        return next_candidates

    def update(self, experiment):
        self._optimizer_in_queue.put(experiment)

    def exit(self):
        """
        Also closes the optimizer.

        It does so by putting "exit" into the in_queue and closing both queues.
        """
        if self._optimizer_in_queue is not None:
            self._optimizer_in_queue.put("exit")
        if self._optimizer_in_queue is not None:
            self._optimizer_in_queue.close()
        if self._optimizer_out_queue is not None:
            self._optimizer_out_queue.close()

class QueueBackend(multiprocessing.Process):
    """
    This is the backend for QueueBasedOptimizer.

    It ensures there's a compatible request to the stored optimizer.

    Parameters
    ----------
    _experiment : Experiment
        The current state of the experiment.
    _out_queue : Queue
        The queue on which to put the candidates.
    _in_queue : Queue
        The queue on which to receive the new experiments.
    _optimizer : Optimizer
        The optimizer this abstracts from
    _min_candidates : int
        The minimum numbers of candidates to keep ready.
    _exited : bool
        Whether this process should exit (has seen the exit signal).
    """
    _experiment = None
    _out_queue = None
    _in_queue = None

    _optimizer = None

    _min_candidates = None
    _exited = None
    _update_time = None

    def __init__(self, optimizer_class, optimizer_params, experiment, out_queue, in_queue):
        """
        Initializes this backend.

        Parameters
        ----------
        optimizer_class : an Optimizer subclass
            The class of optimizer this should abstract from. The optimizer is
            then initialized here.
        experiment : Experiment
            The experiment representing the current state of the execution.
        optimizer_params : dict, optional
            Dictionary of the optimizer parameters. If None, some standard
            parameters will be assumed.
            Supports the parameter "min_candidates", which sets the number
            of candidates that should be kept ready.
        out_queue : Queue
            The queue on which to put the candidates.
        in_queue : Queue
            The queue on which to receive the new experiments.
        """
        self._out_queue = out_queue
        self._in_queue = in_queue
        self._min_candidates = optimizer_params.get("min_candidates", 5)
        self._update_time = optimizer_params.get("update_time", 0.1)
        self._optimizer = optimizer_class(optimizer_params, experiment)
        self._exited = False
        self._experiment = experiment
        multiprocessing.Process.__init__(self)

    def run(self):
        """
        The run function of this process, checking for new updates.

        Every _update_time seconds, it checks both the necessity of a new
        generation of candidates, and whether an update is necessary.

        It also makes sure all queues will be closed.
        """
        try:
            while not self._exited:
                self._check_generation()
                self._check_update()
                sleep(0.1)
        finally:
            if self._in_queue is not None:
                self._in_queue.close()
            if self._out_queue is not None:
                self._out_queue.close()

    def _check_update(self):
        """
        This checks for the availability of updates.

        Specifically, it does the following:
        If the in_queue is not empty (that is, there are one or more
        experiments available) it takes the last, most recently added. If
        one of the elements is "exit", it will exit instead.
        The latest experiment is then used to call the update function of the
        abstracted optimizer.
        Additionally, it will empty the out_queue, since we assume it has more,
        better information available.
        """
        new_update = None
        while not self._in_queue.empty():
            try:
                new_update = self._in_queue.get_nowait()
            except Queue.Empty:
                pass
            if new_update == "exit":
                self._exited = True
                return
        if new_update is not None:
            # clear the out queue. We'll soon have new information.
            try:
                while not self._out_queue.empty():
                    self._out_queue.get_nowait()
            except Queue.Empty:
                pass
            self._experiment = new_update
            self._optimizer.update(self._experiment)

    def _check_generation(self):
        """
        This checks whether new candidates should be generated.

        Specifically, it tests whether less than min_candidates are available
        in the out_queue. If so, it will (via optimizer.get_next_candidates)
        try to add min_candidates candidates.
        """
        try:
            if (self._out_queue.empty() or
                           self._out_queue.qsize < self._min_candidates):
                new_candidates = self._optimizer.get_next_candidates(
                    num_candidates=self._min_candidates)
                if new_candidates is None:
                    return
                for c in new_candidates:
                    self._out_queue.put_nowait(c)
        except Queue.Full:
            return
<commit_msg>Fixed error with the initialization of QueueBasedOptimizer. Assigning QueueBased the supported param types of the optimizer_class it abstracts from.<commit_after>__author__ = 'Frederik Diehl'

from apsis.models.experiment import Experiment
from abc import ABCMeta, abstractmethod
import Queue
import sys
from time import sleep
import signal
import multiprocessing
import Queue

class Optimizer(object):
    """
    This defines a basic Optimizer interface.

    An optimizer contains an _experiment, which represents the last known state
     of the experiment. It can be updated using update, while the next
     candidates can be generated with get_next_candidates, which represent the
     actual work of the optimizer.
    Additionally, it contains an exit function, which can be used to cleanly
    exit the optimizer. Since the optimizer may well implement multicore or
    distributed architectures, it is necessary to cleanly exit.

    Parameters
    ----------
    SUPPORTED_PARAM_TYPES : list
        A list of the supported parameters for this optimizer. Not all
        parameters may be supported by any optimizer.
    _experiment : Experiment
        The current state of the experiment. Is used as a base for the the
        optimization.
    """
    __metaclass__ = ABCMeta

    SUPPORTED_PARAM_TYPES = []

    _experiment = None

    def __init__(self, experiment, optimizer_params):
        """
        Initializes the optimizer.

        Parameters
        ----------
        experiment : Experiment
            The experiment representing the current state of the execution.
        optimizer_params : dict, optional
            Dictionary of the optimizer parameters. If None, some standard
            parameters will be assumed.

        Raises
        ------
        ValueError
            Iff the experiment is not supported.
        """
        if not self._is_experiment_supported(experiment):
            raise ValueError("Experiment contains unsupported parameters. "
                             "Optimizer %s supports %s, experiment parameters "
                             "are %s." %(self.__class__.__name__,
                                         self.SUPPORTED_PARAM_TYPES,
                                         experiment.parameter_definitions))
        self._experiment = experiment

    def update(self, experiment):
        """
        Updates the experiment.

        Implementation note: This function (for the base class) only sets
        the new _experiment. Subclasses can call it to ensure correct setting
        of the experiment parameter.

        Parameters
        ----------
        experiment : Experiment
            The experiment representing the current state of the execution.

        Raises
        ------
        ValueError
            Iff the experiment is not supported.
        """
        if self._is_experiment_supported(experiment):
            raise ValueError("Experiment contains unsupported parameters.")
        self._experiment = experiment

    @abstractmethod
    def get_next_candidates(self, num_candidates=1):
        """
        Returns a number of candidates.

        Parameters
        ----------
        num_candidates : strictly positive int, optional
            The maximum number of candidates returned. Note that there may
            be less than that in the list.

        Returns
        -------
        candidates : list of candidates
            The list of candidates to evaluate next. May be None (which
            represents no current candidates being available), and may have
            between one and num_candidates candidates.
        """
        pass

    def exit(self):
        """
        Cleanly exits this optimizer.

        Note that, since optimizers may be of a multiprocessing or
        distributed nature, this function is important for stopping.

        Internal Note: This function, which will be inherited, does nothing. If
         an optimizer does not require another any other behaviour, it will not
         be necessary to redefine it.
        """
        pass

    def _is_experiment_supported(self, experiment):
        """
        Tests whether all parameter types in experiment are supported by this
        optimizer.

        Parameters
        ----------
        experiment : Experiment
            The experiment to test.

        Returns
        -------
        supported : bool
            False iff one or more of experiment's parameter definitions are not
            supported.
        """
        for name, pd in experiment.parameter_definitions.iteritems():
            if not self._is_supported_param_type(pd):
                return False
        return True

    def _is_supported_param_type(self, param):
        """
        Tests whether a certain parameter is supported by the optimizer.

        Parameters
        ----------
        param :
            The parameter to be tested

        Result
        ------
        is_supported : bool
            True iff param is supported by this optimizer.
        """
        if isinstance(self.SUPPORTED_PARAM_TYPES, list):
            for sup in self.SUPPORTED_PARAM_TYPES:
                if isinstance(param, sup):
                    return True

        return False

class QueueBasedOptimizer(Optimizer):
    """
    This implements a queue-based optimizer.

    A queue-based optimizer is an abstraction of another optimizer. It works by
     putting the other optimizer into another process, then communicating
     with it via queues and a QueueBackend instance. This means easy
     deployability without having to change code.

    Internally, the QueueBackend puts new candidates onto the
    optimizer_out_queue, keeping it at min_candidates, until it receives a
    new update. In that case, all current candidates in the out_queue are
    deleted.

    Parameters
    ----------
    _optimizer_in_queue : Queue
        The queue with which you can send data (experiments) to the optimizer.
    _optimizer_out_queue : Queue
        The queue on which you can receive data.
    """
    _optimizer_in_queue = None
    _optimizer_out_queue = None

    _optimizer_process = None

    def __init__(self, optimizer_class, experiment, optimizer_params=None):
        """
        Initializes a new QueueBasedOptimizer class.

        Parameters
        ----------
        optimizer_class : an Optimizer subclass
            The class of optimizer this should abstract from. The optimizer is
            then initialized here.
        experiment : Experiment
            The experiment representing the current state of the execution.
        optimizer_params : dict, optional
            Dictionary of the optimizer parameters. If None, some standard
            parameters will be assumed.
            Supports the parameter "min_candidates", which sets the number
            of candidates that should be kept ready. Default is 5.
            Supports the parameter "update_time", which sets the minimum time
            in seconds between checking for updates. Default is 0.1s
        """
        self._optimizer_in_queue = multiprocessing.Queue()
        self._optimizer_out_queue = multiprocessing.Queue()

        self.SUPPORTED_PARAM_TYPES = optimizer_class.SUPPORTED_PARAM_TYPES

        optimizer = QueueBackend(optimizer_class, optimizer_params, experiment,
                                 self._optimizer_out_queue,
                                 self._optimizer_in_queue)
        optimizer.start()

        super(QueueBasedOptimizer, self).__init__(experiment, optimizer_params)

    def get_next_candidates(self, num_candidates=1):
        next_candidates = []
        try:
            for i in range(num_candidates):
                new_candidate = self._optimizer_out_queue.get_nowait()
                next_candidates.append(new_candidate)
        except Queue.Empty:
            pass
        return next_candidates

    def update(self, experiment):
        self._optimizer_in_queue.put(experiment)

    def exit(self):
        """
        Also closes the optimizer.

        It does so by putting "exit" into the in_queue and closing both queues.
        """
        if self._optimizer_in_queue is not None:
            self._optimizer_in_queue.put("exit")
        if self._optimizer_in_queue is not None:
            self._optimizer_in_queue.close()
        if self._optimizer_out_queue is not None:
            self._optimizer_out_queue.close()

class QueueBackend(multiprocessing.Process):
    """
    This is the backend for QueueBasedOptimizer.

    It ensures there's a compatible request to the stored optimizer.

    Parameters
    ----------
    _experiment : Experiment
        The current state of the experiment.
    _out_queue : Queue
        The queue on which to put the candidates.
    _in_queue : Queue
        The queue on which to receive the new experiments.
    _optimizer : Optimizer
        The optimizer this abstracts from
    _min_candidates : int
        The minimum numbers of candidates to keep ready.
    _exited : bool
        Whether this process should exit (has seen the exit signal).
    """
    _experiment = None
    _out_queue = None
    _in_queue = None

    _optimizer = None

    _min_candidates = None
    _exited = None
    _update_time = None

    def __init__(self, optimizer_class, optimizer_params, experiment, out_queue, in_queue):
        """
        Initializes this backend.

        Parameters
        ----------
        optimizer_class : an Optimizer subclass
            The class of optimizer this should abstract from. The optimizer is
            then initialized here.
        experiment : Experiment
            The experiment representing the current state of the execution.
        optimizer_params : dict, optional
            Dictionary of the optimizer parameters. If None, some standard
            parameters will be assumed.
            Supports the parameter "min_candidates", which sets the number
            of candidates that should be kept ready.
        out_queue : Queue
            The queue on which to put the candidates.
        in_queue : Queue
            The queue on which to receive the new experiments.
        """
        self._out_queue = out_queue
        self._in_queue = in_queue
        self._min_candidates = optimizer_params.get("min_candidates", 5)
        self._update_time = optimizer_params.get("update_time", 0.1)
        self._optimizer = optimizer_class(experiment, optimizer_params)
        self._exited = False
        self._experiment = experiment
        multiprocessing.Process.__init__(self)

    def run(self):
        """
        The run function of this process, checking for new updates.

        Every _update_time seconds, it checks both the necessity of a new
        generation of candidates, and whether an update is necessary.

        It also makes sure all queues will be closed.
        """
        try:
            while not self._exited:
                self._check_generation()
                self._check_update()
                sleep(0.1)
        finally:
            if self._in_queue is not None:
                self._in_queue.close()
            if self._out_queue is not None:
                self._out_queue.close()

    def _check_update(self):
        """
        This checks for the availability of updates.

        Specifically, it does the following:
        If the in_queue is not empty (that is, there are one or more
        experiments available) it takes the last, most recently added. If
        one of the elements is "exit", it will exit instead.
        The latest experiment is then used to call the update function of the
        abstracted optimizer.
        Additionally, it will empty the out_queue, since we assume it has more,
        better information available.
        """
        new_update = None
        while not self._in_queue.empty():
            try:
                new_update = self._in_queue.get_nowait()
            except Queue.Empty:
                pass
            if new_update == "exit":
                self._exited = True
                return
        if new_update is not None:
            # clear the out queue. We'll soon have new information.
            try:
                while not self._out_queue.empty():
                    self._out_queue.get_nowait()
            except Queue.Empty:
                pass
            self._experiment = new_update
            self._optimizer.update(self._experiment)

    def _check_generation(self):
        """
        This checks whether new candidates should be generated.

        Specifically, it tests whether less than min_candidates are available
        in the out_queue. If so, it will (via optimizer.get_next_candidates)
        try to add min_candidates candidates.
        """
        try:
            if (self._out_queue.empty() or
                           self._out_queue.qsize < self._min_candidates):
                new_candidates = self._optimizer.get_next_candidates(
                    num_candidates=self._min_candidates)
                if new_candidates is None:
                    return
                for c in new_candidates:
                    self._out_queue.put_nowait(c)
        except Queue.Full:
            return
<|endoftext|>


<commit_before>#!/usr/bin/env python

__author__ = 'Copyright (c) 2015 Alan Yorinks All rights reserved.'

"""
Copyright (c) 2015 Alan Yorinks All rights reserved.

This program is free software; you can redistribute it and/or
modify it under the terms of the GNU  General Public
License as published by the Free Software Foundation; either
version 3 of the License, or (at your option) any later version.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
General Public License for more details.

You should have received a copy of the GNU General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

"""

"""
This example illustrates using callbacks for digital input, analog input and analog latches, in addition
to demonstrating both writing to an LED using PWM and with digital outputs.

A switch is used to turn an LED on and off, and a potentiometer sets the intensity of a second LED.
When the potentiometer exceeds a raw value of 1000, the program is terminated.
"""

import time
import signal
import sys

from PyMata.pymata import PyMata

# digital pins
GREEN_LED = 6
RED_LED = 9
PUSH_BUTTON = 12

# analog pin
POTENTIOMETER = 2

# index to retrieve data from an analog or digital callback list
DATA_VALUE = 2

# switch states
ON = 1
OFF = 0

# indices for data list passed to latch callback
LATCH_TYPE = 0
LATCH_PIN = 1
LATCH_DATA_VALUE = 2
LATCH_TIME_STAMP = 3

# callback functions
# set the LED to current state of the pushbutton switch
def cb_push_button(data):
    board.digital_write(GREEN_LED, data[DATA_VALUE])

# maximum value of pot is 1024, and the maximum output value for pwm is 255
# so we divide the current pot value by 4 to keep it in pwm range
def cb_potentiometer(data):
    board.analog_write(RED_LED, data[DATA_VALUE] // 4)

def cb_potentiometer_latch(data):
    # turn off LEDs
    board.analog_write(RED_LED, 0)
    board.digital_write(GREEN_LED, 0)

    # print all data from the latch callback including time stamp
    print('Latching Event Mode:%x  Pin:%d  Data Value:%d Time of Event:%s' % (data[LATCH_TYPE], data[LATCH_PIN],
                                                                              data[LATCH_DATA_VALUE], time.asctime(
        time.gmtime(data[LATCH_TIME_STAMP]))))
    # shut things down
    board.close()

# create a PyMata instance
board = PyMata("/dev/ttyACM0", True, False)

def signal_handler(sig, frame):
    print('You pressed Ctrl+C!!!!')
    if board is not None:
        board.reset()
    sys.exit(0)

signal.signal(signal.SIGINT, signal_handler)

# set pin modes
board.set_pin_mode(GREEN_LED, board.OUTPUT, board.DIGITAL)
board.set_pin_mode(RED_LED, board.PWM, board.DIGITAL)
board.set_pin_mode(PUSH_BUTTON, board.INPUT, board.DIGITAL, cb_push_button)
board.set_pin_mode(POTENTIOMETER, board.INPUT, board.ANALOG, cb_potentiometer)

# set the latch
board.set_analog_latch(POTENTIOMETER, board.ANALOG_LATCH_GTE, 1000, cb_potentiometer_latch)

# do nothing loop - program exits when latch data event occurs for potentiometer or timer expires
time.sleep(15)
print('Timer expired')
board.close()
<commit_msg>minor modifications<commit_after>#!/usr/bin/env python
"""
Copyright (c) 2015 Alan Yorinks All rights reserved.

This program is free software; you can redistribute it and/or
modify it under the terms of the GNU  General Public
License as published by the Free Software Foundation; either
version 3 of the License, or (at your option) any later version.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
General Public License for more details.

You should have received a copy of the GNU General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

This example illustrates using callbacks for digital input, analog input and
analog latches, in addition to demonstrating both writing to an LED using PWM
and with digital outputs.

A switch is used to turn an LED on and off, and a potentiometer sets the
intensity of a second LED. When the potentiometer exceeds a raw value of
1000, the program is terminated.
"""
import time
import signal
import sys

from PyMata.pymata import PyMata

# Digital pins
GREEN_LED = 6
RED_LED = 9
PUSH_BUTTON = 12

# Analog pin
POTENTIOMETER = 2

# Index to retrieve data from an analog or digital callback list
DATA_VALUE = 2

# Switch states
ON = 1
OFF = 0

# Indices for data list passed to latch callback
LATCH_TYPE = 0
LATCH_PIN = 1
LATCH_DATA_VALUE = 2
LATCH_TIME_STAMP = 3

# Callback functions
# Set the LED to current state of the pushbutton switch
def cb_push_button(data):
    board.digital_write(GREEN_LED, data[DATA_VALUE])

# Maximum value of pot is 1024, and the maximum output value for pwm is 255
# so we divide the current pot value by 4 to keep it in pwm range
def cb_potentiometer(data):
    board.analog_write(RED_LED, data[DATA_VALUE] // 4)

def cb_potentiometer_latch(data):
    # Turn off LEDs
    board.analog_write(RED_LED, 0)
    board.digital_write(GREEN_LED, 0)

    # Print all data from the latch callback including time stamp
    print('Latching Event Mode:%x  Pin:%d  Data Value:%d Time of Event:%s' %
          (data[LATCH_TYPE], data[LATCH_PIN],
           data[LATCH_DATA_VALUE],
           time.asctime(time.gmtime(data[LATCH_TIME_STAMP]))))
    # Shut things down
    board.close()

# Create a PyMata instance
board = PyMata("/dev/ttyACM0", verbose=True)

def signal_handler(sig, frame):
    print('You pressed Ctrl+C')
    if board is not None:
        board.reset()
    sys.exit(0)

signal.signal(signal.SIGINT, signal_handler)

# Set pin modes
board.set_pin_mode(GREEN_LED, board.OUTPUT, board.DIGITAL)
board.set_pin_mode(RED_LED, board.PWM, board.DIGITAL)
board.set_pin_mode(PUSH_BUTTON, board.INPUT, board.DIGITAL, cb_push_button)
board.set_pin_mode(POTENTIOMETER, board.INPUT, board.ANALOG, cb_potentiometer)

# Set the latch
board.set_analog_latch(POTENTIOMETER, board.ANALOG_LATCH_GTE, 1000,
                       cb_potentiometer_latch)

# Do nothing loop - program exits when latch data event occurs for
# potentiometer or timer expires
time.sleep(15)
print('Timer expired')
board.close()

<|endoftext|>


<commit_before># Copyright 2016 - 2018  Ternaris.
# SPDX-License-Identifier: AGPL-3.0-only

import logging
import os
from collections import OrderedDict
from contextlib import contextmanager
from pathlib import Path

import click
from pkg_resources import iter_entry_points

try:
    from marv_ee import VERSION_MESSAGE
except ImportError:
    VERSION_MESSAGE = """
%(prog)s (MARV Community Edition) %(version)s
Copyright (C) 2016 - 2021  Ternaris and the MARV Contributors.
License AGPL-3.0-only: GNU Affero General Public License v3.0 <https://spdx.org/licenses/AGPL-3.0.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.
""".strip()

FORMAT = os.environ.get('MARV_LOG_FORMAT', '%(asctime)s %(levelname).4s %(name)s %(message)s')
PDB = None
STATICX_PROG_PATH = os.environ.get('STATICX_PROG_PATH')

# needs to be in line with logging
LOGLEVEL = OrderedDict((
    ('critical', 50),
    ('error', 40),
    ('warning', 30),
    ('info', 20),
    ('verbose', 18),
    ('noisy', 15),
    ('debug', 10),
))

# TODO: duplicated from marv_api.utils
@contextmanager
def launch_pdb_on_exception(launch=True):
    """Return contextmanager launching pdb on exception.

    Example:
      Toggle launch behavior via env variable::

          with launch_pdb_on_exception(os.environ.get('PDB')):
              cli()

    """
    if launch:
        try:
            yield
        except Exception:  # pylint: disable=broad-except
            import pdb  # pylint: disable=import-outside-toplevel
            pdb.xpm()  # pylint: disable=no-member
    else:
        yield

def make_log_method(name, numeric):
    upper = name.upper()
    assert not hasattr(logging, upper)
    setattr(logging, upper, numeric)

    def method(self, msg, *args, **kw):
        if self.isEnabledFor(numeric):
            self._log(numeric, msg, args, **kw)  # pylint: disable=protected-access
    method.__name__ = name
    return method

def create_loglevels():
    cls = logging.getLoggerClass()
    for name, value in LOGLEVEL.items():
        if not hasattr(cls, name):
            logging.addLevelName(value, name.upper())
            method = make_log_method(name, value)
            setattr(cls, name, method)

create_loglevels()

def setup_logging(loglevel, verbosity=0, logfilter=None):
    create_loglevels()

    class Filter(logging.Filter):  # pylint: disable=too-few-public-methods
        def filter(self, record):
            if logfilter and not any(record.name.startswith(x) for x in logfilter):
                return False
            if record.name == 'aiosqlite' and 'returning exception' in record.msg:
                return False
            return True
    filter = Filter()

    formatter = logging.Formatter(FORMAT)
    handler = logging.StreamHandler()
    handler.addFilter(filter)
    handler.setFormatter(formatter)
    root = logging.getLogger()
    root.addHandler(handler)

    levels = list(LOGLEVEL.keys())
    level = levels[min(levels.index(loglevel) + verbosity, len(levels) - 1)]
    root.setLevel(LOGLEVEL[level])

    logging.getLogger('tortoise').setLevel(logging.INFO if os.environ.get('MARV_ECHO_SQL') else
                                           logging.WARN)

@click.group()
@click.option('--config',
              type=click.Path(dir_okay=False, exists=True, resolve_path=True),
              help='Path to config file'
              ' (default: 1. ./marv.conf, 2. /etc/marv/marv.conf)')
@click.option('--loglevel', default='info', show_default=True,
              type=click.Choice(LOGLEVEL.keys()),
              help='Set loglevel directly')
@click.option('--logfilter', multiple=True,
              help='Display only log messages for selected loggers')
@click.option('-v', '--verbose', 'verbosity', count=True,
              help='Increase verbosity beyond --loglevel')
@click.version_option(message=VERSION_MESSAGE)
@click.pass_context
def marv(ctx, config, loglevel, logfilter, verbosity):
    """Manage a Marv site."""
    if config is None:
        cwd = os.path.abspath(os.path.curdir)
        while cwd != os.path.sep:
            config = os.path.join(cwd, 'marv.conf')
            if os.path.exists(config):
                break
            cwd = os.path.dirname(cwd)
        else:
            config = '/etc/marv/marv.conf'
            if not os.path.exists(config):
                config = None
    if config is not None:
        ctx.obj = Path(config)
    setup_logging(loglevel, verbosity, logfilter)

def cli():
    """Setuptools entry_point."""
    ldpath = os.environ.get('LD_LIBRARY_PATH')
    if ldpath:
        os.environ['LD_LIBRARY_PATH'] = ':'.join(
            x
            for x in ldpath.split(':')
            if not x.startswith('/tmp/_MEI')
        )

    global PDB  # pylint: disable=global-statement
    PDB = bool(os.environ.get('PDB'))
    with launch_pdb_on_exception(PDB):
        for entry_point in iter_entry_points(group='marv_cli'):
            entry_point.load()
        prog_name = STATICX_PROG_PATH and Path(STATICX_PROG_PATH).name
        # pylint: disable=unexpected-keyword-arg,no-value-for-parameter
        marv(auto_envvar_prefix='MARV', prog_name=prog_name)

if __name__ == '__main__':
    cli()
<commit_msg>Add cli option to display copyright notes for EE<commit_after># Copyright 2016 - 2018  Ternaris.
# SPDX-License-Identifier: AGPL-3.0-only

import logging
import os
from collections import OrderedDict
from contextlib import contextmanager
from pathlib import Path

import click
from pkg_resources import iter_entry_points

try:
    from marv_ee import VERSION_MESSAGE, copying_option
except ImportError:
    VERSION_MESSAGE = """
%(prog)s (MARV Community Edition) %(version)s
Copyright (C) 2016 - 2021  Ternaris and the MARV Contributors.
License AGPL-3.0-only: GNU Affero General Public License v3.0 <https://spdx.org/licenses/AGPL-3.0.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.
""".strip()

    def copying_option(func):
        return func

FORMAT = os.environ.get('MARV_LOG_FORMAT', '%(asctime)s %(levelname).4s %(name)s %(message)s')
PDB = None
STATICX_PROG_PATH = os.environ.get('STATICX_PROG_PATH')

# needs to be in line with logging
LOGLEVEL = OrderedDict((
    ('critical', 50),
    ('error', 40),
    ('warning', 30),
    ('info', 20),
    ('verbose', 18),
    ('noisy', 15),
    ('debug', 10),
))

# TODO: duplicated from marv_api.utils
@contextmanager
def launch_pdb_on_exception(launch=True):
    """Return contextmanager launching pdb on exception.

    Example:
      Toggle launch behavior via env variable::

          with launch_pdb_on_exception(os.environ.get('PDB')):
              cli()

    """
    if launch:
        try:
            yield
        except Exception:  # pylint: disable=broad-except
            import pdb  # pylint: disable=import-outside-toplevel
            pdb.xpm()  # pylint: disable=no-member
    else:
        yield

def make_log_method(name, numeric):
    upper = name.upper()
    assert not hasattr(logging, upper)
    setattr(logging, upper, numeric)

    def method(self, msg, *args, **kw):
        if self.isEnabledFor(numeric):
            self._log(numeric, msg, args, **kw)  # pylint: disable=protected-access
    method.__name__ = name
    return method

def create_loglevels():
    cls = logging.getLoggerClass()
    for name, value in LOGLEVEL.items():
        if not hasattr(cls, name):
            logging.addLevelName(value, name.upper())
            method = make_log_method(name, value)
            setattr(cls, name, method)

create_loglevels()

def setup_logging(loglevel, verbosity=0, logfilter=None):
    create_loglevels()

    class Filter(logging.Filter):  # pylint: disable=too-few-public-methods
        def filter(self, record):
            if logfilter and not any(record.name.startswith(x) for x in logfilter):
                return False
            if record.name == 'aiosqlite' and 'returning exception' in record.msg:
                return False
            return True
    filter = Filter()

    formatter = logging.Formatter(FORMAT)
    handler = logging.StreamHandler()
    handler.addFilter(filter)
    handler.setFormatter(formatter)
    root = logging.getLogger()
    root.addHandler(handler)

    levels = list(LOGLEVEL.keys())
    level = levels[min(levels.index(loglevel) + verbosity, len(levels) - 1)]
    root.setLevel(LOGLEVEL[level])

    logging.getLogger('tortoise').setLevel(logging.INFO if os.environ.get('MARV_ECHO_SQL') else
                                           logging.WARN)

@click.group()
@click.option('--config',
              type=click.Path(dir_okay=False, exists=True, resolve_path=True),
              help='Path to config file'
              ' (default: 1. ./marv.conf, 2. /etc/marv/marv.conf)')
@click.option('--loglevel', default='info', show_default=True,
              type=click.Choice(LOGLEVEL.keys()),
              help='Set loglevel directly')
@click.option('--logfilter', multiple=True,
              help='Display only log messages for selected loggers')
@click.option('-v', '--verbose', 'verbosity', count=True,
              help='Increase verbosity beyond --loglevel')
@click.version_option(message=VERSION_MESSAGE)
@copying_option
@click.pass_context
def marv(ctx, config, loglevel, logfilter, verbosity):
    """Manage a Marv site."""
    if config is None:
        cwd = os.path.abspath(os.path.curdir)
        while cwd != os.path.sep:
            config = os.path.join(cwd, 'marv.conf')
            if os.path.exists(config):
                break
            cwd = os.path.dirname(cwd)
        else:
            config = '/etc/marv/marv.conf'
            if not os.path.exists(config):
                config = None
    if config is not None:
        ctx.obj = Path(config)
    setup_logging(loglevel, verbosity, logfilter)

def cli():
    """Setuptools entry_point."""
    ldpath = os.environ.get('LD_LIBRARY_PATH')
    if ldpath:
        os.environ['LD_LIBRARY_PATH'] = ':'.join(
            x
            for x in ldpath.split(':')
            if not x.startswith('/tmp/_MEI')
        )

    global PDB  # pylint: disable=global-statement
    PDB = bool(os.environ.get('PDB'))
    with launch_pdb_on_exception(PDB):
        for entry_point in iter_entry_points(group='marv_cli'):
            entry_point.load()
        prog_name = STATICX_PROG_PATH and Path(STATICX_PROG_PATH).name
        # pylint: disable=unexpected-keyword-arg,no-value-for-parameter
        marv(auto_envvar_prefix='MARV', prog_name=prog_name)

if __name__ == '__main__':
    cli()
<|endoftext|>


<commit_before>"""
This examples trains a CrossEncoder for the STSbenchmark task. A CrossEncoder takes a sentence pair
as input and outputs a label. Here, it output a continious labels 0...1 to indicate the similarity between the input pair.

It does NOT produce a sentence embedding and does NOT work for individual sentences.

Usage:
python training_stsbenchmark.py
"""
from torch.utils.data import DataLoader
import math
from sentence_transformers import LoggingHandler, util
from sentence_transformers.cross_encoder import CrossEncoder
from sentence_transformers.cross_encoder.evaluation import CECorrelationEvaluator
from sentence_transformers import InputExample
import logging
from datetime import datetime
import sys
import os
import gzip
import csv

#### Just some code to print debug information to stdout
logging.basicConfig(format='%(asctime)s - %(message)s',
                    datefmt='%Y-%m-%d %H:%M:%S',
                    level=logging.INFO,
                    handlers=[LoggingHandler()])
logger = logging.getLogger(__name__)
#### /print debug information to stdout

#Check if dataset exsist. If not, download and extract  it
sts_dataset_path = 'datasets/stsbenchmark.tsv.gz'

if not os.path.exists(sts_dataset_path):
    util.http_get('https://sbert.net/datasets/stsbenchmark.tsv.gz', sts_dataset_path)

#Define our Cross-Encoder
train_batch_size = 16
num_epochs = 4
model_save_path = 'output/training_stsbenchmark-'+datetime.now().strftime("%Y-%m-%d_%H-%M-%S")

#We use distilroberta-base as base model and set num_labels=1, which predicts a continous score between 0 and 1
model = CrossEncoder('distilroberta-base', num_labels=1)

# Read STSb dataset
logger.info("Read STSbenchmark train dataset")

train_samples = []
dev_samples = []
test_samples = []
with gzip.open(sts_dataset_path, 'rt', encoding='utf8') as fIn:
    reader = csv.DictReader(fIn, delimiter='\t', quoting=csv.QUOTE_NONE)
    for row in reader:
        score = float(row['score']) / 5.0  # Normalize score to range 0 ... 1

        if row['split'] == 'dev':
            dev_samples.append(InputExample(texts=[row['sentence1'], row['sentence2']], label=score))
        elif row['split'] == 'test':
            test_samples.append(InputExample(texts=[row['sentence1'], row['sentence2']], label=score))
        else:
            #As we want to get symmetric scores, i.e. CrossEncoder(A,B) = CrossEncoder(B,A), we pass both combinations to the train set
            train_samples.append(InputExample(texts=[row['sentence1'], row['sentence2']], label=score))
            train_samples.append(InputExample(texts=[row['sentence2'], row['sentence1']], label=score))

# We wrap train_samples (which is a List[InputExample]) into a pytorch DataLoader
train_dataloader = DataLoader(train_samples, shuffle=True, batch_size=train_batch_size)

# We add an evaluator, which evaluates the performance during training
evaluator = CECorrelationEvaluator.from_input_examples(dev_samples, name='sts-dev')

# Configure the training
warmup_steps = math.ceil(len(train_dataloader) * num_epochs * 0.1) #10% of train data for warm-up
logger.info("Warmup-steps: {}".format(warmup_steps))

# Train the model
model.fit(train_dataloader=train_dataloader,
          evaluator=evaluator,
          epochs=num_epochs,
          warmup_steps=warmup_steps,
          output_path=model_save_path)

<commit_msg>add test set score comp<commit_after>"""
This examples trains a CrossEncoder for the STSbenchmark task. A CrossEncoder takes a sentence pair
as input and outputs a label. Here, it output a continious labels 0...1 to indicate the similarity between the input pair.

It does NOT produce a sentence embedding and does NOT work for individual sentences.

Usage:
python training_stsbenchmark.py
"""
from torch.utils.data import DataLoader
import math
from sentence_transformers import LoggingHandler, util
from sentence_transformers.cross_encoder import CrossEncoder
from sentence_transformers.cross_encoder.evaluation import CECorrelationEvaluator
from sentence_transformers import InputExample
import logging
from datetime import datetime
import sys
import os
import gzip
import csv

#### Just some code to print debug information to stdout
logging.basicConfig(format='%(asctime)s - %(message)s',
                    datefmt='%Y-%m-%d %H:%M:%S',
                    level=logging.INFO,
                    handlers=[LoggingHandler()])
logger = logging.getLogger(__name__)
#### /print debug information to stdout

#Check if dataset exsist. If not, download and extract  it
sts_dataset_path = 'datasets/stsbenchmark.tsv.gz'

if not os.path.exists(sts_dataset_path):
    util.http_get('https://sbert.net/datasets/stsbenchmark.tsv.gz', sts_dataset_path)

#Define our Cross-Encoder
train_batch_size = 16
num_epochs = 4
model_save_path = 'output/training_stsbenchmark-'+datetime.now().strftime("%Y-%m-%d_%H-%M-%S")

#We use distilroberta-base as base model and set num_labels=1, which predicts a continous score between 0 and 1
model = CrossEncoder('distilroberta-base', num_labels=1)

# Read STSb dataset
logger.info("Read STSbenchmark train dataset")

train_samples = []
dev_samples = []
test_samples = []
with gzip.open(sts_dataset_path, 'rt', encoding='utf8') as fIn:
    reader = csv.DictReader(fIn, delimiter='\t', quoting=csv.QUOTE_NONE)
    for row in reader:
        score = float(row['score']) / 5.0  # Normalize score to range 0 ... 1

        if row['split'] == 'dev':
            dev_samples.append(InputExample(texts=[row['sentence1'], row['sentence2']], label=score))
        elif row['split'] == 'test':
            test_samples.append(InputExample(texts=[row['sentence1'], row['sentence2']], label=score))
        else:
            #As we want to get symmetric scores, i.e. CrossEncoder(A,B) = CrossEncoder(B,A), we pass both combinations to the train set
            train_samples.append(InputExample(texts=[row['sentence1'], row['sentence2']], label=score))
            train_samples.append(InputExample(texts=[row['sentence2'], row['sentence1']], label=score))

# We wrap train_samples (which is a List[InputExample]) into a pytorch DataLoader
train_dataloader = DataLoader(train_samples, shuffle=True, batch_size=train_batch_size)

# We add an evaluator, which evaluates the performance during training
evaluator = CECorrelationEvaluator.from_input_examples(dev_samples, name='sts-dev')

# Configure the training
warmup_steps = math.ceil(len(train_dataloader) * num_epochs * 0.1) #10% of train data for warm-up
logger.info("Warmup-steps: {}".format(warmup_steps))

# Train the model
model.fit(train_dataloader=train_dataloader,
          evaluator=evaluator,
          epochs=num_epochs,
          warmup_steps=warmup_steps,
          output_path=model_save_path)

##### Load model and eval on test set
model = CrossEncoder(model_save_path)

evaluator = CECorrelationEvaluator.from_input_examples(test_samples, name='sts-test')
evaluator(model)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright 2016 - 2018  Ternaris.
# SPDX-License-Identifier: AGPL-3.0-only

from __future__ import absolute_import, division, print_function

import logging
import os
from contextlib import contextmanager

import click
from collections import OrderedDict
from pkg_resources import iter_entry_points

FORMAT = os.environ.get('MARV_LOG_FORMAT', '%(asctime)s %(levelname).4s %(name)s %(message)s')
PDB = None

# needs to be in line with logging
LOGLEVEL = OrderedDict((
    ('critical', 50),
    ('error', 40),
    ('warning', 30),
    ('info', 20),
    ('verbose', 18),
    ('noisy', 15),
    ('debug', 10),
))

def make_log_method(name, numeric):
    upper = name.upper()
    assert not hasattr(logging, upper)
    setattr(logging, upper, numeric)

    def method(self, msg, *args, **kw):
        if self.isEnabledFor(numeric):
            self._log(numeric, msg, args, **kw)
    method.__name__ = name
    return method

def create_loglevels():
    cls = logging.getLoggerClass()
    for k, v in LOGLEVEL.items():
        if not hasattr(cls, k):
            logging.addLevelName(v, k.upper())
            method = make_log_method(k, v)
            setattr(cls, k, method)

create_loglevels()

@contextmanager
def launch_pdb_on_exception(launch=True):
    """Return contextmanager launching pdb upon exception.

    Use like this, to toggle via env variable:

    with launch_pdb_on_exception(os.environ.get('PDB')):
        cli()
    """
    # pylint: disable=bare-except,no-member
    try:
        yield
    except:  # noqa
        if launch:
            import pdb
            pdb.xpm()
        else:
            raise

def setup_logging(loglevel, verbosity=0, logfilter=None):
    create_loglevels()

    class Filter(logging.Filter):
        def filter(self, record):
            if logfilter and not any(record.name.startswith(x) for x in logfilter):
                return False
            if record.name == 'rospy.core' and \
               record.msg == 'signal_shutdown [atexit]':
                return False
            return True
    filter = Filter()

    formatter = logging.Formatter(FORMAT)
    handler = logging.StreamHandler()
    handler.addFilter(filter)
    handler.setFormatter(formatter)
    root = logging.getLogger()
    root.addHandler(handler)

    levels = LOGLEVEL.keys()
    level = levels[min(levels.index(loglevel) + verbosity, len(levels) - 1)]
    root.setLevel(LOGLEVEL[level])

    if os.environ.get('MARV_ECHO_SQL'):
        logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)

@click.group()
@click.option('--config',
              type=click.Path(dir_okay=False, exists=True, resolve_path=True),
              help='Path to config file'
              ' (default: 1. ./marv.conf, 2. /etc/marv/marv.conf)')
@click.option('--loglevel', default='info', show_default=True,
              type=click.Choice(LOGLEVEL.keys()),
              help='Set loglevel directly')
@click.option('--logfilter', multiple=True,
              help='Display only log messages for selected loggers')
@click.option('-v', '--verbose', 'verbosity', count=True,
              help='Increase verbosity beyond --loglevel')
@click.pass_context
def marv(ctx, config, loglevel, logfilter, verbosity):
    """Manage a Marv site"""
    if config is None:
        cwd = os.path.abspath(os.path.curdir)
        while cwd != os.path.sep:
            config = os.path.join(cwd, 'marv.conf')
            if os.path.exists(config):
                break
            cwd = os.path.dirname(cwd)
        else:
            config = '/etc/marv/marv.conf'
            if not os.path.exists(config):
                config = None
    ctx.obj = config
    setup_logging(loglevel, verbosity, logfilter)

def cli():
    """setuptools entry_point"""
    global PDB
    PDB = bool(os.environ.get('PDB'))
    with launch_pdb_on_exception(PDB):
        for ep in iter_entry_points(group='marv_cli'):
            ep.load()
        marv(auto_envvar_prefix='MARV')

if __name__ == '__main__':
    cli()
<commit_msg>Don't let pdb catch sys.exit among others<commit_after># -*- coding: utf-8 -*-
#
# Copyright 2016 - 2018  Ternaris.
# SPDX-License-Identifier: AGPL-3.0-only

from __future__ import absolute_import, division, print_function

import logging
import os
from contextlib import contextmanager

import click
from collections import OrderedDict
from pkg_resources import iter_entry_points

FORMAT = os.environ.get('MARV_LOG_FORMAT', '%(asctime)s %(levelname).4s %(name)s %(message)s')
PDB = None

# needs to be in line with logging
LOGLEVEL = OrderedDict((
    ('critical', 50),
    ('error', 40),
    ('warning', 30),
    ('info', 20),
    ('verbose', 18),
    ('noisy', 15),
    ('debug', 10),
))

def make_log_method(name, numeric):
    upper = name.upper()
    assert not hasattr(logging, upper)
    setattr(logging, upper, numeric)

    def method(self, msg, *args, **kw):
        if self.isEnabledFor(numeric):
            self._log(numeric, msg, args, **kw)
    method.__name__ = name
    return method

def create_loglevels():
    cls = logging.getLoggerClass()
    for k, v in LOGLEVEL.items():
        if not hasattr(cls, k):
            logging.addLevelName(v, k.upper())
            method = make_log_method(k, v)
            setattr(cls, k, method)

create_loglevels()

@contextmanager
def launch_pdb_on_exception(launch=True):
    """Return contextmanager launching pdb upon exception.

    Use like this, to toggle via env variable:

    with launch_pdb_on_exception(os.environ.get('PDB')):
        cli()
    """
    # pylint: disable=broad-except
    try:
        yield
    except Exception:  # noqa
        if launch:
            import pdb
            pdb.xpm()  # pylint: disable=no-member
        else:
            raise

def setup_logging(loglevel, verbosity=0, logfilter=None):
    create_loglevels()

    class Filter(logging.Filter):
        def filter(self, record):
            if logfilter and not any(record.name.startswith(x) for x in logfilter):
                return False
            if record.name == 'rospy.core' and \
               record.msg == 'signal_shutdown [atexit]':
                return False
            return True
    filter = Filter()

    formatter = logging.Formatter(FORMAT)
    handler = logging.StreamHandler()
    handler.addFilter(filter)
    handler.setFormatter(formatter)
    root = logging.getLogger()
    root.addHandler(handler)

    levels = LOGLEVEL.keys()
    level = levels[min(levels.index(loglevel) + verbosity, len(levels) - 1)]
    root.setLevel(LOGLEVEL[level])

    if os.environ.get('MARV_ECHO_SQL'):
        logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)

@click.group()
@click.option('--config',
              type=click.Path(dir_okay=False, exists=True, resolve_path=True),
              help='Path to config file'
              ' (default: 1. ./marv.conf, 2. /etc/marv/marv.conf)')
@click.option('--loglevel', default='info', show_default=True,
              type=click.Choice(LOGLEVEL.keys()),
              help='Set loglevel directly')
@click.option('--logfilter', multiple=True,
              help='Display only log messages for selected loggers')
@click.option('-v', '--verbose', 'verbosity', count=True,
              help='Increase verbosity beyond --loglevel')
@click.pass_context
def marv(ctx, config, loglevel, logfilter, verbosity):
    """Manage a Marv site"""
    if config is None:
        cwd = os.path.abspath(os.path.curdir)
        while cwd != os.path.sep:
            config = os.path.join(cwd, 'marv.conf')
            if os.path.exists(config):
                break
            cwd = os.path.dirname(cwd)
        else:
            config = '/etc/marv/marv.conf'
            if not os.path.exists(config):
                config = None
    ctx.obj = config
    setup_logging(loglevel, verbosity, logfilter)

def cli():
    """setuptools entry_point"""
    global PDB
    PDB = bool(os.environ.get('PDB'))
    with launch_pdb_on_exception(PDB):
        for ep in iter_entry_points(group='marv_cli'):
            ep.load()
        marv(auto_envvar_prefix='MARV')

if __name__ == '__main__':
    cli()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""

    DS2PageXml.py
    some functionality to handle pageXML files (add regions,...)

    Copyright Xerox(C) 2016 H. Déjean

    Developed  for the EU project READ. The READ project has received funding
    from the European Union’s Horizon 2020 research and innovation programme
    under grant agreement No 674943.

"""
from __future__ import absolute_import
from __future__ import  print_function
from __future__ import unicode_literals

import sys, os
from lxml import etree

try: #to ease the use without proper Python installation
    import TranskribusDU_version
except ImportError:
    sys.path.append( os.path.dirname(os.path.dirname( os.path.abspath(sys.argv[0]) )) )
    import TranskribusDU_version

from common.Component import Component

from xml_formats.PageXml import PageXml
from ObjectModel.xmlDSDocumentClass import XMLDSDocument

from util.unitConversion import convertDot2Pixel

class DS2PageXMLConvertor(Component):
    """
        conversion from DSXML to PageXML
    """
    #DEFINE the version, usage and description of this particular component
    usage = ""
    version = "v.01"
    description = "description: DS2PageXml conversion"

    #--- INIT -------------------------------------------------------------------------------------------------------------
    def __init__(self):
        """
        """
        Component.__init__(self, "DS2PageXml", self.usage, self.version, self.description)

        self.dpi = 300

        self.xrce_id=10000

        self.storagePath = ''

        self.dTagNameMapping = {'PAGE':'Page','TEXT':'TextLine', 'LINE':'TextLine','COLUMN':'TextRegion','REGION':'TextRegion','BLOCK':'TextRegion','GRAPHELT':'LineDrawingRegion','TABLE':'TableRegion','CELL':'TableCell'}

        self.pageXmlNS = None

        self.bMultiPages = False
        self.bRegionOnly = False

    def setParams(self, dParams):
        """
        Always call first the Component setParams
        """
        Component.setParams(self, dParams)
        if "bMultiPage" in dParams: self.bMultiPages =  dParams["bMultiPage"]
        if "bRegionOnly" in dParams: self.bRegionOnly =  dParams["bRegionOnly"]
        if "outdir" in dParams: self.storagePath =  dParams["outdir"]

    def setDPI(self,v): self.dpi=v

    def setStoragePath(self,p): self.storagePath=p

    def getCoord(self,DSObject):
        """
            create Coords value (polylines) from BoundingBox
            if object has @points: return points
        """

        if DSObject.hasAttribute('points'):
            return DSObject.getAttribute('points')
        else:
            return self.BB2Polylines(DSObject.getX(), DSObject.getY(),DSObject.getHeight(), DSObject.getWidth())

    def BB2Polylines(self,x,y,h,w):
        """
            convert BB values to polylines coords
        """
        # schema does not support neg int
        if x <0: x = abs(x)
        if y <0: y = abs(y)

        lx= list(map(lambda x:1.0*x*self.dpi/72.0, ( x,y, x+w,y, x+w,y+h, x,y+h, x,y)))
        myPoints = ' '.join(["%d,%d"%(xa,ya) for xa,ya  in zip(lx[0::2], lx[1::2])])
        return myPoints

    def DSPoint2PagePoints(self,sPoints):
        """
            "52.8,47.76,128.88,44.64"
            to
            451,246 451,1094 781,1094 781,246

        """
        lPoints = sPoints.split(",")
        lx= list(map(lambda x:1.0*float(x)*self.dpi/72.0, lPoints))
        # order left right
        xx =  list(zip(lx[0::2], lx[1::2]))
        xx.sort(key=lambda xy:xy[0])
        return ' '.join(["%d,%d"%(xa,ya) for xa,ya  in xx])

    def convertDSObject(self,DSObject,pageXmlParentNode):
        """
            convert DSObject and add it as child to pageXmlParentNode

             <TextLine id="line_1472550984091_215" custom="readingOrder {index:0;}">
                <Coords points="218,65 280,65 280,100 218,100"/>
                <Baseline points="218,95 280,95"/>
                <TextEquiv>
                    <Unicode>10.</Unicode>
                </TextEquiv>
            </TextLine>

        for table:
        <TableRegion id="Table_1484215666379_5" custom="readingOrder {index:92;}">
            <Coords points="221,246 781,246 781,1094 221,1094"/>
            <TableCell row="0" col="0" colSpan="1" id="TableCell_1484215672011_8">
                <Coords points="221,246 221,1094 451,1094 451,246"/>
                <CornerPts>0 1 2 3</CornerPts>
            </TableCell>
            <TableCell row="0" col="1" colSpan="1" id="TableCell_1484215672011_7">
                <Coords points="451,246 451,1094 781,1094 781,246"/>
                <CornerPts>0 1 2 3</CornerPts>
            </TableCell>
        </TableRegion>

        DS TEXT
             <TEXT x="52.8" y="41.04" height="6.72" width="76.08" font-size="20" y2="47.76" x2="128.88"
                points="52.8,41.04,128.88,41.04,128.88,47.76,52.8,47.76" blpoints="52.8,47.76,128.88,44.64" type="RB" id="p1_CVL-{e0004b40-06e0-4b85-97a0-2df0e1f0fe87}"/>
        """
        try:
            pageXmlName= self.dTagNameMapping[DSObject.getName()]
        except KeyError:
#             print DSObject.getName() ," not declared"
            return
        domNode= PageXml.createPageXmlNode(pageXmlName)
        if DSObject.getID():
            domNode.set("id", "xrce_%s"%DSObject.getID())
        else: self.addXRCEID(domNode)
        pageXmlParentNode.append(domNode)
        coordsNode = etree.Element('{%s}Coords'%(self.pageXmlNS))
#         coordsNode.setNs(self.pageXmlNS)
        coordsNode.set('points', self.BB2Polylines(DSObject.getX(),DSObject.getY(), DSObject.getHeight(),DSObject.getWidth()))
        domNode.append(coordsNode)

        for attr in ['custom', 'structure','col','type','DU_row','DU_header','DU_col']:
            if DSObject.hasAttribute(attr):
                domNode.set(attr, DSObject.getAttribute(attr))
        # if blpoints:  build Baseline

        # type
#         if DSObject.hasAttribute('type'):
#             domNode.set('type', DSObject.getAttribute('type'))

        # if blpoints:  build Baseline
        # <Baseline points="218,95 280,95"/>
        ## Baseline needs to be left-right!!
        if DSObject.hasAttribute('blpoints'):
            domBaseLine=etree.Element('{%s}Baseline'%(self.pageXmlNS))
#             domBaseLine.setNs(self.pageXmlNS)
            domBaseLine.set('points', self.DSPoint2PagePoints(DSObject.getAttribute('blpoints')))
            domNode.append(domBaseLine)

        # collect content and generate a textequiv
        #  <TextEquiv> <Unicode>des</Unicode>        </TextEquiv>
        if DSObject.getContent() is not None:
            TextEquivDom =etree.Element('{%s}TextEquiv'%(self.pageXmlNS))
            unicodeDom= etree.Element('{%s}Unicode'%(self.pageXmlNS))
            unicodeDom.text = DSObject.getContent()
            TextEquivDom.append(unicodeDom)
            domNode.append(TextEquivDom)

        ## specific attributes for cell
        ###  row="0" col="2" colSpan="1
        if pageXmlName == 'TableCell':
            domNode.set('row',str(DSObject.getIndex()[0]))
            domNode.set('col',str(DSObject.getIndex()[1]))
            cornerNode = etree.Element('{%s}CornerPts'%(self.pageXmlNS))
            cornerNode.text = "0 1 2 3"
#             cornerNode.setNs(self.pageXmlNS)
            domNode.append(cornerNode)
            coordsNode.set('colSpan',str(DSObject.getColSpan()))
            coordsNode.set('rowSpan',str(DSObject.getRowSpan()))

        #process objects
        for subobject in DSObject.getObjects():
            self.convertDSObject(subobject, domNode)

    def addXRCEID(self,node):
        node.set("id", "xrce_%d"%self.xrce_id)
        self.xrce_id += 1

    def convertOnlyRegion(self,OPage,pageXmlPageNODE):
        """
            populate pageXml with OPageRegion
        """
        # get REGION elements
        lElts= OPage.getAllNamedObjects('REGION')
        for DSObject in lElts:
            self.convertDSObject(DSObject,pageXmlPageNODE)

    def convertDSPage(self,OPage,pageXmlPageNODE):
        """
            populate pageXml with OPage
        """
        from ObjectModel.XMLDSGRAHPLINEClass import XMLDSGRAPHLINEClass

        lElts = OPage.getObjects()
        for DSObject in lElts:
            self.convertDSObject(DSObject,pageXmlPageNODE)

#         # get graphelt elements
        lElts= OPage.getAllNamedObjects(XMLDSGRAPHLINEClass)
        for DSObject in lElts:
            self.convertDSObject(DSObject,pageXmlPageNODE)

        # get table elements

    def storePageXmlSetofFiles(self,lListOfDocs):
        """
            write on disc the list of dom in the PageXml format
        """
        for i,(doc,img) in enumerate(lListOfDocs):
            if img is None: img='fakeimage.jpg'  # generated
            if self.storagePath == "":
                if os.path.dirname(self.inputFileName) =='':
                    self.outputFileName = img[:-3]+"_%.4d"%(i+1) + ".xml"
                else:
                    self.outputFileName = os.path.dirname(self.inputFileName)+os.sep+img[:-3]+"_%.4d"%(i+1) + ".xml"
            else:
                self.outputFileName = self.storagePath + os.sep+img[:-4]+"_%.4d"%(i+1) + ".xml"
            print("output: %s" % self.outputFileName)
            try:self.writeDom(doc, bIndent=True)
            except IOError as e:
                print(e)
                return -1
        return 0

    def storeMultiPageXml(self,lListDocs,outputFileName=None):
        """
            write a multipagePageXml file
        """
        from xml_formats.PageXml import MultiPageXml
        mp = MultiPageXml()
        newDoc = mp.makeMultiPageXmlMemory(list(map(lambda xy:xy[0],lListDocs)))
        print(outputFileName)
        if outputFileName is None:
            outputFileName = os.path.dirname(self.inputFileName) + os.sep + ".."+os.sep +"col" + os.sep + os.path.basename(self.inputFileName)[:-7] + "_du.mpxml"
        print(outputFileName)
        res= newDoc.write(outputFileName, encoding="UTF-8",pretty_print=True,xml_declaration=True)
#         res= newDoc.saveFormatFileEnc(outputFileName, "UTF-8",True)
#         print res
#         print "output: %s" % outputFileName

    def run(self,domDoc):
        """
            conversion
        """
        ODoc =XMLDSDocument()
#         ODoc.lastPage=1
        ODoc.loadFromDom(domDoc)
        lPageXmlDoc=[]
        lPages= ODoc.getPages()
        for page in lPages:
            try:filename = os.path.basename(page.getAttribute('imageFilename'))
            except:filename="fakename"
            pageXmlDoc,pageNode = PageXml.createPageXmlDocument(creatorName='NLE', filename =filename, imgW = convertDot2Pixel(self.dpi,page.getWidth()), imgH = convertDot2Pixel(self.dpi,page.getHeight()))
            self.pageXmlNS = etree.QName(pageXmlDoc.getroot()).namespace
            if self.bRegionOnly:
                self.convertOnlyRegion(page, pageNode)
            else:
                self.convertDSPage(page,pageNode)
            lPageXmlDoc.append((pageXmlDoc,page.getAttribute('imageFilename')))

        return lPageXmlDoc

if __name__ == "__main__":

    docConv = DS2PageXMLConvertor()

    #prepare for the parsing of the command line
    docConv.createCommandLineParser()
    docConv.add_option("-m", "--multi", dest="bMultiPage", action="store_true", default=False, help="store as multipagePageXml", metavar="B")
    docConv.add_option("-r", "--region", dest="bRegionOnly", action="store_true", default=False, help="convert only regions", metavar="B")
    docConv.add_option("--outdir", dest="outdir", action="store", default="", help="output directory")

    #parse the command line
    dParams, args = docConv.parseCommandLine()

    #Now we are back to the normal programmatic mode, we set the componenet parameters
    docConv.setParams(dParams)
    doc = docConv.loadDom()
    lPageXml = docConv.run(doc)
#     print docConv.bMultiPages
    if lPageXml != []:# and docM.getOutputFileName() != "-":
        if docConv.bMultiPages:
            docConv.storeMultiPageXml(lPageXml,docConv.getOutputFileName())
        else:
            docConv.storePageXmlSetofFiles(lPageXml)

<commit_msg>fix 'points' conversion<commit_after># -*- coding: utf-8 -*-
"""

    DS2PageXml.py
    some functionality to handle pageXML files (add regions,...)

    Copyright Xerox(C) 2016 H. Déjean

    Developed  for the EU project READ. The READ project has received funding
    from the European Union’s Horizon 2020 research and innovation programme
    under grant agreement No 674943.

"""
from __future__ import absolute_import
from __future__ import  print_function
from __future__ import unicode_literals

import sys, os
from lxml import etree

try: #to ease the use without proper Python installation
    import TranskribusDU_version
except ImportError:
    sys.path.append( os.path.dirname(os.path.dirname( os.path.abspath(sys.argv[0]) )) )
    import TranskribusDU_version

from common.Component import Component

from xml_formats.PageXml import PageXml
from ObjectModel.xmlDSDocumentClass import XMLDSDocument

from util.unitConversion import convertDot2Pixel

class DS2PageXMLConvertor(Component):
    """
        conversion from DSXML to PageXML
    """
    #DEFINE the version, usage and description of this particular component
    usage = ""
    version = "v.01"
    description = "description: DS2PageXml conversion"

    #--- INIT -------------------------------------------------------------------------------------------------------------
    def __init__(self):
        """
        """
        Component.__init__(self, "DS2PageXml", self.usage, self.version, self.description)

        self.dpi = 300

        self.xrce_id=10000

        self.storagePath = ''

        self.dTagNameMapping = {'PAGE':'Page','TEXT':'TextLine', 'LINE':'TextLine','COLUMN':'TextRegion','REGION':'TextRegion','BLOCK':'TextRegion','GRAPHELT':'LineDrawingRegion','TABLE':'TableRegion','CELL':'TableCell','SeparatorRegion':'SeparatorRegion'}

        self.pageXmlNS = None

        self.bMultiPages = False
        self.bRegionOnly = False

    def setParams(self, dParams):
        """
        Always call first the Component setParams
        """
        Component.setParams(self, dParams)
        if "bMultiPage" in dParams: self.bMultiPages =  dParams["bMultiPage"]
        if "bRegionOnly" in dParams: self.bRegionOnly =  dParams["bRegionOnly"]
        if "outdir" in dParams: self.storagePath =  dParams["outdir"]

    def setDPI(self,v): self.dpi=v

    def setStoragePath(self,p): self.storagePath=p

    def getCoord(self,DSObject):
        """
            create Coords value (polylines) from BoundingBox
            if object has @points: return points
        """

        if DSObject.hasAttribute('points'):
            return DSObject.getAttribute('points')
        else:
            return self.BB2Polylines(DSObject.getX(), DSObject.getY(),DSObject.getHeight(), DSObject.getWidth())

    def BB2Polylines(self,x,y,h,w):
        """
            convert BB values to polylines coords
        """
        # schema does not support neg int
        if x <0: x = abs(x)
        if y <0: y = abs(y)

        lx= list(map(lambda x:1.0*x*self.dpi/72.0, ( x,y, x+w,y, x+w,y+h, x,y+h, x,y)))
        myPoints = ' '.join(["%d,%d"%(xa,ya) for xa,ya  in zip(lx[0::2], lx[1::2])])
        return myPoints

    def DSPoint2PagePoints(self,sPoints):
        """
            "52.8,47.76,128.88,44.64"
            to
            451,246 451,1094 781,1094 781,246

        """
        lPoints = sPoints.split(",")
        lx= list(map(lambda x:1.0*float(x)*self.dpi/72.0, lPoints))
        # order left right
        xx =  list(zip(lx[0::2], lx[1::2]))
#         xx.sort(key=lambda xy:xy[0])
#         print (sPoints, xx,' '.join(["%d,%d"%(xa,ya) for xa,ya  in xx]))
        return ' '.join(["%d,%d"%(xa,ya) for xa,ya  in xx])

    def convertDSObject(self,DSObject,pageXmlParentNode):
        """
            convert DSObject and add it as child to pageXmlParentNode

             <TextLine id="line_1472550984091_215" custom="readingOrder {index:0;}">
                <Coords points="218,65 280,65 280,100 218,100"/>
                <Baseline points="218,95 280,95"/>
                <TextEquiv>
                    <Unicode>10.</Unicode>
                </TextEquiv>
            </TextLine>

        for table:
        <TableRegion id="Table_1484215666379_5" custom="readingOrder {index:92;}">
            <Coords points="221,246 781,246 781,1094 221,1094"/>
            <TableCell row="0" col="0" colSpan="1" id="TableCell_1484215672011_8">
                <Coords points="221,246 221,1094 451,1094 451,246"/>
                <CornerPts>0 1 2 3</CornerPts>
            </TableCell>
            <TableCell row="0" col="1" colSpan="1" id="TableCell_1484215672011_7">
                <Coords points="451,246 451,1094 781,1094 781,246"/>
                <CornerPts>0 1 2 3</CornerPts>
            </TableCell>
        </TableRegion>

        DS TEXT
             <TEXT x="52.8" y="41.04" height="6.72" width="76.08" font-size="20" y2="47.76" x2="128.88"
                points="52.8,41.04,128.88,41.04,128.88,47.76,52.8,47.76" blpoints="52.8,47.76,128.88,44.64" type="RB" id="p1_CVL-{e0004b40-06e0-4b85-97a0-2df0e1f0fe87}"/>
        """
        try:
            pageXmlName= self.dTagNameMapping[DSObject.getName()]
        except KeyError:
            print (DSObject.getName() ," not declared")
            return
#         print (DSObject.getName())
        domNode= PageXml.createPageXmlNode(pageXmlName)
        if DSObject.getID():
            domNode.set("id", "xrce_%s"%DSObject.getID())
        else: self.addXRCEID(domNode)
        pageXmlParentNode.append(domNode)
        coordsNode = etree.Element('{%s}Coords'%(self.pageXmlNS))
#         coordsNode.setNs(self.pageXmlNS)
        if DSObject.hasAttribute('points'):
            coordsNode.set('points',self.DSPoint2PagePoints(DSObject.getAttribute('points')))
        else:
            coordsNode.set('points', self.BB2Polylines(DSObject.getX(),DSObject.getY(), DSObject.getHeight(),DSObject.getWidth()))
        domNode.append(coordsNode)

        for attr in ['custom', 'structure','col','type','DU_row','DU_header','DU_col']:
            if DSObject.hasAttribute(attr):
                domNode.set(attr, DSObject.getAttribute(attr))
        # if blpoints:  build Baseline

        # type
#         if DSObject.hasAttribute('type'):
#             domNode.set('type', DSObject.getAttribute('type'))

        # if blpoints:  build Baseline
        # <Baseline points="218,95 280,95"/>
        ## Baseline needs to be left-right!!
        if DSObject.hasAttribute('blpoints'):
            domBaseLine=etree.Element('{%s}Baseline'%(self.pageXmlNS))
#             domBaseLine.setNs(self.pageXmlNS)
            domBaseLine.set('points', self.DSPoint2PagePoints(DSObject.getAttribute('blpoints')))
            domNode.append(domBaseLine)

        # collect content and generate a textequiv
        #  <TextEquiv> <Unicode>des</Unicode>        </TextEquiv>
        if DSObject.getContent() is not None:
            TextEquivDom =etree.Element('{%s}TextEquiv'%(self.pageXmlNS))
            unicodeDom= etree.Element('{%s}Unicode'%(self.pageXmlNS))
            unicodeDom.text = DSObject.getContent()
            TextEquivDom.append(unicodeDom)
            domNode.append(TextEquivDom)

        ## specific attributes for cell
        ###  row="0" col="2" colSpan="1
        if pageXmlName == 'TableCell':
            domNode.set('row',str(DSObject.getIndex()[0]))
            domNode.set('col',str(DSObject.getIndex()[1]))
            cornerNode = etree.Element('{%s}CornerPts'%(self.pageXmlNS))
            cornerNode.text = "0 1 2 3"
#             cornerNode.setNs(self.pageXmlNS)
            domNode.append(cornerNode)
            coordsNode.set('colSpan',str(DSObject.getColSpan()))
            coordsNode.set('rowSpan',str(DSObject.getRowSpan()))

        #process objects
        for subobject in DSObject.getObjects():
            self.convertDSObject(subobject, domNode)

    def addXRCEID(self,node):
        node.set("id", "xrce_%d"%self.xrce_id)
        self.xrce_id += 1

    def convertOnlyRegion(self,OPage,pageXmlPageNODE):
        """
            populate pageXml with OPageRegion
        """
        # get REGION elements
        lElts= OPage.getAllNamedObjects('REGION')
        for DSObject in lElts:
            self.convertDSObject(DSObject,pageXmlPageNODE)

    def convertDSPage(self,OPage,pageXmlPageNODE):
        """
            populate pageXml with OPage
        """
        from ObjectModel.XMLDSGRAHPLINEClass import XMLDSGRAPHLINEClass

        lElts = OPage.getObjects()
        for DSObject in lElts:
            self.convertDSObject(DSObject,pageXmlPageNODE)

# #         # get graphelt elements
        lElts= OPage.getAllNamedObjects(XMLDSGRAPHLINEClass)
        for DSObject in lElts:
            self.convertDSObject(DSObject,pageXmlPageNODE)

        # get table elements

    def storePageXmlSetofFiles(self,lListOfDocs):
        """
            write on disc the list of dom in the PageXml format
        """
        for i,(doc,img) in enumerate(lListOfDocs):
            if img is None: img='fakeimage.jpg'  # generated
            if self.storagePath == "":
                if os.path.dirname(self.inputFileName) =='':
                    self.outputFileName = img[:-3]+"_%.4d"%(i+1) + ".xml"
                else:
                    self.outputFileName = os.path.dirname(self.inputFileName)+os.sep+img[:-3]+"_%.4d"%(i+1) + ".xml"
            else:
                self.outputFileName = self.storagePath + os.sep+img[:-4]+"_%.4d"%(i+1) + ".xml"
            print("output: %s" % self.outputFileName)
            try:self.writeDom(doc, bIndent=True)
            except IOError as e:
                print(e)
                return -1
        return 0

    def storeMultiPageXml(self,lListDocs,outputFileName=None):
        """
            write a multipagePageXml file
        """
        from xml_formats.PageXml import MultiPageXml
        mp = MultiPageXml()
        newDoc = mp.makeMultiPageXmlMemory(list(map(lambda xy:xy[0],lListDocs)))
        print(outputFileName)
        if outputFileName is None:
            outputFileName = os.path.dirname(self.inputFileName) + os.sep + ".."+os.sep +"col" + os.sep + os.path.basename(self.inputFileName)[:-7] + "_du.mpxml"
        print(outputFileName)
        res= newDoc.write(outputFileName, encoding="UTF-8",pretty_print=True,xml_declaration=True)
#         res= newDoc.saveFormatFileEnc(outputFileName, "UTF-8",True)
#         print res
#         print "output: %s" % outputFileName

    def run(self,domDoc):
        """
            conversion
        """
        ODoc =XMLDSDocument()
#         ODoc.lastPage=1
        ODoc.loadFromDom(domDoc)
        lPageXmlDoc=[]
        lPages= ODoc.getPages()
        for page in lPages:
            try:filename = os.path.basename(page.getAttribute('imageFilename'))
            except:filename="fakename"
            pageXmlDoc,pageNode = PageXml.createPageXmlDocument(creatorName='NLE', filename =filename, imgW = convertDot2Pixel(self.dpi,page.getWidth()), imgH = convertDot2Pixel(self.dpi,page.getHeight()))
            self.pageXmlNS = etree.QName(pageXmlDoc.getroot()).namespace
            if self.bRegionOnly:
                self.convertOnlyRegion(page, pageNode)
            else:
                self.convertDSPage(page,pageNode)
            lPageXmlDoc.append((pageXmlDoc,page.getAttribute('imageFilename')))

        return lPageXmlDoc

if __name__ == "__main__":

    docConv = DS2PageXMLConvertor()

    #prepare for the parsing of the command line
    docConv.createCommandLineParser()
    docConv.add_option("-m", "--multi", dest="bMultiPage", action="store_true", default=False, help="store as multipagePageXml", metavar="B")
    docConv.add_option("-r", "--region", dest="bRegionOnly", action="store_true", default=False, help="convert only regions", metavar="B")
    docConv.add_option("--outdir", dest="outdir", action="store", default="", help="output directory")

    #parse the command line
    dParams, args = docConv.parseCommandLine()

    #Now we are back to the normal programmatic mode, we set the componenet parameters
    docConv.setParams(dParams)
    doc = docConv.loadDom()
    lPageXml = docConv.run(doc)
#     print docConv.bMultiPages
    if lPageXml != []:# and docM.getOutputFileName() != "-":
        if docConv.bMultiPages:
            docConv.storeMultiPageXml(lPageXml,docConv.getOutputFileName())
        else:
            docConv.storePageXmlSetofFiles(lPageXml)

<|endoftext|>