<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# ======================================================================
# Copyright 2016 Julien LE CLEACH
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ======================================================================

import json
import time

from typing import Any, Optional

from supervisor import events
from supervisor.loggers import Logger
from supervisor.options import make_namespec
from supervisor.states import ProcessStates, _process_states_by_code
from supervisor.xmlrpc import RPCError

from .mainloop import SupvisorsMainLoop
from .process import ProcessStatus
from .supvisorszmq import SupervisorZmq, RequestPusher
from .ttypes import ProcessEvent, ProcessAddedEvent, ProcessRemovedEvent
from .utils import InternalEventHeaders, RemoteCommEvents

# get reverted map for ProcessStates
_process_states_by_name = {y: x for x, y in _process_states_by_code.items()}

def add_process_events() -> None:
    """ Register new events in Supervisor EventTypes.
    The new events are in support of Supervisor issue #177.

    :return: None
    """
    events.register('PROCESS', ProcessEvent)  # abstract
    events.register('PROCESS_ADDED', ProcessAddedEvent)
    events.register('PROCESS_REMOVED', ProcessRemovedEvent)

class SupervisorListener(object):
    """ This class subscribes directly to the internal Supervisor events.
    These events are published to all Supvisors instances.

    Attributes are:

        - supvisors: the Supvisors global structure ;
        - logger: the Supvisors logger ;
        - collector: the statistics compiler ;
        - local_identifier: the identifier of the local Supvisors instance ;
        - pusher: the ZeroMQ socket used to publish Supvisors internal events to all Supvisors instances ;
        - main_loop: the Supvisors' event thread.
    """

    def __init__(self, supvisors: Any):
        """ Initialization of the attributes. """
        self.supvisors = supvisors
        self.logger: Logger = supvisors.logger
        # test if statistics collector can be created for local host
        self.collector = None
        try:
            from .statscollector import instant_statistics
            self.collector = instant_statistics
        except ImportError:
            self.logger.info('SupervisorListener: psutil not installed')
            self.logger.warn('SupervisorListener: this Supvisors instance cannot not collect statistics')
        # other attributes
        self.local_identifier: str = supvisors.supvisors_mapper.local_identifier
        self.pusher: Optional[RequestPusher] = None
        # WARN: the SupvisorsMainLoop cannot be created at this level
        # Before running, Supervisor forks when daemonized and the PyZmq sockets are then lost
        self.main_loop: Optional[SupvisorsMainLoop] = None
        # add new events to Supervisor EventTypes
        add_process_events()
        # subscribe to internal events
        events.subscribe(events.SupervisorRunningEvent, self.on_running)
        events.subscribe(events.SupervisorStoppingEvent, self.on_stopping)
        events.subscribe(events.ProcessStateEvent, self.on_process_state)
        events.subscribe(ProcessAddedEvent, self.on_process_added)
        events.subscribe(ProcessRemovedEvent, self.on_process_removed)
        events.subscribe(events.ProcessGroupAddedEvent, self.on_group_added)
        events.subscribe(events.Tick5Event, self.on_tick)
        events.subscribe(events.RemoteCommunicationEvent, self.on_remote_event)

    def on_running(self, _):
        """ Called when Supervisor is RUNNING.
        This method start the Supvisors main loop. """
        self.logger.info('SupervisorListener.on_running: local supervisord is RUNNING')
        try:
            # replace the default handler for web ui
            self.supvisors.supervisor_data.replace_default_handler()
            # update Supervisor internal data for extra_args
            # WARN: this is also triggered by adding groups in Supervisor, however the initial group added events
            # are sent before the Supvisors RPC interface is created
            self.supvisors.supervisor_data.prepare_extra_args()
            # create zmq sockets
            self.supvisors.zmq = SupervisorZmq(self.supvisors)
            # keep a reference to the internal pusher used to defer the events publication
            self.pusher = self.supvisors.zmq.pusher
            # At this point, Supervisor has forked if necessary so the main loop can be started
            self.main_loop = SupvisorsMainLoop(self.supvisors)
            self.logger.debug('SupervisorListener.on_running: request to start main loop')
            self.main_loop.start()
            self.logger.debug('SupervisorListener.on_running: main loop started')
            # Trigger the FSM
            self.supvisors.fsm.next()
        except Exception as exc:
            # Supvisors shall never endanger the Supervisor thread
            self.logger.critical(f'SupervisorListener.on_running: {exc}')

    def on_stopping(self, _):
        """ Called when Supervisor is STOPPING.
        This method stops the Supvisors main loop. """
        self.logger.warn('SupervisorListener.on_stopping: local supervisord is STOPPING')
        try:
            # force Supervisor to close HTTP servers
            # this will prevent any pending XML-RPC request to block the main loop
            self.supvisors.supervisor_data.close_httpservers()
            # stop the main loop
            self.logger.debug('SupervisorListener.on_stopping: request to stop main loop')
            self.main_loop.stop()
            self.logger.debug('SupervisorListener.on_stopping: end of main loop')
            # close zmq sockets
            self.supvisors.zmq.close()
            # unsubscribe from events
            events.clear()
            # finally, close logger
            # WARN: only if it is not the supervisor logger
            if hasattr(self.logger, 'SUPVISORS'):
                self.logger.close()
        except Exception as exc:
            # Supvisors shall never endanger the Supervisor thread
            self.logger.critical(f'SupervisorListener.on_stopping: {exc}')

    def on_process_state(self, event: events.ProcessStateEvent) -> None:
        """ Called when a ProcessStateEvent is sent by the local Supervisor.
        The event is published to all Supvisors instances. """
        try:
            event_name = events.getEventNameByType(event.__class__)
            process_config = event.process.config
            namespec = make_namespec(event.process.group.config.name, process_config.name)
            self.logger.debug(f'SupervisorListener.on_process_state: got {event_name} for {namespec}')
            # create payload from event
            payload = {'name': process_config.name,
                       'group': event.process.group.config.name,
                       'state': _process_states_by_name[event_name.split('_')[-1]],
                       'now': int(time.time()),
                       'pid': event.process.pid,
                       'expected': event.expected,
                       'spawnerr': event.process.spawnerr}
            if hasattr(process_config, 'extra_args'):
                payload['extra_args'] = process_config.extra_args
            self.logger.trace(f'SupervisorListener.on_process_state: payload={payload}')
            self.pusher.send_process_state_event(payload)
        except Exception as exc:
            # Supvisors shall never endanger the Supervisor thread
            self.logger.critical(f'SupervisorListener.on_process_state: {exc}')

    def on_process_added(self, event: ProcessAddedEvent) -> None:
        """ Called when a process has been added due to a numprocs change.

        :param event: the ProcessAddedEvent object
        :return: None
        """
        try:
            namespec = make_namespec(event.process.group.config.name, event.process.config.name)
            self.logger.debug('SupervisorListener.on_process_added: got ProcessAddedEvent for {namespec}')
            # use Supervisor to get local information on all processes
            rpc_intf = self.supvisors.supervisor_data.supvisors_rpc_interface
            try:
                process_info = rpc_intf.get_local_process_info(namespec)
            except RPCError as e:
                self.logger.error(f'SupervisorListener.on_process_added: failed to get process info for {namespec}:'
                                  f' {e.text}')
            else:
                self.logger.trace(f'SupervisorListener.on_process_added: process_info={process_info}')
                self.pusher.send_process_added_event(process_info)
        except Exception as exc:
            # Supvisors shall never endanger the Supervisor thread
            self.logger.critical(f'SupervisorListener.on_process_added: {exc}')

    def on_process_removed(self, event: ProcessRemovedEvent) -> None:
        """ Called when a process has been removed due to a numprocs change.

        :param event: the ProcessRemovedEvent object
        :return: None
        """
        try:
            namespec = make_namespec(event.process.group.config.name, event.process.config.name)
            self.logger.debug(f'SupervisorListener.on_process_removed: got ProcessRemovedEvent for {namespec}')
            payload = {'name': event.process.config.name, 'group': event.process.group.config.name}
            self.logger.trace(f'SupervisorListener.on_process_removed: payload={payload}')
            self.pusher.send_process_removed_event(payload)
        except Exception as exc:
            # Supvisors shall never endanger the Supervisor thread
            self.logger.critical(f'SupervisorListener.on_process_removed: {exc}')

    def on_group_added(self, event: events.ProcessGroupAddedEvent) -> None:
        """ Called when a group has been added due to a Supervisor configuration update.

        :param event: the ProcessGroupAddedEvent object
        :return: None
        """
        try:
            # update Supervisor internal data for extra_args
            self.logger.debug(f'SupervisorListener.on_group_added: group={event.group}')
            self.supvisors.supervisor_data.prepare_extra_args(event.group)
        except Exception as exc:
            # Supvisors shall never endanger the Supervisor thread
            self.logger.critical(f'SupervisorListener.on_group_added: {exc}')

    def on_tick(self, event: events.TickEvent) -> None:
        """ Called when a TickEvent is notified.
        The event is published to all Supvisors instances.
        Then statistics are published and periodic task is triggered.

        :param event: the Supervisor TICK event
        :return: None
        """
        try:
            self.logger.debug('SupervisorListener.on_tick: got TickEvent from Supervisor')
            payload = {'when': event.when}
            self.logger.trace(f'SupervisorListener.on_tick: payload={payload}')
            self.pusher.send_tick_event(payload)
            # get and publish statistics at tick time (optional)
            if self.collector and self.supvisors.options.stats_enabled:
                status = self.supvisors.context.instances[self.local_identifier]
                stats = self.collector(status.pid_processes())
                self.pusher.send_statistics(stats)
        except Exception as exc:
            # Supvisors shall never endanger the Supervisor thread
            self.logger.critical(f'SupervisorListener.on_tick: {exc}')

    def on_remote_event(self, event: events.RemoteCommunicationEvent) -> None:
        """ Called when a RemoteCommunicationEvent is notified.
        This is used to sequence the events received from the Supvisors thread with the other events handled
        by the local Supervisor. """
        try:
            self.logger.trace(f'SupervisorListener.on_remote_event: got RemoteCommunicationEvent {event.type}'
                              f' / {event.data}')
            if event.type == RemoteCommEvents.SUPVISORS_AUTH:
                self.authorization(event.data)
            elif event.type == RemoteCommEvents.SUPVISORS_EVENT:
                self.unstack_event(event.data)
            elif event.type == RemoteCommEvents.SUPVISORS_INFO:
                self.unstack_info(event.data)
        except Exception as exc:
            # Supvisors shall never endanger the Supervisor thread
            self.logger.critical(f'SupervisorListener.on_remote_event: {exc}')

    def unstack_event(self, message: str):
        """ Unstack and process one event from the event queue. """
        event_type, (event_identifier, event_data) = json.loads(message)
        if event_type == InternalEventHeaders.TICK.value:
            self.logger.trace(f'SupervisorListener.unstack_event: got TICK from {event_identifier}: {event_data}')
            self.supvisors.fsm.on_tick_event(event_identifier, event_data)
        elif event_type == InternalEventHeaders.PROCESS.value:
            self.logger.trace(f'SupervisorListener.unstack_event: got PROCESS from {event_identifier}: {event_data}')
            self.supvisors.fsm.on_process_state_event(event_identifier, event_data)
        elif event_type == InternalEventHeaders.PROCESS_ADDED.value:
            self.logger.trace(f'SupervisorListener.unstack_event: got PROCESS_ADDED from {event_identifier}:'
                              f' {event_data}')
            self.supvisors.fsm.on_process_added_event(event_identifier, event_data)
        elif event_type == InternalEventHeaders.PROCESS_REMOVED.value:
            self.logger.trace(f'SupervisorListener.unstack_event: got PROCESS_REMOVED from {event_identifier}:'
                              f' {event_data}')
            self.supvisors.fsm.on_process_removed_event(event_identifier, event_data)
        elif event_type == InternalEventHeaders.STATISTICS.value:
            # this Supvisors could handle statistics even if psutil is not installed
            # unless the function is explicitly disabled
            self.logger.info(f'SupervisorListener.unstack_event: got STATISTICS from {event_identifier}: {event_data}')
            if self.supvisors.options.stats_enabled:
                self.supvisors.statistician.push_statistics(event_identifier, event_data)
        elif event_type == InternalEventHeaders.STATE.value:
            self.logger.trace(f'SupervisorListener.unstack_event: got STATE from {event_identifier}')
            self.supvisors.fsm.on_state_event(event_identifier, event_data)

    def unstack_info(self, message: str) -> None:
        """ Unstack the process info received. """
        # unstack the queue for process info
        identifier, info = json.loads(message)
        self.logger.trace(f'SupervisorListener.unstack_info: got process info event from {identifier}')
        self.supvisors.fsm.on_process_info(identifier, info)

    def authorization(self, message: str) -> None:
        """ Extract authorization and identifier from data and process event. """
        identifier, authorized, master_identifier = json.loads(message)
        self.logger.trace(f'SupervisorListener.authorization: got authorization event from {identifier}')
        self.supvisors.fsm.on_authorization(identifier, authorized, master_identifier)

    def force_process_state(self, process: ProcessStatus, identifier: str, event_date: int,
                            forced_state: ProcessStates, reason: str) -> None:
        """ Publish the process state requested to all Supvisors instances.

        :param process: the process structure
        :param identifier: the identifier of the Supvisors instance where the process state is expected
        :param event_date: the date of the last process event received, used for the evaluation of the error
        :param forced_state: the process state to force if the expected state has not been received
        :param reason: the reason declared
        :return: None
        """
        # create payload from event
        payload = {'group': process.application_name, 'name': process.process_name, 'state': forced_state,
                   'forced': True, 'identifier': identifier,
                   'now': event_date, 'pid': 0, 'expected': False, 'spawnerr': reason,
                   'extra_args': process.extra_args}
        self.logger.debug(f'SupervisorListener.force_process_state: payload={payload}')
        self.pusher.send_process_state_event(payload)
<commit_msg>fix DEPLOYMENT stuck with late process events<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# ======================================================================
# Copyright 2016 Julien LE CLEACH
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ======================================================================

import json
import time

from typing import Any, Optional

from supervisor import events
from supervisor.loggers import Logger
from supervisor.options import make_namespec
from supervisor.states import ProcessStates, _process_states_by_code
from supervisor.xmlrpc import RPCError

from .mainloop import SupvisorsMainLoop
from .process import ProcessStatus
from .supvisorszmq import SupervisorZmq, RequestPusher
from .ttypes import ProcessEvent, ProcessAddedEvent, ProcessRemovedEvent
from .utils import InternalEventHeaders, RemoteCommEvents

# get reverted map for ProcessStates
_process_states_by_name = {y: x for x, y in _process_states_by_code.items()}

def add_process_events() -> None:
    """ Register new events in Supervisor EventTypes.
    The new events are in support of Supervisor issue #177.

    :return: None
    """
    events.register('PROCESS', ProcessEvent)  # abstract
    events.register('PROCESS_ADDED', ProcessAddedEvent)
    events.register('PROCESS_REMOVED', ProcessRemovedEvent)

class SupervisorListener(object):
    """ This class subscribes directly to the internal Supervisor events.
    These events are published to all Supvisors instances.

    Attributes are:

        - supvisors: the Supvisors global structure ;
        - logger: the Supvisors logger ;
        - collector: the statistics compiler ;
        - local_identifier: the identifier of the local Supvisors instance ;
        - pusher: the ZeroMQ socket used to publish Supvisors internal events to all Supvisors instances ;
        - main_loop: the Supvisors' event thread.
    """

    def __init__(self, supvisors: Any):
        """ Initialization of the attributes. """
        self.supvisors = supvisors
        self.logger: Logger = supvisors.logger
        # test if statistics collector can be created for local host
        self.collector = None
        try:
            from .statscollector import instant_statistics
            self.collector = instant_statistics
        except ImportError:
            self.logger.info('SupervisorListener: psutil not installed')
            self.logger.warn('SupervisorListener: this Supvisors instance cannot not collect statistics')
        # other attributes
        self.local_identifier: str = supvisors.supvisors_mapper.local_identifier
        self.pusher: Optional[RequestPusher] = None
        # WARN: the SupvisorsMainLoop cannot be created at this level
        # Before running, Supervisor forks when daemonized and the PyZmq sockets are then lost
        self.main_loop: Optional[SupvisorsMainLoop] = None
        # add new events to Supervisor EventTypes
        add_process_events()
        # subscribe to internal events
        events.subscribe(events.SupervisorRunningEvent, self.on_running)
        events.subscribe(events.SupervisorStoppingEvent, self.on_stopping)
        events.subscribe(events.ProcessStateEvent, self.on_process_state)
        events.subscribe(ProcessAddedEvent, self.on_process_added)
        events.subscribe(ProcessRemovedEvent, self.on_process_removed)
        events.subscribe(events.ProcessGroupAddedEvent, self.on_group_added)
        events.subscribe(events.Tick5Event, self.on_tick)
        events.subscribe(events.RemoteCommunicationEvent, self.on_remote_event)

    def on_running(self, _):
        """ Called when Supervisor is RUNNING.
        This method start the Supvisors main loop. """
        self.logger.info('SupervisorListener.on_running: local supervisord is RUNNING')
        try:
            # replace the default handler for web ui
            self.supvisors.supervisor_data.replace_default_handler()
            # update Supervisor internal data for extra_args
            # WARN: this is also triggered by adding groups in Supervisor, however the initial group added events
            # are sent before the Supvisors RPC interface is created
            self.supvisors.supervisor_data.prepare_extra_args()
            # create zmq sockets
            self.supvisors.zmq = SupervisorZmq(self.supvisors)
            # keep a reference to the internal pusher used to defer the events publication
            self.pusher = self.supvisors.zmq.pusher
            # At this point, Supervisor has forked if necessary so the main loop can be started
            self.main_loop = SupvisorsMainLoop(self.supvisors)
            self.logger.debug('SupervisorListener.on_running: request to start main loop')
            self.main_loop.start()
            self.logger.debug('SupervisorListener.on_running: main loop started')
            # Trigger the FSM
            self.supvisors.fsm.next()
        except Exception as exc:
            # Supvisors shall never endanger the Supervisor thread
            self.logger.critical(f'SupervisorListener.on_running: {exc}')

    def on_stopping(self, _):
        """ Called when Supervisor is STOPPING.
        This method stops the Supvisors main loop. """
        self.logger.warn('SupervisorListener.on_stopping: local supervisord is STOPPING')
        try:
            # force Supervisor to close HTTP servers
            # this will prevent any pending XML-RPC request to block the main loop
            self.supvisors.supervisor_data.close_httpservers()
            # stop the main loop
            self.logger.debug('SupervisorListener.on_stopping: request to stop main loop')
            self.main_loop.stop()
            self.logger.debug('SupervisorListener.on_stopping: end of main loop')
            # close zmq sockets
            self.supvisors.zmq.close()
            # unsubscribe from events
            events.clear()
            # finally, close logger
            # WARN: only if it is not the supervisor logger
            if hasattr(self.logger, 'SUPVISORS'):
                self.logger.close()
        except Exception as exc:
            # Supvisors shall never endanger the Supervisor thread
            self.logger.critical(f'SupervisorListener.on_stopping: {exc}')

    def on_process_state(self, event: events.ProcessStateEvent) -> None:
        """ Called when a ProcessStateEvent is sent by the local Supervisor.
        The event is published to all Supvisors instances. """
        try:
            event_name = events.getEventNameByType(event.__class__)
            process_config = event.process.config
            namespec = make_namespec(event.process.group.config.name, process_config.name)
            self.logger.debug(f'SupervisorListener.on_process_state: got {event_name} for {namespec}')
            # create payload from event
            payload = {'name': process_config.name,
                       'group': event.process.group.config.name,
                       'state': _process_states_by_name[event_name.split('_')[-1]],
                       'now': int(time.time()),
                       'pid': event.process.pid,
                       'expected': event.expected,
                       'spawnerr': event.process.spawnerr}
            if hasattr(process_config, 'extra_args'):
                payload['extra_args'] = process_config.extra_args
            self.logger.trace(f'SupervisorListener.on_process_state: payload={payload}')
            self.pusher.send_process_state_event(payload)
        except Exception as exc:
            # Supvisors shall never endanger the Supervisor thread
            self.logger.critical(f'SupervisorListener.on_process_state: {exc}')

    def on_process_added(self, event: ProcessAddedEvent) -> None:
        """ Called when a process has been added due to a numprocs change.

        :param event: the ProcessAddedEvent object
        :return: None
        """
        try:
            namespec = make_namespec(event.process.group.config.name, event.process.config.name)
            self.logger.debug('SupervisorListener.on_process_added: got ProcessAddedEvent for {namespec}')
            # use Supervisor to get local information on all processes
            rpc_intf = self.supvisors.supervisor_data.supvisors_rpc_interface
            try:
                process_info = rpc_intf.get_local_process_info(namespec)
            except RPCError as e:
                self.logger.error(f'SupervisorListener.on_process_added: failed to get process info for {namespec}:'
                                  f' {e.text}')
            else:
                self.logger.trace(f'SupervisorListener.on_process_added: process_info={process_info}')
                self.pusher.send_process_added_event(process_info)
        except Exception as exc:
            # Supvisors shall never endanger the Supervisor thread
            self.logger.critical(f'SupervisorListener.on_process_added: {exc}')

    def on_process_removed(self, event: ProcessRemovedEvent) -> None:
        """ Called when a process has been removed due to a numprocs change.

        :param event: the ProcessRemovedEvent object
        :return: None
        """
        try:
            namespec = make_namespec(event.process.group.config.name, event.process.config.name)
            self.logger.debug(f'SupervisorListener.on_process_removed: got ProcessRemovedEvent for {namespec}')
            payload = {'name': event.process.config.name, 'group': event.process.group.config.name}
            self.logger.trace(f'SupervisorListener.on_process_removed: payload={payload}')
            self.pusher.send_process_removed_event(payload)
        except Exception as exc:
            # Supvisors shall never endanger the Supervisor thread
            self.logger.critical(f'SupervisorListener.on_process_removed: {exc}')

    def on_group_added(self, event: events.ProcessGroupAddedEvent) -> None:
        """ Called when a group has been added due to a Supervisor configuration update.

        :param event: the ProcessGroupAddedEvent object
        :return: None
        """
        try:
            # update Supervisor internal data for extra_args
            self.logger.debug(f'SupervisorListener.on_group_added: group={event.group}')
            self.supvisors.supervisor_data.prepare_extra_args(event.group)
        except Exception as exc:
            # Supvisors shall never endanger the Supervisor thread
            self.logger.critical(f'SupervisorListener.on_group_added: {exc}')

    def on_tick(self, event: events.TickEvent) -> None:
        """ Called when a TickEvent is notified.
        The event is published to all Supvisors instances.
        Then statistics are published and periodic task is triggered.

        :param event: the Supervisor TICK event
        :return: None
        """
        try:
            self.logger.debug('SupervisorListener.on_tick: got TickEvent from Supervisor')
            payload = {'when': event.when}
            self.logger.trace(f'SupervisorListener.on_tick: payload={payload}')
            self.pusher.send_tick_event(payload)
            # get and publish statistics at tick time (optional)
            if self.collector and self.supvisors.options.stats_enabled:
                status = self.supvisors.context.instances[self.local_identifier]
                stats = self.collector(status.pid_processes())
                self.pusher.send_statistics(stats)
        except Exception as exc:
            # Supvisors shall never endanger the Supervisor thread
            self.logger.critical(f'SupervisorListener.on_tick: {exc}')

    def on_remote_event(self, event: events.RemoteCommunicationEvent) -> None:
        """ Called when a RemoteCommunicationEvent is notified.
        This is used to sequence the events received from the Supvisors thread with the other events handled
        by the local Supervisor. """
        try:
            self.logger.trace(f'SupervisorListener.on_remote_event: got RemoteCommunicationEvent {event.type}'
                              f' / {event.data}')
            if event.type == RemoteCommEvents.SUPVISORS_AUTH:
                self.authorization(event.data)
            elif event.type == RemoteCommEvents.SUPVISORS_EVENT:
                self.unstack_event(event.data)
            elif event.type == RemoteCommEvents.SUPVISORS_INFO:
                self.unstack_info(event.data)
        except Exception as exc:
            # Supvisors shall never endanger the Supervisor thread
            self.logger.critical(f'SupervisorListener.on_remote_event: {exc}')

    def unstack_event(self, message: str):
        """ Unstack and process one event from the event queue. """
        event_type, (event_identifier, event_data) = json.loads(message)
        if event_type == InternalEventHeaders.TICK.value:
            self.logger.trace(f'SupervisorListener.unstack_event: got TICK from {event_identifier}: {event_data}')
            self.supvisors.fsm.on_tick_event(event_identifier, event_data)
        elif event_type == InternalEventHeaders.PROCESS.value:
            self.logger.trace(f'SupervisorListener.unstack_event: got PROCESS from {event_identifier}: {event_data}')
            self.supvisors.fsm.on_process_state_event(event_identifier, event_data)
        elif event_type == InternalEventHeaders.PROCESS_ADDED.value:
            self.logger.trace(f'SupervisorListener.unstack_event: got PROCESS_ADDED from {event_identifier}:'
                              f' {event_data}')
            self.supvisors.fsm.on_process_added_event(event_identifier, event_data)
        elif event_type == InternalEventHeaders.PROCESS_REMOVED.value:
            self.logger.trace(f'SupervisorListener.unstack_event: got PROCESS_REMOVED from {event_identifier}:'
                              f' {event_data}')
            self.supvisors.fsm.on_process_removed_event(event_identifier, event_data)
        elif event_type == InternalEventHeaders.STATISTICS.value:
            # this Supvisors could handle statistics even if psutil is not installed
            # unless the function is explicitly disabled
            self.logger.trace(f'SupervisorListener.unstack_event: got STATISTICS from {event_identifier}: {event_data}')
            if self.supvisors.options.stats_enabled:
                self.supvisors.statistician.push_statistics(event_identifier, event_data)
        elif event_type == InternalEventHeaders.STATE.value:
            self.logger.trace(f'SupervisorListener.unstack_event: got STATE from {event_identifier}')
            self.supvisors.fsm.on_state_event(event_identifier, event_data)

    def unstack_info(self, message: str) -> None:
        """ Unstack the process info received. """
        # unstack the queue for process info
        identifier, info = json.loads(message)
        self.logger.trace(f'SupervisorListener.unstack_info: got process info event from {identifier}')
        self.supvisors.fsm.on_process_info(identifier, info)

    def authorization(self, message: str) -> None:
        """ Extract authorization and identifier from data and process event. """
        identifier, authorized, master_identifier = json.loads(message)
        self.logger.trace(f'SupervisorListener.authorization: got authorization event from {identifier}')
        self.supvisors.fsm.on_authorization(identifier, authorized, master_identifier)

    def force_process_state(self, process: ProcessStatus, identifier: str, event_date: int,
                            forced_state: ProcessStates, reason: str) -> None:
        """ Publish the process state requested to all Supvisors instances.

        :param process: the process structure
        :param identifier: the identifier of the Supvisors instance where the process state is expected
        :param event_date: the date of the last process event received, used for the evaluation of the error
        :param forced_state: the process state to force if the expected state has not been received
        :param reason: the reason declared
        :return: None
        """
        # create payload from event
        payload = {'group': process.application_name, 'name': process.process_name, 'state': forced_state,
                   'forced': True, 'identifier': identifier,
                   'now': event_date, 'pid': 0, 'expected': False, 'spawnerr': reason,
                   'extra_args': process.extra_args}
        self.logger.debug(f'SupervisorListener.force_process_state: payload={payload}')
        self.pusher.send_process_state_event(payload)
<|endoftext|>


<commit_before># This file is part of Supysonic.
# Supysonic is a Python implementation of the Subsonic server API.
#
# Copyright (C) 2013-2018 Alban 'spl0k' Féron
#                    2017 Óscar García Amor
#
# Distributed under terms of the GNU AGPLv3 license.

NAME = "supysonic"
VERSION = "0.6.1"
DESCRIPTION = "Python implementation of the Subsonic server API."
KEYWORDS = "subsonic music api"
AUTHOR_NAME = "Alban Féron"
AUTHOR_EMAIL = "alban.feron@gmail.com"
URL = "https://github.com/spl0k/supysonic"
LICENSE = "GNU AGPLv3"
LONG_DESCRIPTION = """Supysonic is a Python implementation of the Subsonic server API.
Current supported features are:
* browsing (by folders or tags)
* streaming of various audio file formats
* transcoding
* user or random playlists
* cover arts (cover.jpg files in the same folder as music files)
* starred tracks/albums and ratings
* Last.FM scrobbling"""
<commit_msg>Version bump<commit_after># This file is part of Supysonic.
# Supysonic is a Python implementation of the Subsonic server API.
#
# Copyright (C) 2013-2018 Alban 'spl0k' Féron
#                    2017 Óscar García Amor
#
# Distributed under terms of the GNU AGPLv3 license.

NAME = "supysonic"
VERSION = "0.6.2"
DESCRIPTION = "Python implementation of the Subsonic server API."
KEYWORDS = "subsonic music api"
AUTHOR_NAME = "Alban Féron"
AUTHOR_EMAIL = "alban.feron@gmail.com"
URL = "https://github.com/spl0k/supysonic"
LICENSE = "GNU AGPLv3"
LONG_DESCRIPTION = """Supysonic is a Python implementation of the Subsonic server API.
Current supported features are:
* browsing (by folders or tags)
* streaming of various audio file formats
* transcoding
* user or random playlists
* cover arts (cover.jpg files in the same folder as music files)
* starred tracks/albums and ratings
* Last.FM scrobbling"""
<|endoftext|>


<commit_before>
import pytz
import flask
import babel.dates as dates

from apps.auth import get_user

bp = flask.Blueprint('locales', __name__)

@bp.route('/locales/timezones')
def locales_view():
    if not flask.current_app.auth.authorized([], 'locales', 'GET'):
        flask.abort(401)
    user = get_user()
    lang = user.get('language', flask.current_app.config.get('DEFAULT_LANGUAGE', 'en')).replace('-', '_')
    return flask.jsonify({
        'timezones': [
            {
                'id': tz,
                'name': dates.get_timezone_name(tz, locale=lang),
                'location': dates.get_timezone_location(tz, locale=lang),
            } for tz in pytz.common_timezones
        ]
    })

def init_app(app):
    bp.url_prefix = '/{}'.format(app.config['URL_PREFIX'].lstrip('/'))
    app.register_blueprint(bp)
<commit_msg>fix locales resource cors (#1973)<commit_after>
import pytz
import flask
import babel.dates as dates

from apps.auth import get_user
from eve.render import send_response

bp = flask.Blueprint('locales', __name__)

def get_timezones():
    user = get_user()
    lang = user.get('language', flask.current_app.config.get('DEFAULT_LANGUAGE', 'en')).replace('-', '_')
    return [
        {
            'id': tz,
            'name': dates.get_timezone_name(tz, locale=lang),
            'location': dates.get_timezone_location(tz, locale=lang),
        } for tz in pytz.common_timezones
    ]

@bp.route('/locales/timezones', methods=['GET', 'OPTIONS'])
def locales_view():
    if not flask.current_app.auth.authorized([], 'locales', 'GET'):
        flask.abort(401)
    resp = None
    if flask.request.method == 'GET':
        resp = {'timezones': get_timezones()}
    return send_response(None, (resp, None, None, 200))

def init_app(app):
    bp.url_prefix = '/{}'.format(app.config['URL_PREFIX'].lstrip('/'))
    app.register_blueprint(bp)
<|endoftext|>


<commit_before>import numpy
import os

def save_raw(filename, data):
    with open(filename, 'w') as fout:
        fout.write(" $SEQPAR\n")
        fout.write(" ECHOT = {}\n".format(data.te))
        fout.write(" HZPPPM = {}\n".format(data.f0))
        fout.write(" SEQ = 'PRESS'\n")
        fout.write(" $END\n")
        fout.write(" $NMID\n")
        fout.write(" FMTDAT = '(2E15.6)'\n")
        # convert the volume from mm^3 to cc
        fout.write(" VOLUME = {}\n".format(data.voxel_size() * 1e-3))
        fout.write(" $END\n")
        for point in numpy.nditer(data, order='C'):
            fout.write("  {0: 4.6e}  {1: 4.6e}\n".format(float(point.real), float(point.imag)))

def write_all_files(filename, data, wref_data=None, params=None):
    """
    Creates an LCModel control file for processing the supplied MRSData, and
    optional water reference data, updating the default parameters with any
    values supplied through params.

    :param filename: the location where the control file should be saved.
    :param data: MRSData to be processed.
    :param wref_data: Optional MRSData containing water reference.
    :param params: Optional dictionary containing non-default parameter values.
    :return:
    """

    # we assume that the data has one spectral dimension, any others must be
    # spatial
    if len(data.shape) == 1:
        shape = (1, 1, 1)
    elif len(data.shape) == 2:
        shape = (data.shape[0], 1, 1)
    elif len(data.shape) == 3:
        shape = (data.shape[0], data.shape[1], 1)
    elif len(data.shape) == 4:
        shape = data.shape[0:3]
    elif len(data.shape) > 4:
        raise ValueError("LCModel cannot handle data with more than 4 dimensions")

    # We need to save a bunch of files for LCModel to process: a raw file for
    # the data, possibly a raw file for the wref and a control file for each
    # slice. In addition, in the absence of information in the params file
    # about where to save the output (.ps, .csv, .table etc.) that should also
    # be saved in the same folder as the input data for LCModel.
    folder, file_root = os.path.split(filename)
    file_root, ext = os.path.splitext(file_root)

    base_params = {
        "FILBAS": "/home/spectre/.lcmodel/basis-sets/provencher/press_te30_3t_gsh_v3.basis",
        "ICOLST": 1,
        "ICOLEN": shape[0],
        "NDCOLS": shape[0],
        "IROWST": 1,
        "IROWEN": shape[1],
        "NDROWS": shape[1],
        "NDSLIC": shape[2],
        "DOWS": "T" if wref_data is not None else "F",
        "DOECC": "T" if wref_data is not None else "F",
        "FILRAW": os.path.join(folder, file_root + ".RAW"),
        "FILPS": os.path.join(folder, file_root + ".PS")
    }
    if wref_data is not None:
        base_params["FILH2O"] = os.path.join(folder, file_root + ".H2O")

    # add the user supplied parameters to the list
    if params is not None:
        base_params.update(params)

    # make a few modifications based on user edits
    if "FILTAB" in base_params:
        base_params["LTABLE"] = 7
        base_params["FILTAB"] = "'{}'".format(base_params["FILTAB"])
    elif "LTABLE" in base_params:
        base_params["LTABLE"] = 7
        base_params["FILTAB"] = "'{}'".format(os.path.join(folder, file_root + ".TABLE"))
    if "FILCSV" in base_params:
        base_params["LCSV"] = 11
        base_params["FILCSV"] = "'{}'".format(base_params["FILCSV"])
    elif "LCSV" in base_params:
        base_params["LCSV"] = 11
        base_params["FILCSV"] = "'{}'".format(os.path.join(folder, file_root + ".CSV"))
    if "FILCOO" in base_params:
        base_params["LCOORD"] = 9
        base_params["FILCOO"] = "'{}'".format(base_params["FILCOO"])
    elif "LCOORD" in base_params:
        base_params["LCOORD"] = 9
        base_params["FILCOO"] = "'{}'".format(os.path.join(folder, file_root + ".COORD"))

    save_raw(base_params["FILRAW"], data)
    if wref_data is not None:
        save_raw(base_params["FILH2O"], wref_data)
    # have to add single quotes to the various paths
    base_params["FILRAW"] = "'{}'".format(base_params["FILRAW"])
    base_params["FILBAS"] = "'{}'".format(base_params["FILBAS"])
    base_params["FILPS"] = "'{}'".format(base_params["FILPS"])
    if wref_data is not None:
        base_params["FILH2O"] = "'{}'".format(base_params["FILH2O"])

    for slice_index in range(shape[2]):
        control_filename = "{0}_sl{1}.CONTROL".format(file_root, slice_index)
        control_filepath = os.path.join(folder, control_filename)
        with open(control_filepath, 'wt') as fout:
            fout.write(" $LCMODL\n")
            fout.write(" OWNER = ''\n")
            fout.write(" KEY = 123456789\n")
            fout.write(" DELTAT = {}\n".format(data.dt))
            fout.write(" HZPPPM = {}\n".format(data.f0))
            fout.write(" NUNFIL = {}\n".format(data.np))
            for key, value in base_params.items():
                fout.write(" {0} = {1}\n".format(key, value))
            fout.write(" $END\n")

def read_coord(filename):
    with open(filename, 'rt') as fin:
        coord_lines = fin.readlines()

    metabolite_table_info_line = 4
    metabolite_table_line_count = int(coord_lines[metabolite_table_info_line].split()[0])
    metabolite_table_lines = coord_lines[(metabolite_table_info_line + 2):(metabolite_table_info_line + 1 + metabolite_table_line_count)]
    metabolite_fits = {}
    for metabolite_line in metabolite_table_lines:
        conc = float(metabolite_line[:9])
        sd = int(metabolite_line[9:13])
        cr_ratio = float(metabolite_line[14:22])
        name = metabolite_line[22:].strip()
        metabolite_fits[name] = {
            "concentration": conc,
            "sd": sd,
        }
    # add some aliases for some of the composite metabolites
    metabolite_fits["TNAA"] = metabolite_fits["NAA+NAAG"]
    metabolite_fits["TCho"] = metabolite_fits["GPC+PCh"]
    metabolite_fits["TCr"] = metabolite_fits["Cr+PCr"]
    metabolite_fits["Glx"] = metabolite_fits["Glu+Gln"]

    misc_output_info_line = metabolite_table_info_line + metabolite_table_line_count + 1
    misc_output_line_count = int(coord_lines[misc_output_info_line].split()[0])
    misc_output_lines = coord_lines[(misc_output_info_line + 1):(misc_output_info_line + misc_output_line_count + 1)]
    misc_output = {}
    misc_output["fwhm"] = float(misc_output_lines[0].split()[2])
    misc_output["snr"] = float(misc_output_lines[0].split()[6])
    misc_output["frequency_shift"] = float(misc_output_lines[1].split()[3])
    misc_output["phase0"] = float(misc_output_lines[2].split()[1])
    misc_output["phase1"] = float(misc_output_lines[2].split()[3])

    return {
        "metabolite_fits": metabolite_fits,
        "misc_output": misc_output
    }
<commit_msg>extended lcmodel read_coord function<commit_after>import numpy
import os

def save_raw(filename, data):
    with open(filename, 'w') as fout:
        fout.write(" $SEQPAR\n")
        fout.write(" ECHOT = {}\n".format(data.te))
        fout.write(" HZPPPM = {}\n".format(data.f0))
        fout.write(" SEQ = 'PRESS'\n")
        fout.write(" $END\n")
        fout.write(" $NMID\n")
        fout.write(" FMTDAT = '(2E15.6)'\n")
        # convert the volume from mm^3 to cc
        fout.write(" VOLUME = {}\n".format(data.voxel_size() * 1e-3))
        fout.write(" $END\n")
        for point in numpy.nditer(data, order='C'):
            fout.write("  {0: 4.6e}  {1: 4.6e}\n".format(float(point.real), float(point.imag)))

def write_all_files(filename, data, wref_data=None, params=None):
    """
    Creates an LCModel control file for processing the supplied MRSData, and
    optional water reference data, updating the default parameters with any
    values supplied through params.

    :param filename: the location where the control file should be saved.
    :param data: MRSData to be processed.
    :param wref_data: Optional MRSData containing water reference.
    :param params: Optional dictionary containing non-default parameter values.
    :return:
    """

    # we assume that the data has one spectral dimension, any others must be
    # spatial
    if len(data.shape) == 1:
        shape = (1, 1, 1)
    elif len(data.shape) == 2:
        shape = (data.shape[0], 1, 1)
    elif len(data.shape) == 3:
        shape = (data.shape[0], data.shape[1], 1)
    elif len(data.shape) == 4:
        shape = data.shape[0:3]
    elif len(data.shape) > 4:
        raise ValueError("LCModel cannot handle data with more than 4 dimensions")

    # We need to save a bunch of files for LCModel to process: a raw file for
    # the data, possibly a raw file for the wref and a control file for each
    # slice. In addition, in the absence of information in the params file
    # about where to save the output (.ps, .csv, .table etc.) that should also
    # be saved in the same folder as the input data for LCModel.
    folder, file_root = os.path.split(filename)
    file_root, ext = os.path.splitext(file_root)

    base_params = {
        "FILBAS": "/home/spectre/.lcmodel/basis-sets/provencher/press_te30_3t_gsh_v3.basis",
        "ICOLST": 1,
        "ICOLEN": shape[0],
        "NDCOLS": shape[0],
        "IROWST": 1,
        "IROWEN": shape[1],
        "NDROWS": shape[1],
        "NDSLIC": shape[2],
        "DOWS": "T" if wref_data is not None else "F",
        "DOECC": "T" if wref_data is not None else "F",
        "FILRAW": os.path.join(folder, file_root + ".RAW"),
        "FILPS": os.path.join(folder, file_root + ".PS")
    }
    if wref_data is not None:
        base_params["FILH2O"] = os.path.join(folder, file_root + ".H2O")

    # add the user supplied parameters to the list
    if params is not None:
        base_params.update(params)

    # make a few modifications based on user edits
    if "FILTAB" in base_params:
        base_params["LTABLE"] = 7
        base_params["FILTAB"] = "'{}'".format(base_params["FILTAB"])
    elif "LTABLE" in base_params:
        base_params["LTABLE"] = 7
        base_params["FILTAB"] = "'{}'".format(os.path.join(folder, file_root + ".TABLE"))
    if "FILCSV" in base_params:
        base_params["LCSV"] = 11
        base_params["FILCSV"] = "'{}'".format(base_params["FILCSV"])
    elif "LCSV" in base_params:
        base_params["LCSV"] = 11
        base_params["FILCSV"] = "'{}'".format(os.path.join(folder, file_root + ".CSV"))
    if "FILCOO" in base_params:
        base_params["LCOORD"] = 9
        base_params["FILCOO"] = "'{}'".format(base_params["FILCOO"])
    elif "LCOORD" in base_params:
        base_params["LCOORD"] = 9
        base_params["FILCOO"] = "'{}'".format(os.path.join(folder, file_root + ".COORD"))

    save_raw(base_params["FILRAW"], data)
    if wref_data is not None:
        save_raw(base_params["FILH2O"], wref_data)
    # have to add single quotes to the various paths
    base_params["FILRAW"] = "'{}'".format(base_params["FILRAW"])
    base_params["FILBAS"] = "'{}'".format(base_params["FILBAS"])
    base_params["FILPS"] = "'{}'".format(base_params["FILPS"])
    if wref_data is not None:
        base_params["FILH2O"] = "'{}'".format(base_params["FILH2O"])

    for slice_index in range(shape[2]):
        control_filename = "{0}_sl{1}.CONTROL".format(file_root, slice_index)
        control_filepath = os.path.join(folder, control_filename)
        with open(control_filepath, 'wt') as fout:
            fout.write(" $LCMODL\n")
            fout.write(" OWNER = ''\n")
            fout.write(" KEY = 123456789\n")
            fout.write(" DELTAT = {}\n".format(data.dt))
            fout.write(" HZPPPM = {}\n".format(data.f0))
            fout.write(" NUNFIL = {}\n".format(data.np))
            for key, value in base_params.items():
                fout.write(" {0} = {1}\n".format(key, value))
            fout.write(" $END\n")

def read_coord(filename):
    with open(filename, 'rt') as fin:
        coord_lines = fin.readlines()

    # find start of metabolite table
    for index, line in enumerate(coord_lines):
        if "lines in following concentration table" in line:
            break
    metabolite_table_info_line = index
    metabolite_table_line_count = int(coord_lines[metabolite_table_info_line].split()[0])
    metabolite_table_lines = coord_lines[(metabolite_table_info_line + 2):(metabolite_table_info_line + 1 + metabolite_table_line_count)]
    metabolite_fits = {}
    for metabolite_line in metabolite_table_lines:
        conc = float(metabolite_line[:9])
        sd = int(metabolite_line[9:13])
        cr_ratio = float(metabolite_line[14:22])
        name = metabolite_line[22:].strip()
        metabolite_fits[name] = {
            "concentration": conc,
            "sd": sd,
        }
    # add some aliases for some of the composite metabolites
    metabolite_fits["TNAA"] = metabolite_fits["NAA+NAAG"]
    metabolite_fits["TCho"] = metabolite_fits["GPC+PCh"]
    metabolite_fits["TCr"] = metabolite_fits["Cr+PCr"]
    metabolite_fits["Glx"] = metabolite_fits["Glu+Gln"]

    misc_output_info_line = metabolite_table_info_line + metabolite_table_line_count + 1
    misc_output_line_count = int(coord_lines[misc_output_info_line].split()[0])
    misc_output_lines = coord_lines[(misc_output_info_line + 1):(misc_output_info_line + misc_output_line_count + 1)]
    misc_output = {}
    misc_output["fwhm"] = float(misc_output_lines[0].split()[2])
    misc_output["snr"] = float(misc_output_lines[0].split()[6])
    misc_output["frequency_shift"] = float(misc_output_lines[1].split()[3])
    misc_output["phase0"] = float(misc_output_lines[2].split()[1])
    misc_output["phase1"] = float(misc_output_lines[2].split()[3])

    # get the ppm axis
    ppm_axis_info_line = misc_output_info_line + misc_output_line_count + 1
    num_ppm_points = int(coord_lines[ppm_axis_info_line].split()[0])
    # there are always 10 points of data per line
    number_of_point_lines = int((num_ppm_points + 9) / 10)

    def read_points(starting_line):
        points = []
        # we can use map to convert the 10 string numbers on each line to a
        # list of floats, then extend the current list with the new one
        for line_number in range(starting_line, starting_line + number_of_point_lines):
            points.extend(map(float, coord_lines[line_number].split()))
        return points

    ppm_points = read_points(ppm_axis_info_line + 1)

    # after ppm axis is the phased data
    data_info_line = ppm_axis_info_line + number_of_point_lines + 1
    data_points = read_points(data_info_line + 1)

    # after data is the fit
    fit_info_line = data_info_line + number_of_point_lines + 1
    fit_points = read_points(fit_info_line + 1)

    # after the fit is the baseline
    baseline_info_line = fit_info_line + number_of_point_lines + 1
    baseline_points = read_points(baseline_info_line + 1)

    # now come 0 or more metabolite spectra, followed by the diagnostic table
    metabolite_spectra = {}
    metabolite_info_line = baseline_info_line + number_of_point_lines + 1
    while "diagnostic table" not in coord_lines[metabolite_info_line]:
        metabolite_name = coord_lines[metabolite_info_line].split()[0]
        metabolite_points = read_points(metabolite_info_line + 1)
        metabolite_spectra[metabolite_name] = metabolite_points

        # move to the next block of data
        metabolite_info_line = metabolite_info_line + number_of_point_lines + 1

    return {
        "metabolite_fits": metabolite_fits,
        "misc_output": misc_output,
        "ppm": ppm_points,
        "fit": fit_points,
        "data": data_points,
        "baseline": baseline_points,
        "metabolite_spectra": metabolite_spectra
    }
<|endoftext|>


<commit_before><commit_msg>play around version done and pythonic.<commit_after>import requests
import collections

MovieResult = collections.namedtuple(
    'MovieResult',
    "imdb_code,title,duration,director,year,rating,imdb_score,keywords,genres")

search = input("What movie do you want to search for? ")
url = 'http://movie_service.talkpython.fm/api/search/{}'.format(search)

resp = requests.get(url)
resp.raise_for_status()

movie_data = resp.json()
movies_list = movie_data.get('hits')

# movies = []
# for md in movies_list:
#     m = MovieResult(
#         imdb_code=md.get('imdb_code'),
#         title=md.get('title'),
#         duration=md.get('duration'),
#         director=md.get('director'),
#         year=md.get('year', 0),
#         rating=md.get('rating', 0),
#         imdb_score=md.get('imdb_score', 0.0),
#         keywords=md.get('keywords'),
#         genres=md.get('genres')
#     )
#     movies.append(m)

# def method(x, y, z, **kwargs):
#     print("kwargs=", kwargs)
#
# method(7, 1, z=2, format=True, age=7)

# movies = []
# for md in movies_list:
#     m = MovieResult(**md)
#     movies.append(m)

movies = [
    MovieResult(**md)
    for md in movies_list
]

print("Found {} movies for search {}".format(len(movies), search))
for m in movies:
    print("{} -- {}".format(m.year, m.title))
<|endoftext|>


<commit_before>#! /usr/bin/env python
import subprocess
import sys
import argparse
from datetime import datetime

try:
    import xml.etree.cElementTree as ElementTree
except ImportError:
    import xml.etree.ElementTree as ElementTree

def main(argv):
    parser = argparse.ArgumentParser(description='Generates a ChangeLog from an SVN Repository.')
    parser.add_argument('url', help='URL of the SVN Repository')
    parser.add_argument('output', help='Location of the Output file')
    args = parser.parse_args()

    query_svn(args.url, args.output)

    sys.exit()

def query_svn(repoUrl, outputFile):
    print ('Connecting to SVN server...')

    try:
        s = subprocess.Popen(['svn', 'log', '--xml', repoUrl],
                              stdout=subprocess.PIPE)
    except:
        print ('Error querying SVN Log')
        sys.exit(1)

    print ('Querying SVN server...')

    out = s.communicate()[0]
    s.stdout.close()

    if s.returncode != 0:
        print ('some error' + str(s.returncode))

    try:
        root = ElementTree.fromstring(out)
    except:
        print ('No results from Repository')
        sys.exit(1)

    f = open(outputFile, 'w+')
    f.write('Revision'.ljust(10))
    f.write('Date'.ljust(10))
    f.write('Author'.ljust(10))
    f.write('Message'.ljust(10))
    f.write('\n')

    for elem in root:
        msgs = elem.find('msg').text

        if msgs.startswith('INTERNAL: '):
            continue

        msgs = msgs.encode('utf-8')
        rev = str(elem.attrib.get('revision'))
        author = elem.find('author').text
        date = elem.find('date').text
        date = datetime.strptime(date, '%Y-%m-%dT%H:%M:%S.%fZ')
        date = datetime.strftime(date, '%d-%m-%y')

        f.write(rev.ljust(10))
        f.write(date.ljust(10))
        f.write(author.ljust(10))
        f.write(msgs)
        f.write('\n')

    f.close()

if __name__ == '__main__':
    main(sys.argv[1:])
<commit_msg>Ensure msgs contains a valid string<commit_after>#! /usr/bin/env python
import subprocess
import sys
import argparse
from datetime import datetime

try:
    import xml.etree.cElementTree as ElementTree
except ImportError:
    import xml.etree.ElementTree as ElementTree

def main(argv):
    parser = argparse.ArgumentParser(description='Generates a ChangeLog from an SVN Repository.')
    parser.add_argument('url', help='URL of the SVN Repository')
    parser.add_argument('output', help='Location of the Output file')
    args = parser.parse_args()

    query_svn(args.url, args.output)

    sys.exit()

def query_svn(repoUrl, outputFile):
    print ('Connecting to SVN server...')

    try:
        s = subprocess.Popen(['svn', 'log', '--xml', repoUrl],
                              stdout=subprocess.PIPE)
    except:
        print ('Error querying SVN Log')
        sys.exit(1)

    print ('Querying SVN server...')

    out = s.communicate()[0]
    s.stdout.close()

    if s.returncode != 0:
        print ('some error' + str(s.returncode))

    try:
        root = ElementTree.fromstring(out)
    except:
        print ('No results from Repository')
        sys.exit(1)

    f = open(outputFile, 'w+')
    f.write('Revision'.ljust(10))
    f.write('Date'.ljust(10))
    f.write('Author'.ljust(10))
    f.write('Message'.ljust(10))
    f.write('\n')

    for elem in root:
        msgs = elem.find('msg').text

        if not msgs:
            msgs = ''
        elif msgs.startswith('INTERNAL: '):
            continue

        msgs = msgs.encode('utf-8')
        rev = str(elem.attrib.get('revision'))
        author = elem.find('author').text
        date = elem.find('date').text
        date = datetime.strptime(date, '%Y-%m-%dT%H:%M:%S.%fZ')
        date = datetime.strftime(date, '%d-%m-%y')

        f.write(rev.ljust(10))
        f.write(date.ljust(10))
        f.write(author.ljust(10))
        f.write(msgs)
        f.write('\n')

    f.close()

if __name__ == '__main__':
    main(sys.argv[1:])
<|endoftext|>


<commit_before>__version__ = "1.0b2.dev1"
<commit_msg>dev2 release<commit_after>__version__ = "1.0b2.dev2"
<|endoftext|>


<commit_before>#!/usr/bin/env python2.7
# -*- mode: python; encoding: utf-8; -*-
from __future__ import division, absolute_import, unicode_literals, print_function
from future_builtins import *

import logging
import subprocess

from intensional import Re

log = logging.getLogger('apt-config-tool')

def parse_apt_config(dump):
    for line in dump.splitlines():
        if line not in Re(r'^([^ ]+) "(.*)";$'):
            raise AssertionError('Horrible parsing code has horribly failed!')
        yield (Re._[1], Re._[2])

def get_apt_config():
    return dict(parse_apt_config(subprocess.check_output(('apt-config', 'dump'))))

def get_apt_proxy():
    config = get_apt_config()

    proxy = config.get('Acquire::http::Proxy')
    if proxy:
        log.debug('Using host\'s apt proxy: {}'.format(proxy))
        return proxy

    # You can install squid-deb-proxy-client, which supplies a tiny little script that asks avahi
    # for any zeroconf-advertised apt proxies.
    proxy_autodetect = config.get('Acquire::http::ProxyAutoDetect')
    if proxy_autodetect:
        log.debug('Using host\'s apt proxy autodetection script: {}'.format(proxy_autodetect))
        proxy = subprocess.check_output((proxy_autodetect,)).strip()
    if proxy:
        log.debug('Autodetected apt proxy: {}'.format(proxy))
        return proxy

    log.debug('Couldn\'t find an apt proxy.')
    return None
<commit_msg>Make tool run on non-Debian systems<commit_after>#!/usr/bin/env python2.7
# -*- mode: python; encoding: utf-8; -*-
from __future__ import division, absolute_import, unicode_literals, print_function
from future_builtins import *

import logging
import subprocess

from intensional import Re

log = logging.getLogger('apt-config-tool')

def parse_apt_config(dump):
    for line in dump.splitlines():
        if line not in Re(r'^([^ ]+) "(.*)";$'):
            raise AssertionError('Horrible parsing code has horribly failed!')
        yield (Re._[1], Re._[2])

def get_apt_config():
    try:
        return dict(parse_apt_config(subprocess.check_output(('apt-config', 'dump'))))
    except OSError as exc:
        if exc.errno == 2:
            return {}
        raise

def get_apt_proxy():
    config = get_apt_config()

    proxy = config.get('Acquire::http::Proxy')
    if proxy:
        log.debug('Using host\'s apt proxy: {}'.format(proxy))
        return proxy

    # You can install squid-deb-proxy-client, which supplies a tiny little script that asks avahi
    # for any zeroconf-advertised apt proxies.
    proxy_autodetect = config.get('Acquire::http::ProxyAutoDetect')
    if proxy_autodetect:
        log.debug('Using host\'s apt proxy autodetection script: {}'.format(proxy_autodetect))
        proxy = subprocess.check_output((proxy_autodetect,)).strip()
    if proxy:
        log.debug('Autodetected apt proxy: {}'.format(proxy))
        return proxy

    log.debug('Couldn\'t find an apt proxy.')
    return None
<|endoftext|>


<commit_before><commit_msg>proposal set can be linked only to a closed debate<commit_after><|endoftext|>


<commit_before>#
# Copyright (C) 2012 - 2017 Satoru SATOH <ssato @ redhat.com>
# License: MIT
#
# pylint: disable=unused-argument
r"""Abstract implementation of backend modules:

Backend module must implement a parser class inherits :class:`Parser` or its
children classes of this module and override all or some of the methods as
needed:

  - :meth:`load_from_string`: Load config from string
  - :meth:`load_from_stream`: Load config from a file or file-like object
  - :meth:`load_from_path`: Load config from file of given path
  - :meth:`dump_to_string`: Dump config as a string
  - :meth:`dump_to_stream`: Dump config to a file or file-like object
  - :meth:`dump_to_path`: Dump config to a file of given path

Changelog:

.. versionchanged:: 0.8.3

   - Add `_ordered` membmer and a class method :meth:` ordered to the class
     :class:`Parser`.

.. versionchanged:: 0.2

   - The methods :meth:`load_impl`, :meth:`dump_impl` are deprecated and
     replaced with :meth:`load_from_stream` and :meth:`load_from_path`,
     :meth:`dump_to_string` and :meth:`dump_to_path` respectively.
"""
from __future__ import absolute_import

import functools
import logging
import os

import anyconfig.compat
import anyconfig.mdicts
import anyconfig.utils

LOGGER = logging.getLogger(__name__)

def mk_opt_args(keys, kwargs):
    """
    Make optional kwargs valid and optimized for each backend.

    :param keys: optional argument names
    :param kwargs: keyword arguements to process

    >>> mk_opt_args(("aaa", ), dict(aaa=1, bbb=2))
    {'aaa': 1}
    >>> mk_opt_args(("aaa", ), dict(bbb=2))
    {}
    """
    return dict((k, kwargs[k]) for k in keys if k in kwargs)

def ensure_outdir_exists(filepath):
    """
    Make dir to dump `filepath` if that dir does not exist.

    :param filepath: path of file to dump
    """
    outdir = os.path.dirname(filepath)

    if outdir and not os.path.exists(outdir):
        LOGGER.debug("Making output dir: %s", outdir)
        os.makedirs(outdir)

def to_method(func):
    """
    Lift :func:`func` to a method; it will be called with the first argument
    `self` ignored.

    :param func: Any callable object
    """
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        """Wrapper function.
        """
        return func(*args[1:], **kwargs)

    return wrapper

class Parser(object):
    """
    Abstract parser to provide basic implementation of some methods, interfaces
    and members.
    """
    _type = None
    _priority = 0   # 0 (lowest priority) .. 99  (highest priority)
    _extensions = []
    _load_opts = []
    _dump_opts = []
    _open_flags = ('r', 'w')
    _ordered = False

    @classmethod
    def type(cls):
        """
        Parser's type
        """
        return cls._type

    @classmethod
    def priority(cls):
        """
        Parser's priority
        """
        return cls._priority

    @classmethod
    def extensions(cls):
        """
        File extensions which this parser can process
        """
        return cls._extensions

    @classmethod
    def ordered(cls):
        """
        :return: True if parser can keep the order of keys else False.
        """
        return cls._ordered

    @classmethod
    def ropen(cls, filepath, **kwargs):
        """
        :param filepath: Path to file to open to read data
        """
        return open(filepath, cls._open_flags[0], **kwargs)

    @classmethod
    def wopen(cls, filepath, **kwargs):
        """
        :param filepath: Path to file to open to write data to
        """
        return open(filepath, cls._open_flags[1], **kwargs)

    def _load_options(self, **kwargs):
        """
        Select backend specific loading options from `kwargs` only.
        """
        return mk_opt_args(self._load_opts, kwargs)

    def _container_fn(self, **options):
        """
        :param options: Keyword options may contain 'ac_ordered'.
        :return: Factory (class or function) to make an container.
        """
        if options.get("ac_dict", False):
            return options["ac_dict"]  # Higher priority than ac_ordered.
        elif self.ordered() and options.get("ac_ordered", False):
            return anyconfig.compat.OrderedDict
        else:
            return dict

    def load_from_string(self, content, to_container, **kwargs):
        """
        Load config from given string `content`.

        :param content: Config content string
        :param to_container: callble to make a container object later
        :param kwargs: optional keyword parameters to be sanitized :: dict

        :return: Dict-like object holding config parameters
        """
        return to_container()

    def load_from_path(self, filepath, to_container, **kwargs):
        """
        Load config from given file path `filepath`.

        :param filepath: Config file path
        :param to_container: callble to make a container object later
        :param kwargs: optional keyword parameters to be sanitized :: dict

        :return: Dict-like object holding config parameters
        """
        return to_container()

    def load_from_stream(self, stream, to_container, **kwargs):
        """
        Load config from given file like object `stream`.

        :param stream:  Config file or file like object
        :param to_container: callble to make a container object later
        :param kwargs: optional keyword parameters to be sanitized :: dict

        :return: Dict-like object holding config parameters
        """
        return to_container()

    def loads(self, content, **options):
        """
        Load config from given string `content` after some checks.

        :param content:  Config file content
        :param options:
            options will be passed to backend specific loading functions.
            please note that options have to be sanitized w/ mk_opt_args later
            to filter out options not  in _load_opts.

        :return: dict or dict-like object holding configurations
        """
        to_container = self._container_fn(**options)
        if not content or content is None:
            return to_container()

        return self.load_from_string(content, to_container,
                                     **self._load_options(**options))

    def load(self, path_or_stream, ignore_missing=False, **options):
        """
        Load config from a file path or a file / file-like object
        `path_or_stream` after some checks.

        :param path_or_stream: Config file path or file{,-like} object
        :param ignore_missing:
            Ignore and just return None if given `path_or_stream` is not a file
            / file-like object (thus, it should be a file path) and does not
            exist in actual.
        :param options:
            options will be passed to backend specific loading functions.
            please note that options have to be sanitized w/ mk_opt_args later
            to filter out options not  in _load_opts.

        :return: dict or dict-like object holding configurations
        """
        to_container = self._container_fn(**options)
        options = self._load_options(**options)

        if isinstance(path_or_stream, anyconfig.compat.STR_TYPES):
            if ignore_missing and not os.path.exists(path_or_stream):
                return to_container()

            cnf = self.load_from_path(path_or_stream, to_container, **options)
        else:
            cnf = self.load_from_stream(path_or_stream, to_container,
                                        **options)

        return cnf

    def dump_to_string(self, cnf, **kwargs):
        """
        Dump config `cnf` to a string.

        :param cnf: Configuration data to dump
        :param kwargs: optional keyword parameters to be sanitized :: dict

        :return: string represents the configuration
        """
        pass

    def dump_to_path(self, cnf, filepath, **kwargs):
        """
        Dump config `cnf` to a file `filepath`.

        :param cnf: Configuration data to dump
        :param filepath: Config file path
        :param kwargs: optional keyword parameters to be sanitized :: dict
        """
        pass

    def dump_to_stream(self, cnf, stream, **kwargs):
        """
        Dump config `cnf` to a file-like object `stream`.

        TODO: How to process socket objects same as file objects ?

        :param cnf: Configuration data to dump
        :param stream:  Config file or file like object
        :param kwargs: optional keyword parameters to be sanitized :: dict
        """
        pass

    def dumps(self, cnf, **kwargs):
        """
        Dump config `cnf` to a string.

        :param cnf: Configuration data to dump
        :param kwargs: optional keyword parameters to be sanitized :: dict

        :return: string represents the configuration
        """
        cnf = anyconfig.mdicts.convert_to(cnf, **kwargs)
        kwargs = mk_opt_args(self._dump_opts, kwargs)
        return self.dump_to_string(cnf, **kwargs)

    def dump(self, cnf, path_or_stream, **kwargs):
        """
        Dump config `cnf` to a filepath or file-like object
        `path_or_stream`.

        :param cnf: Configuration data to dump
        :param path_or_stream: Config file path or file{,-like} object
        :param kwargs: optional keyword parameters to be sanitized :: dict
        :raises IOError, OSError, AttributeError: When dump failed.
        """
        cnf = anyconfig.mdicts.convert_to(cnf, **kwargs)
        kwargs = mk_opt_args(self._dump_opts, kwargs)

        if isinstance(path_or_stream, anyconfig.compat.STR_TYPES):
            ensure_outdir_exists(path_or_stream)
            self.dump_to_path(cnf, path_or_stream, **kwargs)
        else:
            self.dump_to_stream(cnf, path_or_stream, **kwargs)

class FromStringLoader(Parser):
    """
    Abstract config parser provides a method to load configuration from string
    content to help implement parser of which backend lacks of such function.

    Parser classes inherit this class have to override the method
    :meth:`load_from_string` at least.
    """
    def load_from_stream(self, stream, to_container, **kwargs):
        """
        Load config from given stream `stream`.

        :param stream: Config file or file-like object
        :param to_container: callble to make a container object later
        :param kwargs: optional keyword parameters to be sanitized :: dict

        :return: Dict-like object holding config parameters
        """
        return self.load_from_string(stream.read(), to_container, **kwargs)

    def load_from_path(self, filepath, to_container, **kwargs):
        """
        Load config from given file path `filepath`.

        :param filepath: Config file path
        :param to_container: callble to make a container object later
        :param kwargs: optional keyword parameters to be sanitized :: dict

        :return: Dict-like object holding config parameters
        """
        return self.load_from_stream(self.ropen(filepath), to_container,
                                     **kwargs)

class FromStreamLoader(Parser):
    """
    Abstract config parser provides a method to load configuration from string
    content to help implement parser of which backend lacks of such function.

    Parser classes inherit this class have to override the method
    :meth:`load_from_stream` at least.
    """
    def load_from_string(self, content, to_container, **kwargs):
        """
        Load config from given string `cnf_content`.

        :param content: Config content string
        :param to_container: callble to make a container object later
        :param kwargs: optional keyword parameters to be sanitized :: dict

        :return: Dict-like object holding config parameters
        """
        return self.load_from_stream(anyconfig.compat.StringIO(content),
                                     to_container, **kwargs)

    def load_from_path(self, filepath, to_container, **kwargs):
        """
        Load config from given file path `filepath`.

        :param filepath: Config file path
        :param to_container: callble to make a container object later
        :param kwargs: optional keyword parameters to be sanitized :: dict

        :return: Dict-like object holding config parameters
        """
        return self.load_from_stream(self.ropen(filepath), to_container,
                                     **kwargs)

class ToStringDumper(Parser):
    """
    Abstract config parser provides a method to dump configuration to a file or
    file-like object (stream) and a file of given path to help implement parser
    of which backend lacks of such functions.

    Parser classes inherit this class have to override the method
    :meth:`dump_to_string` at least.
    """
    def dump_to_path(self, cnf, filepath, **kwargs):
        """
        Dump config `cnf` to a file `filepath`.

        :param cnf: Configuration data to dump
        :param filepath: Config file path
        :param kwargs: optional keyword parameters to be sanitized :: dict
        """
        with self.wopen(filepath) as out:
            out.write(self.dump_to_string(cnf, **kwargs))

    def dump_to_stream(self, cnf, stream, **kwargs):
        """
        Dump config `cnf` to a file-like object `stream`.

        TODO: How to process socket objects same as file objects ?

        :param cnf: Configuration data to dump
        :param stream:  Config file or file like object
        :param kwargs: optional keyword parameters to be sanitized :: dict
        """
        stream.write(self.dump_to_string(cnf, **kwargs))

class ToStreamDumper(Parser):
    """
    Abstract config parser provides methods to dump configuration to a string
    content or a file of given path to help implement parser of which backend
    lacks of such functions.

    Parser classes inherit this class have to override the method
    :meth:`dump_to_stream` at least.
    """
    to_stream = to_method(anyconfig.compat.StringIO)

    def dump_to_string(self, cnf, **kwargs):
        """
        Dump config `cnf` to a string.

        :param cnf: Configuration data to dump
        :param kwargs: optional keyword parameters to be sanitized :: dict

        :return: Dict-like object holding config parameters
        """
        stream = self.to_stream()
        self.dump_to_stream(cnf, stream, **kwargs)
        return stream.getvalue()

    def dump_to_path(self, cnf, filepath, **kwargs):
        """
        Dump config `cnf` to a file `filepath`.

        :param cnf: Configuration data to dump
        :param filepath: Config file path
        :param kwargs: optional keyword parameters to be sanitized :: dict
        """
        with self.wopen(filepath) as out:
            self.dump_to_stream(cnf, out, **kwargs)

# vim:sw=4:ts=4:et:
<commit_msg>change: only use the value of ac_dict if it's callable<commit_after>#
# Copyright (C) 2012 - 2017 Satoru SATOH <ssato @ redhat.com>
# License: MIT
#
# pylint: disable=unused-argument
r"""Abstract implementation of backend modules:

Backend module must implement a parser class inherits :class:`Parser` or its
children classes of this module and override all or some of the methods as
needed:

  - :meth:`load_from_string`: Load config from string
  - :meth:`load_from_stream`: Load config from a file or file-like object
  - :meth:`load_from_path`: Load config from file of given path
  - :meth:`dump_to_string`: Dump config as a string
  - :meth:`dump_to_stream`: Dump config to a file or file-like object
  - :meth:`dump_to_path`: Dump config to a file of given path

Changelog:

.. versionchanged:: 0.8.3

   - Add `_ordered` membmer and a class method :meth:` ordered to the class
     :class:`Parser`.

.. versionchanged:: 0.2

   - The methods :meth:`load_impl`, :meth:`dump_impl` are deprecated and
     replaced with :meth:`load_from_stream` and :meth:`load_from_path`,
     :meth:`dump_to_string` and :meth:`dump_to_path` respectively.
"""
from __future__ import absolute_import

import functools
import logging
import os

import anyconfig.compat
import anyconfig.mdicts
import anyconfig.utils

LOGGER = logging.getLogger(__name__)

def mk_opt_args(keys, kwargs):
    """
    Make optional kwargs valid and optimized for each backend.

    :param keys: optional argument names
    :param kwargs: keyword arguements to process

    >>> mk_opt_args(("aaa", ), dict(aaa=1, bbb=2))
    {'aaa': 1}
    >>> mk_opt_args(("aaa", ), dict(bbb=2))
    {}
    """
    return dict((k, kwargs[k]) for k in keys if k in kwargs)

def ensure_outdir_exists(filepath):
    """
    Make dir to dump `filepath` if that dir does not exist.

    :param filepath: path of file to dump
    """
    outdir = os.path.dirname(filepath)

    if outdir and not os.path.exists(outdir):
        LOGGER.debug("Making output dir: %s", outdir)
        os.makedirs(outdir)

def to_method(func):
    """
    Lift :func:`func` to a method; it will be called with the first argument
    `self` ignored.

    :param func: Any callable object
    """
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        """Wrapper function.
        """
        return func(*args[1:], **kwargs)

    return wrapper

class Parser(object):
    """
    Abstract parser to provide basic implementation of some methods, interfaces
    and members.
    """
    _type = None
    _priority = 0   # 0 (lowest priority) .. 99  (highest priority)
    _extensions = []
    _load_opts = []
    _dump_opts = []
    _open_flags = ('r', 'w')
    _ordered = False

    @classmethod
    def type(cls):
        """
        Parser's type
        """
        return cls._type

    @classmethod
    def priority(cls):
        """
        Parser's priority
        """
        return cls._priority

    @classmethod
    def extensions(cls):
        """
        File extensions which this parser can process
        """
        return cls._extensions

    @classmethod
    def ordered(cls):
        """
        :return: True if parser can keep the order of keys else False.
        """
        return cls._ordered

    @classmethod
    def ropen(cls, filepath, **kwargs):
        """
        :param filepath: Path to file to open to read data
        """
        return open(filepath, cls._open_flags[0], **kwargs)

    @classmethod
    def wopen(cls, filepath, **kwargs):
        """
        :param filepath: Path to file to open to write data to
        """
        return open(filepath, cls._open_flags[1], **kwargs)

    def _load_options(self, **kwargs):
        """
        Select backend specific loading options from `kwargs` only.
        """
        return mk_opt_args(self._load_opts, kwargs)

    def _container_fn(self, **options):
        """
        :param options: Keyword options may contain 'ac_ordered'.
        :return: Factory (class or function) to make an container.
        """
        ac_dict = options.get("ac_dict", False)
        if ac_dict and callable(ac_dict):
            return ac_dict  # Higher priority than ac_ordered.
        elif self.ordered() and options.get("ac_ordered", False):
            return anyconfig.compat.OrderedDict
        else:
            return dict

    def load_from_string(self, content, to_container, **kwargs):
        """
        Load config from given string `content`.

        :param content: Config content string
        :param to_container: callble to make a container object later
        :param kwargs: optional keyword parameters to be sanitized :: dict

        :return: Dict-like object holding config parameters
        """
        return to_container()

    def load_from_path(self, filepath, to_container, **kwargs):
        """
        Load config from given file path `filepath`.

        :param filepath: Config file path
        :param to_container: callble to make a container object later
        :param kwargs: optional keyword parameters to be sanitized :: dict

        :return: Dict-like object holding config parameters
        """
        return to_container()

    def load_from_stream(self, stream, to_container, **kwargs):
        """
        Load config from given file like object `stream`.

        :param stream:  Config file or file like object
        :param to_container: callble to make a container object later
        :param kwargs: optional keyword parameters to be sanitized :: dict

        :return: Dict-like object holding config parameters
        """
        return to_container()

    def loads(self, content, **options):
        """
        Load config from given string `content` after some checks.

        :param content:  Config file content
        :param options:
            options will be passed to backend specific loading functions.
            please note that options have to be sanitized w/ mk_opt_args later
            to filter out options not  in _load_opts.

        :return: dict or dict-like object holding configurations
        """
        to_container = self._container_fn(**options)
        if not content or content is None:
            return to_container()

        return self.load_from_string(content, to_container,
                                     **self._load_options(**options))

    def load(self, path_or_stream, ignore_missing=False, **options):
        """
        Load config from a file path or a file / file-like object
        `path_or_stream` after some checks.

        :param path_or_stream: Config file path or file{,-like} object
        :param ignore_missing:
            Ignore and just return None if given `path_or_stream` is not a file
            / file-like object (thus, it should be a file path) and does not
            exist in actual.
        :param options:
            options will be passed to backend specific loading functions.
            please note that options have to be sanitized w/ mk_opt_args later
            to filter out options not  in _load_opts.

        :return: dict or dict-like object holding configurations
        """
        to_container = self._container_fn(**options)
        options = self._load_options(**options)

        if isinstance(path_or_stream, anyconfig.compat.STR_TYPES):
            if ignore_missing and not os.path.exists(path_or_stream):
                return to_container()

            cnf = self.load_from_path(path_or_stream, to_container, **options)
        else:
            cnf = self.load_from_stream(path_or_stream, to_container,
                                        **options)

        return cnf

    def dump_to_string(self, cnf, **kwargs):
        """
        Dump config `cnf` to a string.

        :param cnf: Configuration data to dump
        :param kwargs: optional keyword parameters to be sanitized :: dict

        :return: string represents the configuration
        """
        pass

    def dump_to_path(self, cnf, filepath, **kwargs):
        """
        Dump config `cnf` to a file `filepath`.

        :param cnf: Configuration data to dump
        :param filepath: Config file path
        :param kwargs: optional keyword parameters to be sanitized :: dict
        """
        pass

    def dump_to_stream(self, cnf, stream, **kwargs):
        """
        Dump config `cnf` to a file-like object `stream`.

        TODO: How to process socket objects same as file objects ?

        :param cnf: Configuration data to dump
        :param stream:  Config file or file like object
        :param kwargs: optional keyword parameters to be sanitized :: dict
        """
        pass

    def dumps(self, cnf, **kwargs):
        """
        Dump config `cnf` to a string.

        :param cnf: Configuration data to dump
        :param kwargs: optional keyword parameters to be sanitized :: dict

        :return: string represents the configuration
        """
        cnf = anyconfig.mdicts.convert_to(cnf, **kwargs)
        kwargs = mk_opt_args(self._dump_opts, kwargs)
        return self.dump_to_string(cnf, **kwargs)

    def dump(self, cnf, path_or_stream, **kwargs):
        """
        Dump config `cnf` to a filepath or file-like object
        `path_or_stream`.

        :param cnf: Configuration data to dump
        :param path_or_stream: Config file path or file{,-like} object
        :param kwargs: optional keyword parameters to be sanitized :: dict
        :raises IOError, OSError, AttributeError: When dump failed.
        """
        cnf = anyconfig.mdicts.convert_to(cnf, **kwargs)
        kwargs = mk_opt_args(self._dump_opts, kwargs)

        if isinstance(path_or_stream, anyconfig.compat.STR_TYPES):
            ensure_outdir_exists(path_or_stream)
            self.dump_to_path(cnf, path_or_stream, **kwargs)
        else:
            self.dump_to_stream(cnf, path_or_stream, **kwargs)

class FromStringLoader(Parser):
    """
    Abstract config parser provides a method to load configuration from string
    content to help implement parser of which backend lacks of such function.

    Parser classes inherit this class have to override the method
    :meth:`load_from_string` at least.
    """
    def load_from_stream(self, stream, to_container, **kwargs):
        """
        Load config from given stream `stream`.

        :param stream: Config file or file-like object
        :param to_container: callble to make a container object later
        :param kwargs: optional keyword parameters to be sanitized :: dict

        :return: Dict-like object holding config parameters
        """
        return self.load_from_string(stream.read(), to_container, **kwargs)

    def load_from_path(self, filepath, to_container, **kwargs):
        """
        Load config from given file path `filepath`.

        :param filepath: Config file path
        :param to_container: callble to make a container object later
        :param kwargs: optional keyword parameters to be sanitized :: dict

        :return: Dict-like object holding config parameters
        """
        return self.load_from_stream(self.ropen(filepath), to_container,
                                     **kwargs)

class FromStreamLoader(Parser):
    """
    Abstract config parser provides a method to load configuration from string
    content to help implement parser of which backend lacks of such function.

    Parser classes inherit this class have to override the method
    :meth:`load_from_stream` at least.
    """
    def load_from_string(self, content, to_container, **kwargs):
        """
        Load config from given string `cnf_content`.

        :param content: Config content string
        :param to_container: callble to make a container object later
        :param kwargs: optional keyword parameters to be sanitized :: dict

        :return: Dict-like object holding config parameters
        """
        return self.load_from_stream(anyconfig.compat.StringIO(content),
                                     to_container, **kwargs)

    def load_from_path(self, filepath, to_container, **kwargs):
        """
        Load config from given file path `filepath`.

        :param filepath: Config file path
        :param to_container: callble to make a container object later
        :param kwargs: optional keyword parameters to be sanitized :: dict

        :return: Dict-like object holding config parameters
        """
        return self.load_from_stream(self.ropen(filepath), to_container,
                                     **kwargs)

class ToStringDumper(Parser):
    """
    Abstract config parser provides a method to dump configuration to a file or
    file-like object (stream) and a file of given path to help implement parser
    of which backend lacks of such functions.

    Parser classes inherit this class have to override the method
    :meth:`dump_to_string` at least.
    """
    def dump_to_path(self, cnf, filepath, **kwargs):
        """
        Dump config `cnf` to a file `filepath`.

        :param cnf: Configuration data to dump
        :param filepath: Config file path
        :param kwargs: optional keyword parameters to be sanitized :: dict
        """
        with self.wopen(filepath) as out:
            out.write(self.dump_to_string(cnf, **kwargs))

    def dump_to_stream(self, cnf, stream, **kwargs):
        """
        Dump config `cnf` to a file-like object `stream`.

        TODO: How to process socket objects same as file objects ?

        :param cnf: Configuration data to dump
        :param stream:  Config file or file like object
        :param kwargs: optional keyword parameters to be sanitized :: dict
        """
        stream.write(self.dump_to_string(cnf, **kwargs))

class ToStreamDumper(Parser):
    """
    Abstract config parser provides methods to dump configuration to a string
    content or a file of given path to help implement parser of which backend
    lacks of such functions.

    Parser classes inherit this class have to override the method
    :meth:`dump_to_stream` at least.
    """
    to_stream = to_method(anyconfig.compat.StringIO)

    def dump_to_string(self, cnf, **kwargs):
        """
        Dump config `cnf` to a string.

        :param cnf: Configuration data to dump
        :param kwargs: optional keyword parameters to be sanitized :: dict

        :return: Dict-like object holding config parameters
        """
        stream = self.to_stream()
        self.dump_to_stream(cnf, stream, **kwargs)
        return stream.getvalue()

    def dump_to_path(self, cnf, filepath, **kwargs):
        """
        Dump config `cnf` to a file `filepath`.

        :param cnf: Configuration data to dump
        :param filepath: Config file path
        :param kwargs: optional keyword parameters to be sanitized :: dict
        """
        with self.wopen(filepath) as out:
            self.dump_to_stream(cnf, out, **kwargs)

# vim:sw=4:ts=4:et:
<|endoftext|>