<commit_before># -*- coding: utf-8 -*-
# -----------------------------------------------------------------------------
# Copyright 2015 by Ecpy Authors, see AUTHORS for more details.
#
# Distributed under the terms of the BSD license.
#
# The full license is in the file LICENCE, distributed with this software.
# -----------------------------------------------------------------------------
"""Instrument manager plugin.

"""
from __future__ import (division, unicode_literals, print_function,
                        absolute_import)

import os
import logging
from functools import partial
from collections import defaultdict

from atom.api import Typed, List, Dict
from watchdog.observers import Observer

from ..utils.watchdog import SystematicFileUpdater
from ..utils.plugin_tools import (HasPrefPlugin, ExtensionsCollector,
                                  DeclaratorsCollector,
                                  make_extension_validator)
from .user import InstrUser
from .starters.base_starter import Starter
from .drivers.driver_decl import Driver
from .connections.base_connection import Connection
from .settings.base_settings import Settings
from .manufacturer_aliases import ManufacturerAlias
from .infos import ManufacturerHolder, ProfileInfos

logger = logging.getLogger(__name__)

DRIVERS_POINT = 'ecpy.instruments.drivers'

STARTERS_POINT = 'ecpy.instruments.starters'

USERS_POINT = 'ecpy.instruments.users'

CONNECTIONS_POINT = 'ecpy.instruments.connections'

SETTINGS_POINT = 'ecpy.instuments.settings'

ALIASES_POINT = 'ecpy.instrumets.manufaturer_aliases'

class InstrumentManagerPlugin(HasPrefPlugin):
    """The instrument plugin manages the instrument drivers and their use.

    """
    #: List of the known instrument profile ids.
    profiles = List()

    #: List of instruments for which at least one driver is declared.
    instruments = List()

    #: List of registered intrument users.
    #: Only registered users can be granted the use of an instrument.
    users = List()

    #: List of registered instrument starters.
    starters = List()

    #: List of registered connection types.
    connections = List()

    #: List of registered settings.
    settings = List()

    #: Currently used profiles.
    #: This dict should be edited by user code.
    used_profiles = Dict()

    def start(self):
        """Start the plugin lifecycle by collecting all contributions.

        """
        super(InstrumentManagerPlugin, self).start()

        core = self.workbench.get_plugin('enaml.workbench.core')
        core.invoke_command('ecpy.app.errors.enter_error_gathering')

        core = self.workbench.get_plugin('enaml.workbench.core')
        state = core.invoke_command('ecpy.app.states.get',
                                    {'state_id': 'ecpy.app.directory'})

        i_dir = os.path.join(state.app_directory, 'instruments')
        # Create instruments subfolder if it does not exist.
        if not os.path.isdir(i_dir):
            os.mkdir(i_dir)

        p_dir = os.path.join(state.app_directory, 'profiles')
        # Create profiles subfolder if it does not exist.
        if not os.path.isdir(p_dir):
            os.mkdir(p_dir)

        self._profiles_folders = [p_dir]

        checker = make_extension_validator(InstrUser, (), ('plugin_id'))
        self._users = ExtensionsCollector(workbench=self.workbench,
                                          point=USERS_POINT,
                                          ext_class=InstrUser,
                                          validate_ext=checker)
        self._users.start()

        checker = make_extension_validator(Starter,
                                           ('initialize', 'check_infos',
                                            'finalize'),
                                           ('id', 'description'))
        self._starters = ExtensionsCollector(workbench=self.workbench,
                                             point=STARTERS_POINT,
                                             ext_class=Starter,
                                             validate_ext=checker)
        self._starters.start()

        checker = make_extension_validator(Connection, ('new',),
                                           ('id', 'description'))
        self._connections = ExtensionsCollector(workbench=self.workbench,
                                                point=CONNECTIONS_POINT,
                                                ext_class=Connection,
                                                validate_ext=checker)
        self._connections.start()

        checker = make_extension_validator(Settings, ('new',),
                                           ('id', 'description'))
        self._settings = ExtensionsCollector(workbench=self.workbench,
                                             point=SETTINGS_POINT,
                                             ext_class=Settings,
                                             validate_ext=checker)
        self._settings.start()

        checker = make_extension_validator(ManufacturerAlias, (),
                                           ('name', 'aliases'))
        self._aliases = ExtensionsCollector(workbench=self.workbench,
                                            point=ALIASES_POINT,
                                            ext_class=ManufacturerAlias,
                                            validate_ext=checker)
        self._aliases.start()

        self._drivers = DeclaratorsCollector(workbench=self.workbench,
                                             point=DRIVERS_POINT,
                                             ext_class=Driver)
        self._drivers.start()

        for contrib in ('users', 'starters', 'connections', 'settings'):
            self._update_contribs(contrib, None)

        # XXXX handle dynamic addition of drivers by observing contributions
        # and updating the manufacturers infos accordingly.

        self._refresh_profiles()

        self._bind_observers()

        core.invoke_command('ecpy.app.errors.exit_error_gathering')

    def stop(self):
        """Stop the plugin and remove all observers.

        """
        self._unbind_observers()

        for contrib in ('drivers', 'users', 'starters', 'connections',
                        'settings'):
            getattr(self, '_'+contrib).stop()

    def create_connection(self, connection_id, infos):
        """Create a connection and initialize it.

        Parameters
        ----------
        connection_id : unicode
            Id of the the connection to instantiate.

        infos : dict
            Dictionarry to use to initialize the state of the connection.

        Returns
        -------
        connection : BaseConnection
            Ready to use widget.

        """
        c_decl = self._connections.contributions[connection_id]
        return c_decl.new(self.workbench, infos)

    def create_settings(self, settings_id, infos):
        """Create a settings and initialize it.

        Parameters
        ----------
        settings_id : unicode
            Id of the the settings to instantiate.

        infos : dict
            Dictionarry to use to initialize the state of the settings.

        Returns
        -------
        connection : BaseSettings
            Ready to use widget.

        """
        s_decl = self._settings.contributions[settings_id]
        return s_decl.new(self.workbench, infos)

    def get_drivers(self, drivers):
        """Query drivers class.

        Parameters
        ----------
        drivers : list
            List of driver ids for which the matching class should be returned.

        Returns
        -------
        drivers : dict
            Requested drivers per id.

        """
        pass

    def get_profiles(self, user_id, profiles):
        """Query profiles for use by a declared user.

        Parameters
        ----------
        user_id : unicode
            Id of the user which request the authorization to use the
            instrument.

        profile_id : list
            Ids of the instrument profiles which are requested.

        Returns
        -------
        profiles : dict
            Requested profiles as a dictionary.

        """
        pass  # XXXX

    def release_profiles(self, user_id, profiles):
        """Release some previously acquired profiles.

        The user should not maintain any communication with the instruments
        whose profiles have been released after calling this method.

        """
        pass  # XXXX

    def get_aliases(self, manufacturer):
        """
        """
        pass  # XXXX

    # =========================================================================
    # --- Private API ---------------------------------------------------------
    # =========================================================================

    #: Collector of drivers.
    _drivers = Typed(DeclaratorsCollector)

    #: Collector for the manufacturer aliases.
    _aliases = Typed(ExtensionsCollector)

    #: Declared manufacturers storing the corresponding model infos.
    _manufacturers = Typed(ManufacturerHolder)

    #: Collector of users.
    _users = Typed(ExtensionsCollector)

    #: Collector of starters.
    _starters = Typed(ExtensionsCollector)

    #: Collectorsof connections.
    _connections = Typed(ExtensionsCollector)

    #: Collector of settings.
    _settings = Typed(ExtensionsCollector)

    #: List of folders in which to search for profiles.
    # TODO make that list editable and part of the preferences
    _profiles_folders = List()

    #: Mapping of profile name to profile infos.
    _profiles = Dict()

    #: Watchdog observer tracking changes to the profiles folders.
    _observer = Typed(Observer, ())

    def _update_contribs(self, name, change):
        """Update the list of available contributions (editors, engines, tools)
        when they change.

        """
        setattr(self, name, list(getattr(self, '_'+name).contributions))

    def _refresh_profiles(self):
        """List of profiles living in the profiles folders.

        """
        profiles = {}
        for path in self._profiles_folders:
            if os.path.isdir(path):
                filenames = sorted(f for f in os.listdir(path)
                                   if f.endswith('.instr.ini') and
                                   (os.path.isfile(os.path.join(path, f))))

                for filename in filenames:
                    profile_path = os.path.join(path, filename)
                    # Beware redundant names are overwrited
                    name = filename[:-len('.instr.ini')]
                    profiles[name] = ProfileInfos(path=profile_path,
                                                  plugin=self)
            else:
                logger = logging.getLogger(__name__)
                logger.warn('{} is not a valid directory'.format(path))

        self._profiles = profiles

    def _update_profiles(self, change):
        """Update the known profiles after a modification into one of the
        profiles folders.

        """
        self._refresh_profiles()

    def _bind_observers(self):
        """Start the observers.

        """
        for contrib in ('users', 'starters', 'connections', 'settings'):
            callback = partial(self._update_contribs, contrib)
            getattr(self, '_'+contrib).observe('contributions', callback)

        for folder in self._profiles_folders:
            handler = SystematicFileUpdater(self._update_profiles)
            self._observer.schedule(handler, folder, recursive=True)

        self._observer.start()

    def _unbind_observers(self):
        """Stop the observers.

        """
        for contrib in ('users', 'starters', 'connections', 'settings'):
            callback = partial(self._update_contribs, contrib)
            getattr(self, '_'+contrib).observe('contributions', callback)

        self._observer.unschedule_all()
        self._observer.stop()
        try:
            self._observer.join()
        except RuntimeError:
            pass

    def _default__manufacturers(self):
        """Delayed till this is first needed.

        """
        holder = ManufacturerHolder()
        manufacturers = defaultdict(list)
        for d in self._drivers.contributions:
            manufacturers[d.infos['manufacturer']].append(d)

        for m in manufacturers.values():
            holder.update_manufacturer(m)

        return holder

# XXXX
# Request driver profile starter triplet

# Explore drivers (by manufacturer or kind), connections, settings
<commit_msg>instruments : complete plugin (still some TODOs but they will be tackled later).<commit_after># -*- coding: utf-8 -*-
# -----------------------------------------------------------------------------
# Copyright 2015 by Ecpy Authors, see AUTHORS for more details.
#
# Distributed under the terms of the BSD license.
#
# The full license is in the file LICENCE, distributed with this software.
# -----------------------------------------------------------------------------
"""Instrument manager plugin.

"""
from __future__ import (division, unicode_literals, print_function,
                        absolute_import)

import os
import logging
from functools import partial
from collections import defaultdict

from atom.api import Typed, List, Dict
from watchdog.observers import Observer

from ..utils.watchdog import SystematicFileUpdater
from ..utils.plugin_tools import (HasPrefPlugin, ExtensionsCollector,
                                  DeclaratorsCollector,
                                  make_extension_validator)
from .user import InstrUser
from .starters.base_starter import Starter
from .drivers.driver_decl import Driver
from .connections.base_connection import Connection
from .settings.base_settings import Settings
from .manufacturer_aliases import ManufacturerAlias
from .infos import ManufacturerHolder, ProfileInfos

logger = logging.getLogger(__name__)

DRIVERS_POINT = 'ecpy.instruments.drivers'

STARTERS_POINT = 'ecpy.instruments.starters'

USERS_POINT = 'ecpy.instruments.users'

CONNECTIONS_POINT = 'ecpy.instruments.connections'

SETTINGS_POINT = 'ecpy.instuments.settings'

ALIASES_POINT = 'ecpy.instrumets.manufaturer_aliases'

# TODO add a way to specify default values fr settings from the preferences
class InstrumentManagerPlugin(HasPrefPlugin):
    """The instrument plugin manages the instrument drivers and their use.

    """
    #: List of the known instrument profile ids.
    profiles = List()

    #: List of instruments for which at least one driver is declared.
    instruments = List()

    #: List of registered intrument users.
    #: Only registered users can be granted the use of an instrument.
    users = List()

    #: List of registered instrument starters.
    starters = List()

    #: List of registered connection types.
    connections = List()

    #: List of registered settings.
    settings = List()

    #: Currently used profiles.
    #: This dict should be edited by user code.
    used_profiles = Dict()

    def start(self):
        """Start the plugin lifecycle by collecting all contributions.

        """
        super(InstrumentManagerPlugin, self).start()

        core = self.workbench.get_plugin('enaml.workbench.core')
        core.invoke_command('ecpy.app.errors.enter_error_gathering')

        core = self.workbench.get_plugin('enaml.workbench.core')
        state = core.invoke_command('ecpy.app.states.get',
                                    {'state_id': 'ecpy.app.directory'})

        i_dir = os.path.join(state.app_directory, 'instruments')
        # Create instruments subfolder if it does not exist.
        if not os.path.isdir(i_dir):
            os.mkdir(i_dir)

        p_dir = os.path.join(state.app_directory, 'profiles')
        # Create profiles subfolder if it does not exist.
        if not os.path.isdir(p_dir):
            os.mkdir(p_dir)

        self._profiles_folders = [p_dir]

        checker = make_extension_validator(InstrUser, (), ('plugin_id'))
        self._users = ExtensionsCollector(workbench=self.workbench,
                                          point=USERS_POINT,
                                          ext_class=InstrUser,
                                          validate_ext=checker)
        self._users.start()

        checker = make_extension_validator(Starter,
                                           ('initialize', 'check_infos',
                                            'finalize'),
                                           ('id', 'description'))
        self._starters = ExtensionsCollector(workbench=self.workbench,
                                             point=STARTERS_POINT,
                                             ext_class=Starter,
                                             validate_ext=checker)
        self._starters.start()

        checker = make_extension_validator(Connection, ('new',),
                                           ('id', 'description'))
        self._connections = ExtensionsCollector(workbench=self.workbench,
                                                point=CONNECTIONS_POINT,
                                                ext_class=Connection,
                                                validate_ext=checker)
        self._connections.start()

        checker = make_extension_validator(Settings, ('new',),
                                           ('id', 'description'))
        self._settings = ExtensionsCollector(workbench=self.workbench,
                                             point=SETTINGS_POINT,
                                             ext_class=Settings,
                                             validate_ext=checker)
        self._settings.start()

        checker = make_extension_validator(ManufacturerAlias, (),
                                           ('name', 'aliases'))
        self._aliases = ExtensionsCollector(workbench=self.workbench,
                                            point=ALIASES_POINT,
                                            ext_class=ManufacturerAlias,
                                            validate_ext=checker)
        self._aliases.start()

        self._drivers = DeclaratorsCollector(workbench=self.workbench,
                                             point=DRIVERS_POINT,
                                             ext_class=Driver)
        self._drivers.start()

        err = False
        details = {}
        for d_id, d_infos in self._drivers.contributions.items():
            res, tb = d_infos.validate(self)
            if res:
                err = True
                details[d_id] = tb

        if err:
            core.invoke_command('ecpy.app.errors.signal',
                                {'kind': 'ecpy.driver-validation',
                                 'details': details})
        # TODO providing in app a way to have a splash screen while starting to
        # let the user know what is going on would be nice

        for contrib in ('users', 'starters', 'connections', 'settings'):
            self._update_contribs(contrib, None)

        # TODO handle dynamic addition of drivers by observing contributions
        # and updating the manufacturers infos accordingly.

        self._refresh_profiles()

        self._bind_observers()

        core.invoke_command('ecpy.app.errors.exit_error_gathering')

    def stop(self):
        """Stop the plugin and remove all observers.

        """
        self._unbind_observers()

        for contrib in ('drivers', 'users', 'starters', 'connections',
                        'settings'):
            getattr(self, '_'+contrib).stop()

    def create_connection(self, connection_id, infos):
        """Create a connection and initialize it.

        Parameters
        ----------
        connection_id : unicode
            Id of the the connection to instantiate.

        infos : dict
            Dictionarry to use to initialize the state of the connection.

        Returns
        -------
        connection : BaseConnection
            Ready to use widget.

        """
        c_decl = self._connections.contributions[connection_id]
        return c_decl.new(self.workbench, infos)

    def create_settings(self, settings_id, infos):
        """Create a settings and initialize it.

        Parameters
        ----------
        settings_id : unicode
            Id of the the settings to instantiate.

        infos : dict
            Dictionarry to use to initialize the state of the settings.

        Returns
        -------
        connection : BaseSettings
            Ready to use widget.

        """
        s_decl = self._settings.contributions[settings_id]
        return s_decl.new(self.workbench, infos)

    def get_drivers(self, drivers):
        """Query drivers class and the associated starters.

        Parameters
        ----------
        drivers : list
            List of driver ids for which the matching class should be returned.

        Returns
        -------
        drivers : dict
            Requested drivers and associated starter indexed by id.

        missing : list
            List of ids which do not correspond to any known valid driver.

        """
        ds = self._drivers.contributions
        knowns = {d_id: ds[d_id] for d_id in drivers if d_id in ds}
        missing = list(set(drivers) - set(knowns))

        return {d_id: (infos.cls, self._starters[infos.starter])
                for d_id, infos in knowns.items()}, missing

    def get_profiles(self, user_id, profiles, try_release=True, partial=False):
        """Query profiles for use by a declared user.

        Parameters
        ----------
        user_id : unicode
            Id of the user which request the authorization to use the
            instrument.

        profile_id : list
            Ids of the instrument profiles which are requested.

        try_release : bool, optional
            Should we attempt to release currently used profiles.

        partial : bool, optional
            Should only a subset of the requested profiles be returned if some
            profiles are not available.

        Returns
        -------
        profiles : dict
            Requested profiles as a dictionary.

        unavailable : list
            List of profiles that are not currently available and cannot be
            released.

        """
        if user_id not in self.users:
            raise ValueError('Unknown instrument user tried to query profiles')

        used = [p for p in profiles if p in self.used_profiles]
        if used:
            if not try_release:
                unavailable = used
                released = []
            else:
                used_by_owner = defaultdict(set)
                unavailable = []
                released = []
                for p in used:
                    used_by_owner[self.used_profiles[p]].add(p)
                for o in used_by_owner:
                    user = self._users.contributions[o]
                    if user.policy == 'releasable':
                        to_release = used_by_owner[o]
                        r = o.release_profiles(to_release)
                        unavailable.extend(set(to_release) - set(r))
                        released.extend(r)
                    else:
                        unavailable.extend(used_by_owner[0])

        if unavailable and not partial:
            if released:
                used = {k: v for k, v in self.used_profiles.items()
                        if k not in released}
            return {}, unavailable

        available = ([p for p in profiles if p not in unavailable]
                     if unavailable else profiles)

        queried = {}
        for p in available:
            queried[p] = self._profiles[p]._config.to_dict()

        return queried, unavailable

    def release_profiles(self, user_id, profiles):
        """Release some previously acquired profiles.

        The user should not maintain any communication with the instruments
        whose profiles have been released after calling this method.

        """
        self.used_profiles = {k: v for k, v in self.used_profiles.items()
                              if k not in profiles or v != user_id}

    def get_aliases(self, manufacturer):
        """List the known aliases of a manufacturer.

        Parameters
        ----------
        manufacturer : id
            Name of the manufacturer for which to retun the aliases.

        Returns
        -------
        aliases : list[unicode]
            Known aliases of the manufacturer.

        """
        return self._aliases.contributions.get(manufacturer, [])

    # =========================================================================
    # --- Private API ---------------------------------------------------------
    # =========================================================================

    #: Collector of drivers.
    _drivers = Typed(DeclaratorsCollector)

    #: Collector for the manufacturer aliases.
    _aliases = Typed(ExtensionsCollector)

    #: Declared manufacturers storing the corresponding model infos.
    _manufacturers = Typed(ManufacturerHolder)

    #: Collector of users.
    _users = Typed(ExtensionsCollector)

    #: Collector of starters.
    _starters = Typed(ExtensionsCollector)

    #: Collectorsof connections.
    _connections = Typed(ExtensionsCollector)

    #: Collector of settings.
    _settings = Typed(ExtensionsCollector)

    #: List of folders in which to search for profiles.
    # TODO make that list editable and part of the preferences
    _profiles_folders = List()

    #: Mapping of profile name to profile infos.
    _profiles = Dict()

    #: Watchdog observer tracking changes to the profiles folders.
    _observer = Typed(Observer, ())

    def _update_contribs(self, name, change):
        """Update the list of available contributions (editors, engines, tools)
        when they change.

        """
        setattr(self, name, list(getattr(self, '_'+name).contributions))

    def _refresh_profiles(self):
        """List of profiles living in the profiles folders.

        """
        profiles = {}
        for path in self._profiles_folders:
            if os.path.isdir(path):
                filenames = sorted(f for f in os.listdir(path)
                                   if f.endswith('.instr.ini') and
                                   (os.path.isfile(os.path.join(path, f))))

                for filename in filenames:
                    profile_path = os.path.join(path, filename)
                    # Beware redundant names are overwrited
                    name = filename[:-len('.instr.ini')]
                    profiles[name] = ProfileInfos(path=profile_path,
                                                  plugin=self)
            else:
                logger = logging.getLogger(__name__)
                logger.warn('{} is not a valid directory'.format(path))

        self._profiles = profiles

    def _update_profiles(self, change):
        """Update the known profiles after a modification into one of the
        profiles folders.

        """
        self._refresh_profiles()

    def _bind_observers(self):
        """Start the observers.

        """
        for contrib in ('users', 'starters', 'connections', 'settings'):
            callback = partial(self._update_contribs, contrib)
            getattr(self, '_'+contrib).observe('contributions', callback)

        for folder in self._profiles_folders:
            handler = SystematicFileUpdater(self._update_profiles)
            self._observer.schedule(handler, folder, recursive=True)

        self._observer.start()

    def _unbind_observers(self):
        """Stop the observers.

        """
        for contrib in ('users', 'starters', 'connections', 'settings'):
            callback = partial(self._update_contribs, contrib)
            getattr(self, '_'+contrib).observe('contributions', callback)

        self._observer.unschedule_all()
        self._observer.stop()
        try:
            self._observer.join()
        except RuntimeError:
            pass

    def _default__manufacturers(self):
        """Delayed till this is first needed.

        """
        holder = ManufacturerHolder()
        manufacturers = defaultdict(list)
        valid_drivers = [d for d in self._drivers.contributions.values()]
        for d in valid_drivers:
            manufacturers[d.infos['manufacturer']].append(d)

        for m in manufacturers.values():
            holder.update_manufacturer(m)

        return holder
<|endoftext|>


<commit_before>#
# This source file is part of the EdgeDB open source project.
#
# Copyright 2016-present MagicStack Inc. and the EdgeDB authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from __future__ import annotations

import contextlib
import functools
import os
import pathlib
import shutil
import sys
import tempfile
import unittest

import click
import psutil

import edb
from edb.common import devmode
from edb.testbase.server import get_test_cases
from edb.tools.edb import edbcommands

from .decorators import not_implemented
from .decorators import xfail
from .decorators import skip

from . import loader
from . import mproc_fixes
from . import runner
from . import styles

__all__ = ('not_implemented', 'xfail', 'skip')

@edbcommands.command()
@click.argument('files', nargs=-1, metavar='[file or directory]...')
@click.option('-v', '--verbose', is_flag=True,
              help='increase verbosity')
@click.option('-q', '--quiet', is_flag=True,
              help='decrease verbosity')
@click.option('--debug', is_flag=True,
              help='output internal debug logs')
@click.option('--output-format',
              type=click.Choice(runner.OutputFormat.__members__),
              help='test progress output style',
              default=runner.OutputFormat.auto.value)
@click.option('--warnings/--no-warnings',
              help='enable or disable warnings (enabled by default)',
              default=True)
@click.option('-j', '--jobs', type=int,
              default=lambda: psutil.cpu_count(logical=False),
              help='number of parallel processes to use')
@click.option('-s', '--shard', type=str,
              default='1/1',
              help='run tests in shards (current/total)')
@click.option('-k', '--include', type=str, multiple=True, metavar='REGEXP',
              help='only run tests which match the given regular expression')
@click.option('-e', '--exclude', type=str, multiple=True, metavar='REGEXP',
              help='do not run tests which match the given regular expression')
@click.option('-x', '--failfast', is_flag=True,
              help='stop tests after a first failure/error')
@click.option('--shuffle', is_flag=True,
              help='shuffle the order in which tests are run')
@click.option('--repeat', type=int, default=1,
              help='repeat tests N times or until first unsuccessful run')
@click.option('--cov', type=str, multiple=True,
              help='package name to measure code coverage for, '
                   'can be specified multiple times '
                   '(e.g --cov edb.common --cov edb.server)')
@click.option('--running-times-log', 'running_times_log_file',
              type=click.File('a+'), metavar='FILEPATH',
              help='Maintain a running time log file at FILEPATH')
@click.option('--list', 'list_tests', is_flag=True,
              help='list all the tests and exit')
@click.option('--postgres-dsn', type=str,
              help='Use the specified Postgres cluster instead of starting a '
                   'temporary local one.')
def test(*, files, jobs, shard, include, exclude, verbose, quiet, debug,
         output_format, warnings, failfast, shuffle, cov, repeat,
         running_times_log_file, list_tests, postgres_dsn):
    """Run EdgeDB test suite.

    Discovers and runs tests in the specified files or directories.
    If no files or directories are specified, current directory is assumed.
    """
    if quiet:
        if verbose:
            click.secho(
                'Warning: both --quiet and --verbose are '
                'specified, assuming --quiet.', fg='yellow')
        verbosity = 0
    elif verbose:
        verbosity = 2
    else:
        verbosity = 1

    mproc_fixes.patch_multiprocessing(debug=debug)

    output_format = runner.OutputFormat(output_format)
    if verbosity > 1 and output_format is runner.OutputFormat.stacked:
        click.secho(
            'Error: cannot use stacked output format in verbose mode.',
            fg='red')
        sys.exit(1)

    if repeat < 1:
        click.secho(
            'Error: --repeat must be a positive non-zero number.', fg='red')
        sys.exit(1)

    if not files:
        cwd = os.path.abspath(os.getcwd())
        if os.path.exists(os.path.join(cwd, 'tests')):
            files = ('tests',)
        else:
            click.secho(
                'Error: no test path specified and no "tests" directory found',
                fg='red')
            sys.exit(1)

    for file in files:
        if not os.path.exists(file):
            click.secho(
                f'Error: test path {file!r} does not exist', fg='red')
            sys.exit(1)

    try:
        selected_shard, total_shards = map(int, shard.split('/'))
    except Exception:
        click.secho(f'Error: --shard {shard} must match format e.g. 2/5')
        sys.exit(1)

    if selected_shard < 1 or selected_shard > total_shards:
        click.secho(f'Error: --shard {shard} is out of bound')
        sys.exit(1)

    run = functools.partial(
        _run,
        include=include,
        exclude=exclude,
        verbosity=verbosity,
        files=files,
        jobs=jobs,
        output_format=output_format,
        warnings=warnings,
        failfast=failfast,
        shuffle=shuffle,
        repeat=repeat,
        selected_shard=selected_shard,
        total_shards=total_shards,
        running_times_log_file=running_times_log_file,
        list_tests=list_tests,
        postgres_dsn=postgres_dsn,
    )

    if cov:
        for pkg in cov:
            if '\\' in pkg or '/' in pkg or pkg.endswith('.py'):
                click.secho(
                    f'Error: --cov argument {pkg!r} looks like a path, '
                    f'expected a Python package name', fg='red')
                sys.exit(1)

        with _coverage_wrapper(cov):
            result = run()
    else:
        result = run()

    sys.exit(result)

@contextlib.contextmanager
def _coverage_wrapper(paths):
    try:
        import coverage  # NoQA
    except ImportError:
        click.secho(
            'Error: "coverage" package is missing, cannot run tests '
            'with --cov')
        sys.exit(1)

    for path in edb.__path__:
        cov_rc = pathlib.Path(path).parent / '.coveragerc'
        if cov_rc.exists():
            break
    else:
        raise RuntimeError('cannot locate the .coveragerc file')

    with tempfile.TemporaryDirectory() as td:
        cov_config = devmode.CoverageConfig(
            paths=paths,
            config=str(cov_rc),
            datadir=td)
        cov_config.save_to_environ()

        main_cov = cov_config.new_coverage_object()
        main_cov.start()

        try:
            yield
        finally:
            main_cov.stop()
            main_cov.save()

            data = coverage.CoverageData()

            with os.scandir(td) as it:
                for entry in it:
                    new_data = coverage.CoverageData()
                    new_data.read_file(entry.path)
                    data.update(new_data)

            covfile = str(pathlib.Path(td) / '.coverage')
            data.write_file(covfile)
            report_cov = cov_config.new_custom_coverage_object(
                config_file=str(cov_rc),
                data_file=covfile,
            )
            report_cov.load()
            click.secho('Coverage:')
            report_cov.report()
            # store the coverage file in cwd, so it can be used to produce
            # additional reports with coverage cli
            shutil.copy(covfile, '.')

def _run(*, include, exclude, verbosity, files, jobs, output_format,
         warnings, failfast, shuffle, repeat, selected_shard, total_shards,
         running_times_log_file, list_tests, postgres_dsn):
    suite = unittest.TestSuite()

    total = 0
    total_unfiltered = 0

    if verbosity > 0:
        def _update_progress(n, unfiltered_n):
            nonlocal total, total_unfiltered
            total += n
            total_unfiltered += unfiltered_n
            click.echo(styles.status(
                f'Collected {total}/{total_unfiltered} tests.\r'),
                nl=False, err=list_tests)
    else:
        _update_progress = None

    test_loader = loader.TestLoader(
        verbosity=verbosity, exclude=exclude, include=include,
        progress_cb=_update_progress)

    for file in files:
        if not os.path.exists(file) and verbosity > 0:
            click.echo(styles.warning(
                f'Warning: {file}: no such file or directory.'))

        if os.path.isdir(file):
            tests = test_loader.discover(file)
        else:
            tests = test_loader.discover(
                os.path.dirname(file),
                pattern=os.path.basename(file))

        suite.addTest(tests)

    if list_tests:
        click.echo(err=True)
        cases = get_test_cases([suite])
        for tests in cases.values():
            for test in tests:
                click.echo(str(test))
        return 0

    jobs = max(min(total, jobs), 1)

    if verbosity > 0:
        click.echo()
        if jobs > 1:
            click.echo(styles.status(
                f'Using up to {jobs} processes to run tests.'))

    for rnum in range(repeat):
        if repeat > 1:
            click.echo(styles.status(
                f'Repeat #{rnum + 1} out of {repeat}.'))

        test_runner = runner.ParallelTextTestRunner(
            verbosity=verbosity, output_format=output_format,
            warnings=warnings, num_workers=jobs,
            failfast=failfast, shuffle=shuffle, postgres_dsn=postgres_dsn)

        result = test_runner.run(
            suite, selected_shard, total_shards, running_times_log_file,
        )

        if not result.wasSuccessful():
            return 1

    return 0
<commit_msg>Make edb test -j0 mean the current default of number of CPU cores (#2512)<commit_after>#
# This source file is part of the EdgeDB open source project.
#
# Copyright 2016-present MagicStack Inc. and the EdgeDB authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from __future__ import annotations

import contextlib
import functools
import os
import pathlib
import shutil
import sys
import tempfile
import unittest

import click
import psutil

import edb
from edb.common import devmode
from edb.testbase.server import get_test_cases
from edb.tools.edb import edbcommands

from .decorators import not_implemented
from .decorators import xfail
from .decorators import skip

from . import loader
from . import mproc_fixes
from . import runner
from . import styles

__all__ = ('not_implemented', 'xfail', 'skip')

@edbcommands.command()
@click.argument('files', nargs=-1, metavar='[file or directory]...')
@click.option('-v', '--verbose', is_flag=True,
              help='increase verbosity')
@click.option('-q', '--quiet', is_flag=True,
              help='decrease verbosity')
@click.option('--debug', is_flag=True,
              help='output internal debug logs')
@click.option('--output-format',
              type=click.Choice(runner.OutputFormat.__members__),
              help='test progress output style',
              default=runner.OutputFormat.auto.value)
@click.option('--warnings/--no-warnings',
              help='enable or disable warnings (enabled by default)',
              default=True)
@click.option('-j', '--jobs', type=int,
              default=0,
              help='number of parallel processes to use, default is 0, which '
                   'means choose automatically based on the number of '
                   'available CPU cores')
@click.option('-s', '--shard', type=str,
              default='1/1',
              help='run tests in shards (current/total)')
@click.option('-k', '--include', type=str, multiple=True, metavar='REGEXP',
              help='only run tests which match the given regular expression')
@click.option('-e', '--exclude', type=str, multiple=True, metavar='REGEXP',
              help='do not run tests which match the given regular expression')
@click.option('-x', '--failfast', is_flag=True,
              help='stop tests after a first failure/error')
@click.option('--shuffle', is_flag=True,
              help='shuffle the order in which tests are run')
@click.option('--repeat', type=int, default=1,
              help='repeat tests N times or until first unsuccessful run')
@click.option('--cov', type=str, multiple=True,
              help='package name to measure code coverage for, '
                   'can be specified multiple times '
                   '(e.g --cov edb.common --cov edb.server)')
@click.option('--running-times-log', 'running_times_log_file',
              type=click.File('a+'), metavar='FILEPATH',
              help='Maintain a running time log file at FILEPATH')
@click.option('--list', 'list_tests', is_flag=True,
              help='list all the tests and exit')
@click.option('--postgres-dsn', type=str,
              help='Use the specified Postgres cluster instead of starting a '
                   'temporary local one.')
def test(*, files, jobs, shard, include, exclude, verbose, quiet, debug,
         output_format, warnings, failfast, shuffle, cov, repeat,
         running_times_log_file, list_tests, postgres_dsn):
    """Run EdgeDB test suite.

    Discovers and runs tests in the specified files or directories.
    If no files or directories are specified, current directory is assumed.
    """
    if quiet:
        if verbose:
            click.secho(
                'Warning: both --quiet and --verbose are '
                'specified, assuming --quiet.', fg='yellow')
        verbosity = 0
    elif verbose:
        verbosity = 2
    else:
        verbosity = 1

    if jobs == 0:
        jobs = psutil.cpu_count(logical=False)

    mproc_fixes.patch_multiprocessing(debug=debug)

    output_format = runner.OutputFormat(output_format)
    if verbosity > 1 and output_format is runner.OutputFormat.stacked:
        click.secho(
            'Error: cannot use stacked output format in verbose mode.',
            fg='red')
        sys.exit(1)

    if repeat < 1:
        click.secho(
            'Error: --repeat must be a positive non-zero number.', fg='red')
        sys.exit(1)

    if not files:
        cwd = os.path.abspath(os.getcwd())
        if os.path.exists(os.path.join(cwd, 'tests')):
            files = ('tests',)
        else:
            click.secho(
                'Error: no test path specified and no "tests" directory found',
                fg='red')
            sys.exit(1)

    for file in files:
        if not os.path.exists(file):
            click.secho(
                f'Error: test path {file!r} does not exist', fg='red')
            sys.exit(1)

    try:
        selected_shard, total_shards = map(int, shard.split('/'))
    except Exception:
        click.secho(f'Error: --shard {shard} must match format e.g. 2/5')
        sys.exit(1)

    if selected_shard < 1 or selected_shard > total_shards:
        click.secho(f'Error: --shard {shard} is out of bound')
        sys.exit(1)

    run = functools.partial(
        _run,
        include=include,
        exclude=exclude,
        verbosity=verbosity,
        files=files,
        jobs=jobs,
        output_format=output_format,
        warnings=warnings,
        failfast=failfast,
        shuffle=shuffle,
        repeat=repeat,
        selected_shard=selected_shard,
        total_shards=total_shards,
        running_times_log_file=running_times_log_file,
        list_tests=list_tests,
        postgres_dsn=postgres_dsn,
    )

    if cov:
        for pkg in cov:
            if '\\' in pkg or '/' in pkg or pkg.endswith('.py'):
                click.secho(
                    f'Error: --cov argument {pkg!r} looks like a path, '
                    f'expected a Python package name', fg='red')
                sys.exit(1)

        with _coverage_wrapper(cov):
            result = run()
    else:
        result = run()

    sys.exit(result)

@contextlib.contextmanager
def _coverage_wrapper(paths):
    try:
        import coverage  # NoQA
    except ImportError:
        click.secho(
            'Error: "coverage" package is missing, cannot run tests '
            'with --cov')
        sys.exit(1)

    for path in edb.__path__:
        cov_rc = pathlib.Path(path).parent / '.coveragerc'
        if cov_rc.exists():
            break
    else:
        raise RuntimeError('cannot locate the .coveragerc file')

    with tempfile.TemporaryDirectory() as td:
        cov_config = devmode.CoverageConfig(
            paths=paths,
            config=str(cov_rc),
            datadir=td)
        cov_config.save_to_environ()

        main_cov = cov_config.new_coverage_object()
        main_cov.start()

        try:
            yield
        finally:
            main_cov.stop()
            main_cov.save()

            data = coverage.CoverageData()

            with os.scandir(td) as it:
                for entry in it:
                    new_data = coverage.CoverageData()
                    new_data.read_file(entry.path)
                    data.update(new_data)

            covfile = str(pathlib.Path(td) / '.coverage')
            data.write_file(covfile)
            report_cov = cov_config.new_custom_coverage_object(
                config_file=str(cov_rc),
                data_file=covfile,
            )
            report_cov.load()
            click.secho('Coverage:')
            report_cov.report()
            # store the coverage file in cwd, so it can be used to produce
            # additional reports with coverage cli
            shutil.copy(covfile, '.')

def _run(*, include, exclude, verbosity, files, jobs, output_format,
         warnings, failfast, shuffle, repeat, selected_shard, total_shards,
         running_times_log_file, list_tests, postgres_dsn):
    suite = unittest.TestSuite()

    total = 0
    total_unfiltered = 0

    if verbosity > 0:
        def _update_progress(n, unfiltered_n):
            nonlocal total, total_unfiltered
            total += n
            total_unfiltered += unfiltered_n
            click.echo(styles.status(
                f'Collected {total}/{total_unfiltered} tests.\r'),
                nl=False, err=list_tests)
    else:
        _update_progress = None

    test_loader = loader.TestLoader(
        verbosity=verbosity, exclude=exclude, include=include,
        progress_cb=_update_progress)

    for file in files:
        if not os.path.exists(file) and verbosity > 0:
            click.echo(styles.warning(
                f'Warning: {file}: no such file or directory.'))

        if os.path.isdir(file):
            tests = test_loader.discover(file)
        else:
            tests = test_loader.discover(
                os.path.dirname(file),
                pattern=os.path.basename(file))

        suite.addTest(tests)

    if list_tests:
        click.echo(err=True)
        cases = get_test_cases([suite])
        for tests in cases.values():
            for test in tests:
                click.echo(str(test))
        return 0

    jobs = max(min(total, jobs), 1)

    if verbosity > 0:
        click.echo()
        if jobs > 1:
            click.echo(styles.status(
                f'Using up to {jobs} processes to run tests.'))

    for rnum in range(repeat):
        if repeat > 1:
            click.echo(styles.status(
                f'Repeat #{rnum + 1} out of {repeat}.'))

        test_runner = runner.ParallelTextTestRunner(
            verbosity=verbosity, output_format=output_format,
            warnings=warnings, num_workers=jobs,
            failfast=failfast, shuffle=shuffle, postgres_dsn=postgres_dsn)

        result = test_runner.run(
            suite, selected_shard, total_shards, running_times_log_file,
        )

        if not result.wasSuccessful():
            return 1

    return 0
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Return JSON metadata for nexrad information
$Id: $:
"""
import sys
sys.path.insert(1, "/mesonet/www/apps/iemwebsite/scripts/lib/")
import cgi
import json
import mx.DateTime
import os
import iemdb
import glob

NIDS = {
    'N0Q': 'Base Reflectivity (High Res)',
    'N0U': 'Base Radial Velocity (High Res)',
    'N0S': 'Storm Relative Radial Velocity',
    'NET': 'Echo Tops',
    'N0R': 'Base Reflectivity',
    'N0V': 'Base Radial Velocity',
    'N0Z': 'Base Reflectivity',
    'TR0': 'TDWR Base Reflectivity',
    'TV0': 'TDWR Radial Velocity',
}

def parse_time(s):
    """
    Convert ISO something into a mx.DateTime
    """
    try:
        date = mx.DateTime.strptime(s, '%Y-%m-%dT%H:%MZ')
    except:
        date = mx.DateTime.gmt()
    return date

def available_radars(form):
    """
    Return available RADAR sites for the given location and date!
    """
    lat = form.getvalue('lat', None)
    lon = form.getvalue('lon', None)
    start_gts = parse_time( form.getvalue('start', '2012-01-27T00:00Z') )
    MESOSITE = iemdb.connect('mesosite', bypass=True)
    mcursor = MESOSITE.cursor()
    root = {'radars': []}
    if lat is None or lon is None:
        sql = """
        select id, name,
        x(geom) as lon, y(geom) as lat, network
        from stations where network in ('NEXRAD','ASR4','ASR11','TWDR')
        ORDER by id asc"""
    else:
        sql = """
        select id, name, x(geom) as lon, y(geom) as lat, network,
        ST_Distance(geom, GeomFromEWKT('SRID=4326;POINT(%s %s)')) as dist
        from stations where network in ('NEXRAD','ASR4','ASR11','TWDR')
        and ST_Distance(geom, GeomFromEWKT('SRID=4326;POINT(%s %s)')) < 3
        ORDER by dist asc
        """ % (lon, lat, lon, lat)
    mcursor.execute(sql)
    root['radars'].append({'id': 'USCOMP', 'name': 'National Composite',
                           'lat': 42.5, 'lon': -95, 'type': 'COMPOSITE'})
    for row in mcursor:
        radar = row[0]
        if not os.path.isdir(start_gts.strftime("/mesonet/ARCHIVE/data/%Y/%m/%d/GIS/ridge/"+
                    radar)):
            continue
        root['radars'].append({'id': radar, 'name': row[1], 'lat': row[3],
                               'lon': row[2], 'type': row[4]})
    mcursor.close()
    MESOSITE.close()
    return root

def find_scans(root, radar, product, sts, ets):
    """
    Find scans for a given radar, product, and start and end time
    """
    now = sts
    if radar in ['USCOMP',]:
        now -= mx.DateTime.RelativeDateTime(minutes=(now.minute % 5))
        while now < ets and len(root['scans']) < 300:
            if os.path.isfile( now.strftime("/mesonet/ARCHIVE/data/%Y/%m/%d/GIS/uscomp/"+
                    product.lower() +"_%Y%m%d%H%M.png") ):
                root['scans'].append({'ts': now.strftime("%Y-%m-%dT%H:%MZ")})
            now += mx.DateTime.RelativeDateTime(minutes=5)
    else:
        while now < ets and len(root['scans']) < 300:
            if os.path.isfile( now.strftime("/mesonet/ARCHIVE/data/%Y/%m/%d/GIS/ridge/"+
                    radar+"/"+product+"/"+radar+"_"+product+"_%Y%m%d%H%M.png") ):
                root['scans'].append({'ts': now.strftime("%Y-%m-%dT%H:%MZ")})
            now += mx.DateTime.RelativeDateTime(minutes=1)

def is_realtime(sts):
    """
    Check to see if this time is close to realtime...
    """
    if (mx.DateTime.gmt() - sts).seconds > 3600:
        return False
    return True

def list_files(form):
    """
    List available NEXRAD files based on the form request
    """
    radar = form.getvalue('radar', 'DMX')[:10]
    product = form.getvalue('product', 'N0Q')[:3]
    start_gts = parse_time( form.getvalue('start', '2012-01-27T00:00Z') )
    end_gts = parse_time( form.getvalue('end', '2012-01-27T01:00Z') )
    #root = {'metaData': {'nexrad': nexrad, 'product': product}, 'scans' : []}
    root = {'scans': []}
    find_scans(root, radar, product, start_gts, end_gts)
    if len(root['scans']) == 0 and is_realtime(start_gts):
        now = start_gts - mx.DateTime.RelativeDateTime(minutes=10)
        find_scans(root, radar, product, now, end_gts)

    return root

def list_products(form):
    """
    List available NEXRAD products
    """
    radar = form.getvalue('radar', 'DMX')[:10]
    now = parse_time( form.getvalue('start', '2012-01-27T00:00Z') )
    #root = {'metaData': {'nexrad': nexrad, 'product': product}, 'scans' : []}
    root = {'products': []}
    if radar == 'USCOMP':
        for dir in ['N0Q','N0R']:
            testfp = now.strftime("/mesonet/ARCHIVE/data/%Y/%m/%d/GIS/uscomp/"+
                                  dir.lower()+"_%Y%m%d0000.png")
            if os.path.isfile(testfp):
                root['products'].append({'id': dir, 'name': NIDS.get(dir,dir)})
    else:
        basedir = now.strftime("/mesonet/ARCHIVE/data/%Y/%m/%d/GIS/ridge/"+radar)
        if os.path.isdir( basedir ):
            os.chdir( basedir )
            for dir in glob.glob("???"):
                root['products'].append({'id': dir, 'name': NIDS.get(dir,dir)})
    return root

def main():
    """

    """
    form = cgi.FieldStorage()
    operation = form.getvalue('operation', None)
    if operation == "list":
        print json.dumps( list_files(form) )
    elif operation == "available":
        print json.dumps( available_radars(form) )
    elif operation == "products":
        print json.dumps( list_products(form) )

if __name__ == "__main__":
    print "Content-type: text/plain\n"
    main()<commit_msg>remove svn tag<commit_after>#!/usr/bin/env python
"""
Return JSON metadata for nexrad information
"""
import sys
sys.path.insert(1, "/mesonet/www/apps/iemwebsite/scripts/lib/")
import cgi
import json
import mx.DateTime
import os
import iemdb
import glob

NIDS = {
    'N0Q': 'Base Reflectivity (High Res)',
    'N0U': 'Base Radial Velocity (High Res)',
    'N0S': 'Storm Relative Radial Velocity',
    'NET': 'Echo Tops',
    'N0R': 'Base Reflectivity',
    'N0V': 'Base Radial Velocity',
    'N0Z': 'Base Reflectivity',
    'TR0': 'TDWR Base Reflectivity',
    'TV0': 'TDWR Radial Velocity',
}

def parse_time(s):
    """
    Convert ISO something into a mx.DateTime
    """
    try:
        date = mx.DateTime.strptime(s, '%Y-%m-%dT%H:%MZ')
    except:
        date = mx.DateTime.gmt()
    return date

def available_radars(form):
    """
    Return available RADAR sites for the given location and date!
    """
    lat = form.getvalue('lat', None)
    lon = form.getvalue('lon', None)
    start_gts = parse_time( form.getvalue('start', '2012-01-27T00:00Z') )
    MESOSITE = iemdb.connect('mesosite', bypass=True)
    mcursor = MESOSITE.cursor()
    root = {'radars': []}
    if lat is None or lon is None:
        sql = """
        select id, name,
        x(geom) as lon, y(geom) as lat, network
        from stations where network in ('NEXRAD','ASR4','ASR11','TWDR')
        ORDER by id asc"""
    else:
        sql = """
        select id, name, x(geom) as lon, y(geom) as lat, network,
        ST_Distance(geom, GeomFromEWKT('SRID=4326;POINT(%s %s)')) as dist
        from stations where network in ('NEXRAD','ASR4','ASR11','TWDR')
        and ST_Distance(geom, GeomFromEWKT('SRID=4326;POINT(%s %s)')) < 3
        ORDER by dist asc
        """ % (lon, lat, lon, lat)
    mcursor.execute(sql)
    root['radars'].append({'id': 'USCOMP', 'name': 'National Composite',
                           'lat': 42.5, 'lon': -95, 'type': 'COMPOSITE'})
    for row in mcursor:
        radar = row[0]
        if not os.path.isdir(start_gts.strftime("/mesonet/ARCHIVE/data/%Y/%m/%d/GIS/ridge/"+
                    radar)):
            continue
        root['radars'].append({'id': radar, 'name': row[1], 'lat': row[3],
                               'lon': row[2], 'type': row[4]})
    mcursor.close()
    MESOSITE.close()
    return root

def find_scans(root, radar, product, sts, ets):
    """
    Find scans for a given radar, product, and start and end time
    """
    now = sts
    if radar in ['USCOMP',]:
        now -= mx.DateTime.RelativeDateTime(minutes=(now.minute % 5))
        while now < ets and len(root['scans']) < 300:
            if os.path.isfile( now.strftime("/mesonet/ARCHIVE/data/%Y/%m/%d/GIS/uscomp/"+
                    product.lower() +"_%Y%m%d%H%M.png") ):
                root['scans'].append({'ts': now.strftime("%Y-%m-%dT%H:%MZ")})
            now += mx.DateTime.RelativeDateTime(minutes=5)
    else:
        while now < ets and len(root['scans']) < 300:
            if os.path.isfile( now.strftime("/mesonet/ARCHIVE/data/%Y/%m/%d/GIS/ridge/"+
                    radar+"/"+product+"/"+radar+"_"+product+"_%Y%m%d%H%M.png") ):
                root['scans'].append({'ts': now.strftime("%Y-%m-%dT%H:%MZ")})
            now += mx.DateTime.RelativeDateTime(minutes=1)

def is_realtime(sts):
    """
    Check to see if this time is close to realtime...
    """
    if (mx.DateTime.gmt() - sts).seconds > 3600:
        return False
    return True

def list_files(form):
    """
    List available NEXRAD files based on the form request
    """
    radar = form.getvalue('radar', 'DMX')[:10]
    product = form.getvalue('product', 'N0Q')[:3]
    start_gts = parse_time( form.getvalue('start', '2012-01-27T00:00Z') )
    end_gts = parse_time( form.getvalue('end', '2012-01-27T01:00Z') )
    #root = {'metaData': {'nexrad': nexrad, 'product': product}, 'scans' : []}
    root = {'scans': []}
    find_scans(root, radar, product, start_gts, end_gts)
    if len(root['scans']) == 0 and is_realtime(start_gts):
        now = start_gts - mx.DateTime.RelativeDateTime(minutes=10)
        find_scans(root, radar, product, now, end_gts)

    return root

def list_products(form):
    """
    List available NEXRAD products
    """
    radar = form.getvalue('radar', 'DMX')[:10]
    now = parse_time( form.getvalue('start', '2012-01-27T00:00Z') )
    #root = {'metaData': {'nexrad': nexrad, 'product': product}, 'scans' : []}
    root = {'products': []}
    if radar == 'USCOMP':
        for dir in ['N0Q','N0R']:
            testfp = now.strftime("/mesonet/ARCHIVE/data/%Y/%m/%d/GIS/uscomp/"+
                                  dir.lower()+"_%Y%m%d0000.png")
            if os.path.isfile(testfp):
                root['products'].append({'id': dir, 'name': NIDS.get(dir,dir)})
    else:
        basedir = now.strftime("/mesonet/ARCHIVE/data/%Y/%m/%d/GIS/ridge/"+radar)
        if os.path.isdir( basedir ):
            os.chdir( basedir )
            for dir in glob.glob("???"):
                root['products'].append({'id': dir, 'name': NIDS.get(dir,dir)})
    return root

def main():
    """

    """
    form = cgi.FieldStorage()
    operation = form.getvalue('operation', None)
    if operation == "list":
        print json.dumps( list_files(form) )
    elif operation == "available":
        print json.dumps( available_radars(form) )
    elif operation == "products":
        print json.dumps( list_products(form) )

if __name__ == "__main__":
    print "Content-type: text/plain\n"
    main()<|endoftext|>


<commit_before>#!/usr/bin/env python2.7
#
# Copyright 2014 Canonical Ltd. released under AGPL
#
# Authors:
#  Tycho Andersen <tycho.andersen@canonical.com>
#

# This file is part of the glance-simplestreams sync charm.
#
# The glance-simplestreams sync charm is free software: you can
# redistribute it and/or modify it under the terms of the GNU Affero General
# Public License as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later version.
#
# The charm is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this charm.  If not, see <http://www.gnu.org/licenses/>.

import glob
import os
import sys
import shutil

from charmhelpers.fetch import apt_install, add_source, apt_update
from charmhelpers.core import hookenv
from charmhelpers.payload.execd import execd_preinstall

from charmhelpers.contrib.openstack.context import (AMQPContext,
                                                    IdentityServiceContext,
                                                    OSContextGenerator)
from charmhelpers.contrib.openstack.utils import get_os_codename_package
from charmhelpers.contrib.openstack.templating import OSConfigRenderer

CONF_FILE_DIR = '/etc/glance-simplestreams-sync'
USR_SHARE_DIR = '/usr/share/glance-simplestreams-sync'

MIRRORS_CONF_FILE_NAME = os.path.join(CONF_FILE_DIR, 'mirrors.yaml')
ID_CONF_FILE_NAME = os.path.join(CONF_FILE_DIR, 'identity.yaml')

SCRIPT_NAME = "glance-simplestreams-sync.py"

CRON_JOB_FILENAME = 'glance_simplestreams_sync'
CRON_POLL_FILENAME = 'glance_simplestreams_sync_fastpoll'
CRON_POLL_FILEPATH = os.path.join('/etc/cron.d', CRON_POLL_FILENAME)

hooks = hookenv.Hooks()

class MultipleImageModifierSubordinatesIsNotSupported(Exception):
    """Raise this if multiple image-modifier subordinates are related to
    this charm.
    """

class MirrorsConfigServiceContext(OSContextGenerator):
    """Context for mirrors.yaml template.

    Uses image-modifier relation if available to set
    modify_hook_scripts config value.

    """
    interfaces = ['simplestreams-image-service']

    def __call__(self):
        hookenv.log("Generating template ctxt for simplestreams-image-service")
        config = hookenv.config()

        modify_hook_scripts = []
        image_modifiers = hookenv.relations_of_type('image-modifier')
        if len(image_modifiers) > 1:
            raise MultipleImageModifierSubordinatesIsNotSupported()

        if len(image_modifiers) == 1:
            im = image_modifiers[0]
            try:
                modify_hook_scripts.append(im['script-path'])

            except KeyError as ke:
                hookenv.log('relation {} yielded '
                            'exception {} - ignoring.'.format(repr(im),
                                                              repr(ke)))

        # default no-op so that None still means "missing" for config
        # validation (see elsewhere)
        if len(modify_hook_scripts) == 0:
            modify_hook_scripts.append('/bin/true')

        return dict(mirror_list=config['mirror_list'],
                    modify_hook_scripts=', '.join(modify_hook_scripts),
                    name_prefix=config['name_prefix'],
                    content_id_template=config['content_id_template'],
                    use_swift=config['use_swift'],
                    region=config['region'],
                    cloud_name=config['cloud_name'])

release = get_os_codename_package('glance-common', fatal=False) or 'icehouse'
configs = OSConfigRenderer(templates_dir='templates/',
                           openstack_release=release)

configs.register(MIRRORS_CONF_FILE_NAME, [MirrorsConfigServiceContext()])
configs.register(ID_CONF_FILE_NAME, [IdentityServiceContext(),
                                     AMQPContext()])

def install_cron_script():
    """Installs cron job in /etc/cron.$frequency/ for repeating sync

    Script is not a template but we always overwrite, to ensure it is
    up-to-date.

    """
    sync_script_source = os.path.join("scripts", SCRIPT_NAME)
    shutil.copy(sync_script_source, USR_SHARE_DIR)

    config = hookenv.config()
    installed_script = os.path.join(USR_SHARE_DIR, SCRIPT_NAME)
    linkname = '/etc/cron.{f}/{s}'.format(f=config['frequency'],
                                          s=CRON_JOB_FILENAME)
    os.symlink(installed_script, linkname)

def install_cron_poll():
    "Installs /etc/cron.d every-minute job in crontab for quick polling."
    poll_file_source = os.path.join('scripts', CRON_POLL_FILENAME)
    shutil.copy(poll_file_source, '/etc/cron.d/')

def uninstall_cron_script():
    "Removes sync program from any cron place it might be"
    for fn in glob.glob("/etc/cron.*/" + CRON_JOB_FILENAME):
        if os.path.exists(fn):
            os.remove(fn)

def uninstall_cron_poll():
    "Removes cron poll"
    if os.path.exists(CRON_POLL_FILEPATH):
        os.remove(CRON_POLL_FILEPATH)

@hooks.hook('identity-service-relation-joined')
def identity_service_joined(relation_id=None):
    config = hookenv.config()

    # Generate temporary bogus service URL to make keystone charm
    # happy. The sync script will replace it with the endpoint for
    # swift, because when this hook is fired, we do not yet
    # necessarily know the swift endpoint URL (it might not even exist
    # yet).

    url = 'http://' + hookenv.unit_get('private-address')
    relation_data = {
        'service': 'image-stream',
        'region': config['region'],
        'public_url': url,
        'admin_url': url,
        'internal_url': url}

    hookenv.relation_set(relation_id=relation_id, **relation_data)

@hooks.hook('identity-service-relation-changed')
def identity_service_changed():
    configs.write(ID_CONF_FILE_NAME)

@hooks.hook('install')
def install():
    execd_preinstall()
    for directory in [CONF_FILE_DIR, USR_SHARE_DIR]:
        hookenv.log("creating config dir at {}".format(directory))
        if not os.path.isdir(directory):
            if os.path.exists(directory):
                hookenv.log("error: {} exists but is not a directory."
                            " exiting.".format(directory))
                return
            os.mkdir(directory)

    hookenv.log('adding cloud-installer PPA')
    add_source('ppa:cloud-installer/simplestreams-testing')
    apt_update()

    apt_install(packages=['python-simplestreams', 'python-glanceclient',
                          'python-yaml', 'python-keystoneclient',
                          'python-kombu',
                          'python-swiftclient', 'ubuntu-cloudimage-keyring'])

    hookenv.log('end install hook.')

@hooks.hook('config-changed',
            'image-modifier-relation-changed',
            'image-modifier-relation-joined')
def config_changed():
    hookenv.log('begin config-changed hook.')

    configs.write(MIRRORS_CONF_FILE_NAME)

    config = hookenv.config()

    if config.changed('frequency'):
        hookenv.log("'frequency' changed, removing cron job")
        uninstall_cron_script()
        if config['run']:
            hookenv.log("moving cron job to "
                        "/etc/cron.{}".format(config['frequency']))
            install_cron_script()

    if config.changed('run'):
        hookenv.log("'run' changed, removing existing cron jobs")
        uninstall_cron_script()
        uninstall_cron_poll()

        if not config['run']:
            hookenv.log("'run' config now disabled, exiting")
        else:
            hookenv.log("'run' config now enabled, installing to "
                        "/etc/cron.{}".format(config['frequency']))
            hookenv.log("installing {} for polling".format(CRON_POLL_FILEPATH))
            install_cron_poll()
            install_cron_script()
    config.save()

@hooks.hook('upgrade-charm')
def upgrade_charm():
    install()
    configs.write_all()

@hooks.hook('amqp-relation-joined')
def amqp_joined():
    conf = hookenv.config()
    hookenv.relation_set(username=conf['rabbit-user'], vhost=conf['rabbit-vhost'])

@hooks.hook('amqp-relation-changed')
def amqp_changed():
    if 'amqp' not in configs.complete_contexts():
        hookenv.log('amqp relation incomplete. Peer not ready?')
        return
    configs.write(ID_CONF_FILE_NAME)

if __name__ == '__main__':
    try:
        hooks.execute(sys.argv)
    except hookenv.UnregisteredHookError as e:
        hookenv.log('Unknown hook {} - skipping.'.format(e))
<commit_msg>pep8 fixes<commit_after>#!/usr/bin/env python2.7
#
# Copyright 2014 Canonical Ltd. released under AGPL
#
# Authors:
#  Tycho Andersen <tycho.andersen@canonical.com>
#

# This file is part of the glance-simplestreams sync charm.
#
# The glance-simplestreams sync charm is free software: you can
# redistribute it and/or modify it under the terms of the GNU Affero General
# Public License as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later version.
#
# The charm is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this charm.  If not, see <http://www.gnu.org/licenses/>.

import glob
import os
import sys
import shutil

from charmhelpers.fetch import apt_install, add_source, apt_update
from charmhelpers.core import hookenv
from charmhelpers.payload.execd import execd_preinstall

from charmhelpers.contrib.openstack.context import (AMQPContext,
                                                    IdentityServiceContext,
                                                    OSContextGenerator)
from charmhelpers.contrib.openstack.utils import get_os_codename_package
from charmhelpers.contrib.openstack.templating import OSConfigRenderer

CONF_FILE_DIR = '/etc/glance-simplestreams-sync'
USR_SHARE_DIR = '/usr/share/glance-simplestreams-sync'

MIRRORS_CONF_FILE_NAME = os.path.join(CONF_FILE_DIR, 'mirrors.yaml')
ID_CONF_FILE_NAME = os.path.join(CONF_FILE_DIR, 'identity.yaml')

SCRIPT_NAME = "glance-simplestreams-sync.py"

CRON_JOB_FILENAME = 'glance_simplestreams_sync'
CRON_POLL_FILENAME = 'glance_simplestreams_sync_fastpoll'
CRON_POLL_FILEPATH = os.path.join('/etc/cron.d', CRON_POLL_FILENAME)

hooks = hookenv.Hooks()

class MultipleImageModifierSubordinatesIsNotSupported(Exception):
    """Raise this if multiple image-modifier subordinates are related to
    this charm.
    """

class MirrorsConfigServiceContext(OSContextGenerator):
    """Context for mirrors.yaml template.

    Uses image-modifier relation if available to set
    modify_hook_scripts config value.

    """
    interfaces = ['simplestreams-image-service']

    def __call__(self):
        hookenv.log("Generating template ctxt for simplestreams-image-service")
        config = hookenv.config()

        modify_hook_scripts = []
        image_modifiers = hookenv.relations_of_type('image-modifier')
        if len(image_modifiers) > 1:
            raise MultipleImageModifierSubordinatesIsNotSupported()

        if len(image_modifiers) == 1:
            im = image_modifiers[0]
            try:
                modify_hook_scripts.append(im['script-path'])

            except KeyError as ke:
                hookenv.log('relation {} yielded '
                            'exception {} - ignoring.'.format(repr(im),
                                                              repr(ke)))

        # default no-op so that None still means "missing" for config
        # validation (see elsewhere)
        if len(modify_hook_scripts) == 0:
            modify_hook_scripts.append('/bin/true')

        return dict(mirror_list=config['mirror_list'],
                    modify_hook_scripts=', '.join(modify_hook_scripts),
                    name_prefix=config['name_prefix'],
                    content_id_template=config['content_id_template'],
                    use_swift=config['use_swift'],
                    region=config['region'],
                    cloud_name=config['cloud_name'])

release = get_os_codename_package('glance-common', fatal=False) or 'icehouse'
configs = OSConfigRenderer(templates_dir='templates/',
                           openstack_release=release)

configs.register(MIRRORS_CONF_FILE_NAME, [MirrorsConfigServiceContext()])
configs.register(ID_CONF_FILE_NAME, [IdentityServiceContext(),
                                     AMQPContext()])

def install_cron_script():
    """Installs cron job in /etc/cron.$frequency/ for repeating sync

    Script is not a template but we always overwrite, to ensure it is
    up-to-date.

    """
    sync_script_source = os.path.join("scripts", SCRIPT_NAME)
    shutil.copy(sync_script_source, USR_SHARE_DIR)

    config = hookenv.config()
    installed_script = os.path.join(USR_SHARE_DIR, SCRIPT_NAME)
    linkname = '/etc/cron.{f}/{s}'.format(f=config['frequency'],
                                          s=CRON_JOB_FILENAME)
    os.symlink(installed_script, linkname)

def install_cron_poll():
    "Installs /etc/cron.d every-minute job in crontab for quick polling."
    poll_file_source = os.path.join('scripts', CRON_POLL_FILENAME)
    shutil.copy(poll_file_source, '/etc/cron.d/')

def uninstall_cron_script():
    "Removes sync program from any cron place it might be"
    for fn in glob.glob("/etc/cron.*/" + CRON_JOB_FILENAME):
        if os.path.exists(fn):
            os.remove(fn)

def uninstall_cron_poll():
    "Removes cron poll"
    if os.path.exists(CRON_POLL_FILEPATH):
        os.remove(CRON_POLL_FILEPATH)

@hooks.hook('identity-service-relation-joined')
def identity_service_joined(relation_id=None):
    config = hookenv.config()

    # Generate temporary bogus service URL to make keystone charm
    # happy. The sync script will replace it with the endpoint for
    # swift, because when this hook is fired, we do not yet
    # necessarily know the swift endpoint URL (it might not even exist
    # yet).

    url = 'http://' + hookenv.unit_get('private-address')
    relation_data = {
        'service': 'image-stream',
        'region': config['region'],
        'public_url': url,
        'admin_url': url,
        'internal_url': url}

    hookenv.relation_set(relation_id=relation_id, **relation_data)

@hooks.hook('identity-service-relation-changed')
def identity_service_changed():
    configs.write(ID_CONF_FILE_NAME)

@hooks.hook('install')
def install():
    execd_preinstall()
    for directory in [CONF_FILE_DIR, USR_SHARE_DIR]:
        hookenv.log("creating config dir at {}".format(directory))
        if not os.path.isdir(directory):
            if os.path.exists(directory):
                hookenv.log("error: {} exists but is not a directory."
                            " exiting.".format(directory))
                return
            os.mkdir(directory)

    hookenv.log('adding cloud-installer PPA')
    add_source('ppa:cloud-installer/simplestreams-testing')
    apt_update()

    apt_install(packages=['python-simplestreams', 'python-glanceclient',
                          'python-yaml', 'python-keystoneclient',
                          'python-kombu',
                          'python-swiftclient', 'ubuntu-cloudimage-keyring'])

    hookenv.log('end install hook.')

@hooks.hook('config-changed',
            'image-modifier-relation-changed',
            'image-modifier-relation-joined')
def config_changed():
    hookenv.log('begin config-changed hook.')

    configs.write(MIRRORS_CONF_FILE_NAME)

    config = hookenv.config()

    if config.changed('frequency'):
        hookenv.log("'frequency' changed, removing cron job")
        uninstall_cron_script()
        if config['run']:
            hookenv.log("moving cron job to "
                        "/etc/cron.{}".format(config['frequency']))
            install_cron_script()

    if config.changed('run'):
        hookenv.log("'run' changed, removing existing cron jobs")
        uninstall_cron_script()
        uninstall_cron_poll()

        if not config['run']:
            hookenv.log("'run' config now disabled, exiting")
        else:
            hookenv.log("'run' config now enabled, installing to "
                        "/etc/cron.{}".format(config['frequency']))
            hookenv.log("installing {} for polling".format(CRON_POLL_FILEPATH))
            install_cron_poll()
            install_cron_script()
    config.save()

@hooks.hook('upgrade-charm')
def upgrade_charm():
    install()
    configs.write_all()

@hooks.hook('amqp-relation-joined')
def amqp_joined():
    conf = hookenv.config()
    hookenv.relation_set(username=conf['rabbit-user'],
                         vhost=conf['rabbit-vhost'])

@hooks.hook('amqp-relation-changed')
def amqp_changed():
    if 'amqp' not in configs.complete_contexts():
        hookenv.log('amqp relation incomplete. Peer not ready?')
        return
    configs.write(ID_CONF_FILE_NAME)

if __name__ == '__main__':
    try:
        hooks.execute(sys.argv)
    except hookenv.UnregisteredHookError as e:
        hookenv.log('Unknown hook {} - skipping.'.format(e))
<|endoftext|>


<commit_before>#!/usr/bin/python
import setup

setup.pre_install()
from charmhelpers.core.hookenv import Hooks, UnregisteredHookError, log, relation_get, related_units
import json
import netifaces
import os
import sys
import signal
import subprocess
import time

hooks = Hooks()

@hooks.hook('config-changed')
def config_changed():
    pass

# Find the network interfaces to listen on based on the ceph.conf info
def find_interfaces():
    # 1. Is this machine running an OSD?
    # 2. What interface is it listening on?
    listen_interfaces = []
    interfaces = netifaces.interfaces()
    osd_dump = subprocess.check_output(['ceph', 'osd', 'dump', '--format', 'json'])
    try:
        osd_json = json.loads(osd_dump)
        for osd in osd_json['osds']:
            public_addr = osd['public_addr']
            log('OSD public addr: ' + str(public_addr))
            # Example ipv4 output: 10.0.3.213:6800/10784
            parts = public_addr.rstrip().split(':')
            if len(parts) != 2:
                log('Unable to decipher the ip address of the osd from: ' + public_addr)
                return -1
            addr = parts[0]
            log('OSD addr: ' + str(addr))
            # For each interface check to see if the IP addr matches an OSD ip addr.
            # If it does then we'll save it so that we can listen on that addr
            try:
                for i in interfaces:
                    interface_ip_info = netifaces.ifaddresses(i)[netifaces.AF_INET]
                    log('interface ip info: ' + str(interface_ip_info))
                    # If the OSD addr == the addr we own then add that to the listening list
                    if interface_ip_info is addr:
                        listen_interfaces.append(i)
            except KeyError:
                # I don't care about interfaces that don't have IP info on them
                pass
    except ValueError as err:
        log('Unable to decode json from ceph.  Error is: ' + err.message)

@hooks.hook('start')
def start():
    working_dir = os.getcwd()
    log('working dir: ' + str(working_dir))
    proc = subprocess.Popen(["hooks/decode_ceph", "-i", "eth0"], shell=True, cwd=working_dir)
    # Write pid to /var/run/decode_ceph
    with open('/var/run/decode_ceph', 'w+') as pid_file:
        pid_file.write(str(proc.pid))

@hooks.hook('stop')
def stop():
    with open('/var/run/decode_ceph', 'r') as pid_file:
        pid = pid_file.readlines()
        # Give a chance to stop nicely
        pid_id = int(pid[0].strip())
        assert isinstance(pid_id, int)

        os.kill(pid_id, signal.SIGTERM)
        time.sleep(5)
        # It should exit quickly but if it doesn't
        os.kill(pid_id, signal.SIGKILL)

def restart_collectors():
    pass

@hooks.hook('elasticsearch-relation-changed')
def elasticsearch_relation_changed():
    cluster_name = relation_get('cluster-name')
    es_host_list = []
    for member in related_units():
        es_host_list.append(relation_get('private-address {}'.format(member)))
    restart_collectors()

if __name__ == '__main__':
    listen_interfaces = find_interfaces()
    log('Found listen interfaces: ' + str(listen_interfaces))
    try:
        hooks.execute(sys.argv)
    except UnregisteredHookError as e:
        log('Unknown hook {} - skipping.'.format(e))
<commit_msg>A few bugfixes<commit_after>#!/usr/bin/python
import setup

setup.pre_install()
from charmhelpers.core.hookenv import Hooks, UnregisteredHookError, log, relation_get, related_units
import json
import netifaces
import os
import sys
import signal
import subprocess
import time

hooks = Hooks()

@hooks.hook('config-changed')
def config_changed():
    pass

# Find the network interfaces to listen on based on the ceph.conf info
def find_interfaces():
    # 1. Is this machine running an OSD?
    # 2. What interface is it listening on?
    interfaces_to_listen_on = []
    interfaces = netifaces.interfaces()
    osd_dump = subprocess.check_output(['ceph', 'osd', 'dump', '--format', 'json'])
    try:
        osd_json = json.loads(osd_dump)
        for osd in osd_json['osds']:
            public_addr = osd['public_addr']
            log('OSD public addr: ' + str(public_addr))
            # Example ipv4 output: 10.0.3.213:6800/10784
            parts = public_addr.rstrip().split(':')
            if len(parts) != 2:
                log('Unable to decipher the ip address of the osd from: ' + public_addr)
                return -1
            addr = parts[0]
            log('OSD addr: ' + str(addr))
            # For each interface check to see if the IP addr matches an OSD ip addr.
            # If it does then we'll save it so that we can listen on that addr
            try:
                for i in interfaces:
                    interface_ip_info = netifaces.ifaddresses(i)[netifaces.AF_INET]
                    log('interface ip info: ' + str(interface_ip_info))
                    # If the OSD addr == the addr we own then add that to the listening list
                    if interface_ip_info[0]['addr'] is addr:
                        interfaces_to_listen_on.append(i)
                return interfaces_to_listen_on
            except KeyError:
                # I don't care about interfaces that don't have IP info on them
                pass
    except ValueError as err:
        log('Unable to decode json from ceph.  Error is: ' + err.message)

@hooks.hook('start')
def start():
    working_dir = os.getcwd()
    log('working dir: ' + str(working_dir))
    listen_interfaces = find_interfaces()
    log('Found listen interfaces: ' + str(listen_interfaces))
    proc = subprocess.Popen(["hooks/decode_ceph", "-i", "eth0"], shell=True, cwd=working_dir)
    # Write pid to /var/run/decode_ceph
    with open('/var/run/decode_ceph', 'w+') as pid_file:
        pid_file.write(str(proc.pid))

@hooks.hook('stop')
def stop():
    with open('/var/run/decode_ceph', 'r') as pid_file:
        pid = pid_file.readlines()
        # Give a chance to stop nicely
        pid_id = int(pid[0].strip())
        assert isinstance(pid_id, int)

        try:
            os.kill(pid_id, signal.SIGTERM)
            time.sleep(5)
            # It should exit quickly but if it doesn't
            os.kill(pid_id, signal.SIGKILL)
        except OSError as err:
            log('Unable to find decode_ceph process: ' + err.message)

def restart_collectors():
    pass

@hooks.hook('elasticsearch-relation-changed')
def elasticsearch_relation_changed():
    cluster_name = relation_get('cluster-name')
    es_host_list = []
    for member in related_units():
        es_host_list.append(relation_get('private-address {}'.format(member)))
    restart_collectors()

if __name__ == '__main__':
    try:
        hooks.execute(sys.argv)
    except UnregisteredHookError as e:
        log('Unknown hook {} - skipping.'.format(e))
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
##############################################################################
#
#    Base Location Geonames Import module for OpenERP
#    Copyright (C) 2014 Akretion (http://www.akretion.com)
#    @author Alexis de Lattre <alexis.delattre@akretion.com>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from openerp.osv import orm, fields
from openerp.tools.translate import _
import requests
import tempfile
import StringIO
import unicodecsv
import zipfile
import os
import logging

logger = logging.getLogger(__name__)

class better_zip_geonames_import(orm.TransientModel):
    _name = 'better.zip.geonames.import'
    _description = 'Import Better Zip from Geonames'

    _columns = {
        'country_id': fields.many2one('res.country', 'Country', required=True),
    }

    def _prepare_better_zip(
            self, cr, uid, row, country_id, states, context=None):
        '''This function is designed to be inherited'''
        state_id = False
        if states and row[4] and row[4] in states:
            state_id = states[row[4].upper()]
        if row[0] == 'FR' and 'CEDEX' in row[1]:
            return False
        vals = {
            'name': row[1],
            'city': row[2],
            'state_id': state_id,
            'country_id': country_id,
            }
        return vals

    def create_better_zip(
            self, cr, uid, row, country_id, country_code, states,
            context=None):
        bzip_id = False
        if row[0] != country_code:
            raise orm.except_orm(
                _('Error:'),
                _("The country code inside the file (%s) doesn't "
                    "correspond to the selected country (%s).")
                % (row[0], country_code))
        logger.debug('ZIP = %s - City = %s' % (row[1], row[2]))
        if row[1] and row[2]:
            vals = self._prepare_better_zip(
                cr, uid, row, country_id, states, context=context)
            if vals:
                bzip_id = self.pool['res.better.zip'].create(
                    cr, uid, vals, context=context)
        return bzip_id

    def run_import(self, cr, uid, ids, context=None):
        assert len(ids) == 1, 'Only one ID for the better zip import wizard'
        bzip_obj = self.pool['res.better.zip']
        wizard = self.browse(cr, uid, ids[0], context=context)
        country_id = wizard.country_id.id
        country_code = wizard.country_id.code.upper()
        url = 'http://download.geonames.org/export/zip/%s.zip' % country_code
        logger.info('Starting to download %s' % url)
        res_request = requests.get(url)
        if res_request.status_code != requests.codes.ok:
            raise orm.except_orm(
                _('Error:'),
                _('Got an error %d when trying to download the file %s.')
                % (res_request.status_code, url))
        bzip_ids_to_delete = bzip_obj.search(
            cr, uid, [('country_id', '=', country_id)], context=context)
        if bzip_ids_to_delete:
            cr.execute('SELECT id FROM res_better_zip WHERE id in %s '
                'FOR UPDATE NOWAIT', (tuple(bzip_ids_to_delete), ))
            bzip_obj.unlink(cr, uid, bzip_ids_to_delete, context=context)
            logger.info(
                '%d better zip entries deleted for country %s'
                % (len(bzip_ids_to_delete), wizard.country_id.name))
        state_ids = self.pool['res.country.state'].search(
            cr, uid, [('country_id', '=', country_id)], context=context)
        states = {}
        # key = code of the state ; value = ID of the state in OpenERP
        if state_ids:
            states_r = self.pool['res.country.state'].read(
                cr, uid, state_ids, ['code', 'country_id'], context=context)
            for state in states_r:
                states[state['code'].upper()] = state['id']
        f_geonames = zipfile.ZipFile(StringIO.StringIO(res_request.content))
        tempdir = tempfile.mkdtemp(prefix='openerp')
        f_geonames.extract('%s.txt' % country_code, tempdir)
        logger.info('The geonames zipfile has been decompressed')
        data_file = open(os.path.join(tempdir, '%s.txt' % country_code), 'r')
        data_file.seek(0)
        logger.info(
            'Starting to create the better zip entries %s state information'
            % (states and 'with' or 'without'))
        for row in unicodecsv.reader(
                data_file, encoding='utf-8', delimiter='	'):
            self.create_better_zip(
                cr, uid, row, country_id, country_code, states,
                context=context)
        data_file.close()
        logger.info(
            'The wizard to create better zip entries from geonames '
            'has been successfully completed.')
        return True
<commit_msg>Remove France-specific code ; moved to new module l10n_fr_base_location_geonames_import on lp:openerp-french-localization<commit_after># -*- encoding: utf-8 -*-
##############################################################################
#
#    Base Location Geonames Import module for OpenERP
#    Copyright (C) 2014 Akretion (http://www.akretion.com)
#    @author Alexis de Lattre <alexis.delattre@akretion.com>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from openerp.osv import orm, fields
from openerp.tools.translate import _
import requests
import tempfile
import StringIO
import unicodecsv
import zipfile
import os
import logging

logger = logging.getLogger(__name__)

class better_zip_geonames_import(orm.TransientModel):
    _name = 'better.zip.geonames.import'
    _description = 'Import Better Zip from Geonames'

    _columns = {
        'country_id': fields.many2one('res.country', 'Country', required=True),
    }

    def _prepare_better_zip(
            self, cr, uid, row, country_id, states, context=None):
        '''This function is designed to be inherited'''
        state_id = False
        if states and row[4] and row[4] in states:
            state_id = states[row[4].upper()]
        vals = {
            'name': row[1],
            'city': row[2],
            'state_id': state_id,
            'country_id': country_id,
            }
        return vals

    def create_better_zip(
            self, cr, uid, row, country_id, country_code, states,
            context=None):
        bzip_id = False
        if row[0] != country_code:
            raise orm.except_orm(
                _('Error:'),
                _("The country code inside the file (%s) doesn't "
                    "correspond to the selected country (%s).")
                % (row[0], country_code))
        logger.debug('ZIP = %s - City = %s' % (row[1], row[2]))
        if row[1] and row[2]:
            vals = self._prepare_better_zip(
                cr, uid, row, country_id, states, context=context)
            if vals:
                bzip_id = self.pool['res.better.zip'].create(
                    cr, uid, vals, context=context)
        return bzip_id

    def run_import(self, cr, uid, ids, context=None):
        assert len(ids) == 1, 'Only one ID for the better zip import wizard'
        bzip_obj = self.pool['res.better.zip']
        wizard = self.browse(cr, uid, ids[0], context=context)
        country_id = wizard.country_id.id
        country_code = wizard.country_id.code.upper()
        url = 'http://download.geonames.org/export/zip/%s.zip' % country_code
        logger.info('Starting to download %s' % url)
        res_request = requests.get(url)
        if res_request.status_code != requests.codes.ok:
            raise orm.except_orm(
                _('Error:'),
                _('Got an error %d when trying to download the file %s.')
                % (res_request.status_code, url))
        bzip_ids_to_delete = bzip_obj.search(
            cr, uid, [('country_id', '=', country_id)], context=context)
        if bzip_ids_to_delete:
            cr.execute('SELECT id FROM res_better_zip WHERE id in %s '
                'FOR UPDATE NOWAIT', (tuple(bzip_ids_to_delete), ))
            bzip_obj.unlink(cr, uid, bzip_ids_to_delete, context=context)
            logger.info(
                '%d better zip entries deleted for country %s'
                % (len(bzip_ids_to_delete), wizard.country_id.name))
        state_ids = self.pool['res.country.state'].search(
            cr, uid, [('country_id', '=', country_id)], context=context)
        states = {}
        # key = code of the state ; value = ID of the state in OpenERP
        if state_ids:
            states_r = self.pool['res.country.state'].read(
                cr, uid, state_ids, ['code', 'country_id'], context=context)
            for state in states_r:
                states[state['code'].upper()] = state['id']
        f_geonames = zipfile.ZipFile(StringIO.StringIO(res_request.content))
        tempdir = tempfile.mkdtemp(prefix='openerp')
        f_geonames.extract('%s.txt' % country_code, tempdir)
        logger.info('The geonames zipfile has been decompressed')
        data_file = open(os.path.join(tempdir, '%s.txt' % country_code), 'r')
        data_file.seek(0)
        logger.info(
            'Starting to create the better zip entries %s state information'
            % (states and 'with' or 'without'))
        for row in unicodecsv.reader(
                data_file, encoding='utf-8', delimiter='	'):
            self.create_better_zip(
                cr, uid, row, country_id, country_code, states,
                context=context)
        data_file.close()
        logger.info(
            'The wizard to create better zip entries from geonames '
            'has been successfully completed.')
        return True
<|endoftext|>