<commit_before>"""Implements classes for generating data by schema."""
import csv
import json
import pickle
import typing as t
import warnings

from mimesis.exceptions import FieldError, SchemaError
from mimesis.locales import Locale
from mimesis.providers.generic import Generic
from mimesis.types import JSON, CallableSchema, Key, Seed

__all__ = ["BaseField", "Field", "Schema"]

class BaseField:
    """
    BaseField is a class for generating data by the name of the method.

    Instance of this object takes any string which represents the name
    of any method of any supported data provider (:class:`~mimesis.Generic`)
    and the ``**kwargs`` of the method.

    See :class:`~mimesis.schema.BaseField.perform` for more details.
    """

    class Meta:
        base = True

    def __init__(
        self,
        locale: Locale = Locale.DEFAULT,
        seed: Seed = None,
        providers: t.Optional[t.Sequence[t.Any]] = None,
    ) -> None:
        """Initialize field.

        :param locale: Locale
        :param seed: Seed for random.
        """
        self._gen = Generic(locale, seed)

        if providers:
            self._gen.add_providers(*providers)

        self._table = {}  # type: ignore

    def perform(
        self,
        name: t.Optional[str] = None,
        key: Key = None,
        **kwargs: t.Any,
    ) -> t.Any:
        """Performs the value of the field by its name.

        It takes any string which represents the name of any method of
        any supported data provider and the ``**kwargs`` of this method.

        .. note:: Some data providers have methods with the same names
            and in such cases, you can explicitly define that the method
            belongs to data-provider ``name='provider.name'`` otherwise
            it will return the data from the first provider which
            has a method ``name``.

        You can apply a *key function* to the result returned by
        the method, bt passing a parameter **key** with a callable
        object which returns the final result.

        :param name: Name of the method.
        :param key: A key function (or any other callable object)
            which will be applied to result.
        :param kwargs: Kwargs of method.
        :return: Value which represented by method.
        :raises ValueError: if provider not
            supported or if field not defined.
        """
        if name is None:
            raise FieldError()

        def tail_parser(tails: str, obj: t.Any) -> t.Any:
            """Return method from end of tail.

            :param tails: Tail string
            :param obj: Search tail from this object
            :return last tailed method
            """
            provider_name, method_name = tails.split(".", 1)

            if "." in method_name:
                raise FieldError(name)

            attr = getattr(obj, provider_name)
            if attr is not None:
                try:
                    return getattr(attr, method_name)
                except AttributeError:
                    raise FieldError(name)

        try:
            if name not in self._table:
                if "." not in name:
                    # Fix https://github.com/lk-geimfari/mimesis/issues/619
                    if name == self._gen.choice.Meta.name:
                        self._table[name] = self._gen.choice
                    else:
                        for provider in dir(self._gen):
                            provider = getattr(self._gen, provider)
                            if name in dir(provider):
                                self._table[name] = getattr(provider, name)
                else:
                    self._table[name] = tail_parser(name, self._gen)

            result = self._table[name](**kwargs)
            if key and callable(key):
                return key(result)
            return result
        except KeyError:
            raise FieldError(name)

    def __str__(self) -> str:
        return f"{self.__class__.__name__} <{self._gen.locale}>"

class Field(BaseField):
    """Greedy field.

    The field whcih evaluates immediately.

    Example:
        >>> _ = Field()
        >>> _('username')
        Dogtag_1836
    """

    def __call__(self, *args: t.Any, **kwargs: t.Any) -> t.Any:
        return self.perform(*args, **kwargs)

class Schema:
    """Class which return list of filled schemas."""

    _MIN_ITERATIONS_VALUE: t.ClassVar[int] = 1

    __slots__ = ("_schema",)

    def __init__(self, schema: CallableSchema) -> None:
        """Initialize schema.

        :param schema: A schema (must be a callable object).
        """
        if schema and callable(schema):
            self._schema = schema
        else:
            raise SchemaError()

    def to_csv(self, file_path: str, iterations: int = 100, **kwargs: t.Any) -> None:
        """Export a schema as a CSV file.

        :param file_path: File path.
        :param iterations: The required number of rows.
        :param kwargs: The keyword arguments for :py:class:`csv.DictWriter` class.

        *New in version 5.3.0*
        """
        data = self.create(iterations)
        fieldnames = list(data[0])

        with open(file_path, "w", encoding="utf-8", newline="") as fp:
            dict_writer = csv.DictWriter(fp, fieldnames, **kwargs)
            dict_writer.writeheader()
            dict_writer.writerows(data)

    def to_json(self, file_path: str, iterations: int = 100, **kwargs: t.Any) -> None:
        """Export a schema as a JSON file.

        :param file_path: File path.
        :param iterations: The required number of rows.
        :param kwargs: Extra keyword arguments for :py:func:`json.dump` class.

        *New in version 5.3.0*
        """
        data = self.create(iterations)
        with open(file_path, "w", encoding="utf-8") as fp:
            json.dump(data, fp, **kwargs)

    def to_pickle(self, file_path: str, iterations: int = 100, **kwargs: t.Any) -> None:
        """Export a schema as the pickled representation of the object to the file.

        :param file_path: File path.
        :param iterations: The required number of rows.
        :param kwargs: Extra keyword arguments for :py:func:`pickle.dump` class.

        *New in version 5.3.0*
        """
        data = self.create(iterations)
        with open(file_path, "wb") as fp:
            pickle.dump(data, fp, **kwargs)

    def create(self, iterations: int = 1) -> t.List[JSON]:
        """Creates a list of a fulfilled schemas.

        .. note::
            This method evaluates immediately, so be careful on creating
            large datasets otherwise you're risking running out of memory.

            If you need a lazy version of this method, see :meth:`iterator`.

        :param iterations: Number of iterations.
        :return: List of fulfilled schemas.
        """

        if iterations < self._MIN_ITERATIONS_VALUE:
            raise ValueError("The number of iterations must be greater than 0.")

        return [self._schema() for _ in range(iterations)]

    def loop(self) -> t.Iterator[JSON]:
        """Fulfills a schema **infinitely** in a lazy way.

        This method can be useful when you have some dynamic
        conditions in depend on which the generation must be interrupted.

        If you're accepting all risks below and want to suppress
        the warnings then use :py:class:`warnings.catch_warnings`

        .. note::
            Since data `mimesis` provides are limited, frequent calls of
            this method can cause data duplication.

        .. note::
            Before using this method, ask yourself: **Do I really need this**?

            In most cases, the answer is: Nah, :meth:`iterator` is enough.

        .. warning::
            Do not use this method without interrupt conditions, otherwise,
            you're risking running out of memory.

        .. warning::
            **Never** (seriously) call `list()`, `tuple()` or any other callable which tries to
            evaluate the whole lazy object on this method — infinite called infinite
            for a reason.

        :return: An infinite iterator with fulfilled schemas.
        """

        warnings.warn(
            "You're iterating over the infinite object! "
            "The schema.loop() can cause a serious memory leak."
            "Please, see: https://mimesis.name/en/latest/api.html#mimesis.schema.Schema.loop"
        )

        while True:
            yield self._schema()

    def iterator(self, iterations: int = 1) -> t.Iterator[JSON]:
        """Fulfills schema in a lazy way.

        :param iterations: Number of iterations.
        :return: List of fulfilled schemas.
        """

        if iterations < self._MIN_ITERATIONS_VALUE:
            raise ValueError("The number of iterations must be greater than 0.")

        for item in range(iterations):
            yield self._schema()
<commit_msg>Update docstrings for loop<commit_after>"""Implements classes for generating data by schema."""
import csv
import json
import pickle
import typing as t
import warnings

from mimesis.exceptions import FieldError, SchemaError
from mimesis.locales import Locale
from mimesis.providers.generic import Generic
from mimesis.types import JSON, CallableSchema, Key, Seed

__all__ = ["BaseField", "Field", "Schema"]

class BaseField:
    """
    BaseField is a class for generating data by the name of the method.

    Instance of this object takes any string which represents the name
    of any method of any supported data provider (:class:`~mimesis.Generic`)
    and the ``**kwargs`` of the method.

    See :class:`~mimesis.schema.BaseField.perform` for more details.
    """

    class Meta:
        base = True

    def __init__(
        self,
        locale: Locale = Locale.DEFAULT,
        seed: Seed = None,
        providers: t.Optional[t.Sequence[t.Any]] = None,
    ) -> None:
        """Initialize field.

        :param locale: Locale
        :param seed: Seed for random.
        """
        self._gen = Generic(locale, seed)

        if providers:
            self._gen.add_providers(*providers)

        self._table = {}  # type: ignore

    def perform(
        self,
        name: t.Optional[str] = None,
        key: Key = None,
        **kwargs: t.Any,
    ) -> t.Any:
        """Performs the value of the field by its name.

        It takes any string which represents the name of any method of
        any supported data provider and the ``**kwargs`` of this method.

        .. note:: Some data providers have methods with the same names
            and in such cases, you can explicitly define that the method
            belongs to data-provider ``name='provider.name'`` otherwise
            it will return the data from the first provider which
            has a method ``name``.

        You can apply a *key function* to the result returned by
        the method, bt passing a parameter **key** with a callable
        object which returns the final result.

        :param name: Name of the method.
        :param key: A key function (or any other callable object)
            which will be applied to result.
        :param kwargs: Kwargs of method.
        :return: Value which represented by method.
        :raises ValueError: if provider not
            supported or if field not defined.
        """
        if name is None:
            raise FieldError()

        def tail_parser(tails: str, obj: t.Any) -> t.Any:
            """Return method from end of tail.

            :param tails: Tail string
            :param obj: Search tail from this object
            :return last tailed method
            """
            provider_name, method_name = tails.split(".", 1)

            if "." in method_name:
                raise FieldError(name)

            attr = getattr(obj, provider_name)
            if attr is not None:
                try:
                    return getattr(attr, method_name)
                except AttributeError:
                    raise FieldError(name)

        try:
            if name not in self._table:
                if "." not in name:
                    # Fix https://github.com/lk-geimfari/mimesis/issues/619
                    if name == self._gen.choice.Meta.name:
                        self._table[name] = self._gen.choice
                    else:
                        for provider in dir(self._gen):
                            provider = getattr(self._gen, provider)
                            if name in dir(provider):
                                self._table[name] = getattr(provider, name)
                else:
                    self._table[name] = tail_parser(name, self._gen)

            result = self._table[name](**kwargs)
            if key and callable(key):
                return key(result)
            return result
        except KeyError:
            raise FieldError(name)

    def __str__(self) -> str:
        return f"{self.__class__.__name__} <{self._gen.locale}>"

class Field(BaseField):
    """Greedy field.

    The field whcih evaluates immediately.

    Example:
        >>> _ = Field()
        >>> _('username')
        Dogtag_1836
    """

    def __call__(self, *args: t.Any, **kwargs: t.Any) -> t.Any:
        return self.perform(*args, **kwargs)

class Schema:
    """Class which return list of filled schemas."""

    _MIN_ITERATIONS_VALUE: t.ClassVar[int] = 1

    __slots__ = ("_schema",)

    def __init__(self, schema: CallableSchema) -> None:
        """Initialize schema.

        :param schema: A schema (must be a callable object).
        """
        if schema and callable(schema):
            self._schema = schema
        else:
            raise SchemaError()

    def to_csv(self, file_path: str, iterations: int = 100, **kwargs: t.Any) -> None:
        """Export a schema as a CSV file.

        :param file_path: File path.
        :param iterations: The required number of rows.
        :param kwargs: The keyword arguments for :py:class:`csv.DictWriter` class.

        *New in version 5.3.0*
        """
        data = self.create(iterations)
        fieldnames = list(data[0])

        with open(file_path, "w", encoding="utf-8", newline="") as fp:
            dict_writer = csv.DictWriter(fp, fieldnames, **kwargs)
            dict_writer.writeheader()
            dict_writer.writerows(data)

    def to_json(self, file_path: str, iterations: int = 100, **kwargs: t.Any) -> None:
        """Export a schema as a JSON file.

        :param file_path: File path.
        :param iterations: The required number of rows.
        :param kwargs: Extra keyword arguments for :py:func:`json.dump` class.

        *New in version 5.3.0*
        """
        data = self.create(iterations)
        with open(file_path, "w", encoding="utf-8") as fp:
            json.dump(data, fp, **kwargs)

    def to_pickle(self, file_path: str, iterations: int = 100, **kwargs: t.Any) -> None:
        """Export a schema as the pickled representation of the object to the file.

        :param file_path: File path.
        :param iterations: The required number of rows.
        :param kwargs: Extra keyword arguments for :py:func:`pickle.dump` class.

        *New in version 5.3.0*
        """
        data = self.create(iterations)
        with open(file_path, "wb") as fp:
            pickle.dump(data, fp, **kwargs)

    def create(self, iterations: int = 1) -> t.List[JSON]:
        """Creates a list of a fulfilled schemas.

        .. note::
            This method evaluates immediately, so be careful on creating
            large datasets otherwise you're risking running out of memory.

            If you need a lazy version of this method, see :meth:`iterator`.

        :param iterations: Number of iterations.
        :return: List of fulfilled schemas.
        """

        if iterations < self._MIN_ITERATIONS_VALUE:
            raise ValueError("The number of iterations must be greater than 0.")

        return [self._schema() for _ in range(iterations)]

    def loop(self) -> t.Iterator[JSON]:
        """Fulfills a schema **infinitely** in a lazy way.

        This method can be useful when you have some dynamic
        conditions in depend on which the generation must be interrupted.

        Since data `mimesis` provides are limited, frequent calls of
        this method can cause data duplication.

        Before using this method, ask yourself: **Do I really need this**?
        In most cases, the answer is: Nah, :meth:`iterator` is enough.

        **Do not use** this method without **interrupt conditions**, otherwise,
        you're risking running out of memory.

        If you're accepting all risks below and want to suppress
        the warnings then use :py:class:`warnings.catch_warnings`

        .. warning::

            **Never** (seriously) call :py:class:`list`, :py:class:`tuple`, :py:class:`set`
            or any other callable which tries to evaluate the whole lazy object on this
            method — **infinite** called infinite for a reason.

        :return: An infinite iterator with fulfilled schemas.
        """

        warnings.warn(
            "You're iterating over the infinite object! "
            "The schema.loop() can cause a serious memory leak."
            "Please, see: https://mimesis.name/en/latest/api.html#mimesis.schema.Schema.loop"
        )

        while True:
            yield self._schema()

    def iterator(self, iterations: int = 1) -> t.Iterator[JSON]:
        """Fulfills schema in a lazy way.

        :param iterations: Number of iterations.
        :return: List of fulfilled schemas.
        """

        if iterations < self._MIN_ITERATIONS_VALUE:
            raise ValueError("The number of iterations must be greater than 0.")

        for item in range(iterations):
            yield self._schema()
<|endoftext|>


<commit_before>#-*- coding:utf-8 -*-
from django.db import models

class Minions_status(models.Model):
    class Meta:
        db_table = "minions_status"
    minion_id = models.CharField(max_length=128, null=True, blank=True)
    minion_version = models.CharField(max_length=128, null=True, blank=True)
    minion_status = models.CharField(max_length=128, null=True, blank=True)
    minion_config = models.CharField(max_length=128, default='unconfig')

    def __unicode__(self):
        return u'%s %s %s' % (self.minion_id, self.minion_version, self.minion_status)<commit_msg>修改config字段类型为布尔类型<commit_after>#-*- coding:utf-8 -*-
from django.db import models

class Minions_status(models.Model):
    class Meta:
        db_table = "minions_status"
    minion_id = models.CharField(max_length=128, null=True, blank=True)
    minion_version = models.CharField(max_length=128, null=True, blank=True)
    minion_status = models.CharField(max_length=128, null=True, blank=True)
    minion_config = models.BooleanField(default=False)

    def __unicode__(self):
        return u'%s %s %s' % (self.minion_id, self.minion_version, self.minion_status)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright 2013 - Mirantis, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.

"""
Configuration options registration and useful routines.
"""

import itertools

from oslo_config import cfg
from oslo_log import log

from mistral import version

launch_opt = cfg.ListOpt(
    'server',
    default=['all'],
    help='Specifies which mistral server to start by the launch script. '
         'Valid options are all or any combination of '
         'api, engine, and executor.'
)

api_opts = [
    cfg.StrOpt('host', default='0.0.0.0', help='Mistral API server host'),
    cfg.PortOpt('port', default=8989, help='Mistral API server port'),
    cfg.BoolOpt('allow_action_execution_deletion', default=False,
                help='Enables the ability to delete action_execution which '
                     'has no relationship with workflows.'),
]

pecan_opts = [
    cfg.StrOpt('root', default='mistral.api.controllers.root.RootController',
               help='Pecan root controller'),
    cfg.ListOpt('modules', default=["mistral.api"],
                help='A list of modules where pecan will search for '
                     'applications.'),
    cfg.BoolOpt('debug', default=False,
                help='Enables the ability to display tracebacks in the '
                     'browser and interactively debug during '
                     'development.'),
    cfg.BoolOpt('auth_enable', default=True,
                help='Enables user authentication in pecan.')
]

use_debugger = cfg.BoolOpt(
    "use-debugger",
    default=False,
    help='Enables debugger. Note that using this option changes how the '
         'eventlet library is used to support async IO. This could result '
         'in failures that do not occur under normal operation. '
         'Use at your own risk.'
)

engine_opts = [
    cfg.StrOpt('engine', default='default',
               help='Mistral engine plugin'),
    cfg.StrOpt('host', default='0.0.0.0',
               help='Name of the engine node. This can be an opaque '
                    'identifier. It is not necessarily a hostname, '
                    'FQDN, or IP address.'),
    cfg.StrOpt('topic', default='mistral_engine',
               help='The message topic that the engine listens on.'),
    cfg.StrOpt('version', default='1.0',
               help='The version of the engine.'),
    cfg.IntOpt('execution_field_size_limit_kb', default=1024,
               help='The default maximum size in KB of large text fields '
                    'of runtime execution objects. Use -1 for no limit.'),
]

executor_opts = [
    cfg.StrOpt('host', default='0.0.0.0',
               help='Name of the executor node. This can be an opaque '
                    'identifier. It is not necessarily a hostname, '
                    'FQDN, or IP address.'),
    cfg.StrOpt('topic', default='mistral_executor',
               help='The message topic that the executor listens on.'),
    cfg.StrOpt('version', default='1.0',
               help='The version of the executor.')
]

execution_expiration_policy_opts = [
    cfg.IntOpt('evaluation_interval',
               help='How often will the executions be evaluated '
                    '(in minutes). For example for value 120 the interval '
                    'will be 2 hours (every 2 hours).'),

    cfg.IntOpt('older_than',
               help='Evaluate from which time remove executions in minutes. '
                    'For example when older_than = 60, remove all executions '
                    'that finished a 60 minutes ago or more. '
                    'Minimum value is 1. '
                    'Note that only final state execution will remove '
                    '( SUCCESS / ERROR ).')
]

wf_trace_log_name_opt = cfg.StrOpt(
    'workflow_trace_log_name',
    default='workflow_trace',
    help='Logger name for pretty '
         'workflow trace output.'
)

coordination_opts = [
    cfg.StrOpt('backend_url',
               help='The backend URL to be used for coordination'),
    cfg.FloatOpt('heartbeat_interval',
                 default=5.0,
                 help='Number of seconds between heartbeats for coordination.')
]

CONF = cfg.CONF

API_GROUP = 'api'
ENGINE_GROUP = 'engine'
EXECUTOR_GROUP = 'executor'
PECAN_GROUP = 'pecan'
COORDINATION_GROUP = 'coordination'
EXECUTION_EXPIRATION_POLICY_GROUP = 'execution_expiration_policy'

CONF.register_opts(api_opts, group=API_GROUP)
CONF.register_opts(engine_opts, group=ENGINE_GROUP)
CONF.register_opts(pecan_opts, group=PECAN_GROUP)
CONF.register_opts(executor_opts, group=EXECUTOR_GROUP)
CONF.register_opts(execution_expiration_policy_opts,
                   group=EXECUTION_EXPIRATION_POLICY_GROUP)
CONF.register_opt(wf_trace_log_name_opt)
CONF.register_opts(coordination_opts, group=COORDINATION_GROUP)

CLI_OPTS = [
    use_debugger,
    launch_opt
]

CONF.register_cli_opts(CLI_OPTS)

_DEFAULT_LOG_LEVELS = [
    'amqp=WARN',
    'sqlalchemy=WARN',
    'oslo_messaging=INFO',
    'eventlet.wsgi.server=WARN',
    'stevedore=INFO',
    'oslo_service.periodic_task=INFO',
    'oslo_service.loopingcall=INFO',
    'mistral.services.periodic=INFO',
    'kazoo.client=WARN'
]

def list_opts():
    return [
        (API_GROUP, api_opts),
        (ENGINE_GROUP, engine_opts),
        (EXECUTOR_GROUP, executor_opts),
        (PECAN_GROUP, pecan_opts),
        (COORDINATION_GROUP, coordination_opts),
        (EXECUTION_EXPIRATION_POLICY_GROUP, execution_expiration_policy_opts),
        (None, itertools.chain(
            CLI_OPTS,
            [wf_trace_log_name_opt]
        ))
    ]

def parse_args(args=None, usage=None, default_config_files=None):
    log.set_defaults(default_log_levels=_DEFAULT_LOG_LEVELS)
    log.register_options(CONF)
    CONF(
        args=args,
        project='mistral',
        version=version,
        usage=usage,
        default_config_files=default_config_files
    )
<commit_msg>Numerous debug messages due to iso8601 log level<commit_after># -*- coding: utf-8 -*-
#
# Copyright 2013 - Mirantis, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.

"""
Configuration options registration and useful routines.
"""

import itertools

from oslo_config import cfg
from oslo_log import log

from mistral import version

launch_opt = cfg.ListOpt(
    'server',
    default=['all'],
    help='Specifies which mistral server to start by the launch script. '
         'Valid options are all or any combination of '
         'api, engine, and executor.'
)

api_opts = [
    cfg.StrOpt('host', default='0.0.0.0', help='Mistral API server host'),
    cfg.PortOpt('port', default=8989, help='Mistral API server port'),
    cfg.BoolOpt('allow_action_execution_deletion', default=False,
                help='Enables the ability to delete action_execution which '
                     'has no relationship with workflows.'),
]

pecan_opts = [
    cfg.StrOpt('root', default='mistral.api.controllers.root.RootController',
               help='Pecan root controller'),
    cfg.ListOpt('modules', default=["mistral.api"],
                help='A list of modules where pecan will search for '
                     'applications.'),
    cfg.BoolOpt('debug', default=False,
                help='Enables the ability to display tracebacks in the '
                     'browser and interactively debug during '
                     'development.'),
    cfg.BoolOpt('auth_enable', default=True,
                help='Enables user authentication in pecan.')
]

use_debugger = cfg.BoolOpt(
    "use-debugger",
    default=False,
    help='Enables debugger. Note that using this option changes how the '
         'eventlet library is used to support async IO. This could result '
         'in failures that do not occur under normal operation. '
         'Use at your own risk.'
)

engine_opts = [
    cfg.StrOpt('engine', default='default',
               help='Mistral engine plugin'),
    cfg.StrOpt('host', default='0.0.0.0',
               help='Name of the engine node. This can be an opaque '
                    'identifier. It is not necessarily a hostname, '
                    'FQDN, or IP address.'),
    cfg.StrOpt('topic', default='mistral_engine',
               help='The message topic that the engine listens on.'),
    cfg.StrOpt('version', default='1.0',
               help='The version of the engine.'),
    cfg.IntOpt('execution_field_size_limit_kb', default=1024,
               help='The default maximum size in KB of large text fields '
                    'of runtime execution objects. Use -1 for no limit.'),
]

executor_opts = [
    cfg.StrOpt('host', default='0.0.0.0',
               help='Name of the executor node. This can be an opaque '
                    'identifier. It is not necessarily a hostname, '
                    'FQDN, or IP address.'),
    cfg.StrOpt('topic', default='mistral_executor',
               help='The message topic that the executor listens on.'),
    cfg.StrOpt('version', default='1.0',
               help='The version of the executor.')
]

execution_expiration_policy_opts = [
    cfg.IntOpt('evaluation_interval',
               help='How often will the executions be evaluated '
                    '(in minutes). For example for value 120 the interval '
                    'will be 2 hours (every 2 hours).'),

    cfg.IntOpt('older_than',
               help='Evaluate from which time remove executions in minutes. '
                    'For example when older_than = 60, remove all executions '
                    'that finished a 60 minutes ago or more. '
                    'Minimum value is 1. '
                    'Note that only final state execution will remove '
                    '( SUCCESS / ERROR ).')
]

wf_trace_log_name_opt = cfg.StrOpt(
    'workflow_trace_log_name',
    default='workflow_trace',
    help='Logger name for pretty '
         'workflow trace output.'
)

coordination_opts = [
    cfg.StrOpt('backend_url',
               help='The backend URL to be used for coordination'),
    cfg.FloatOpt('heartbeat_interval',
                 default=5.0,
                 help='Number of seconds between heartbeats for coordination.')
]

CONF = cfg.CONF

API_GROUP = 'api'
ENGINE_GROUP = 'engine'
EXECUTOR_GROUP = 'executor'
PECAN_GROUP = 'pecan'
COORDINATION_GROUP = 'coordination'
EXECUTION_EXPIRATION_POLICY_GROUP = 'execution_expiration_policy'

CONF.register_opts(api_opts, group=API_GROUP)
CONF.register_opts(engine_opts, group=ENGINE_GROUP)
CONF.register_opts(pecan_opts, group=PECAN_GROUP)
CONF.register_opts(executor_opts, group=EXECUTOR_GROUP)
CONF.register_opts(execution_expiration_policy_opts,
                   group=EXECUTION_EXPIRATION_POLICY_GROUP)
CONF.register_opt(wf_trace_log_name_opt)
CONF.register_opts(coordination_opts, group=COORDINATION_GROUP)

CLI_OPTS = [
    use_debugger,
    launch_opt
]

CONF.register_cli_opts(CLI_OPTS)

_DEFAULT_LOG_LEVELS = [
    'amqp=WARN',
    'sqlalchemy=WARN',
    'oslo_messaging=INFO',
    'iso8601=WARN',
    'eventlet.wsgi.server=WARN',
    'stevedore=INFO',
    'oslo_service.periodic_task=INFO',
    'oslo_service.loopingcall=INFO',
    'mistral.services.periodic=INFO',
    'kazoo.client=WARN'
]

def list_opts():
    return [
        (API_GROUP, api_opts),
        (ENGINE_GROUP, engine_opts),
        (EXECUTOR_GROUP, executor_opts),
        (PECAN_GROUP, pecan_opts),
        (COORDINATION_GROUP, coordination_opts),
        (EXECUTION_EXPIRATION_POLICY_GROUP, execution_expiration_policy_opts),
        (None, itertools.chain(
            CLI_OPTS,
            [wf_trace_log_name_opt]
        ))
    ]

def parse_args(args=None, usage=None, default_config_files=None):
    log.set_defaults(default_log_levels=_DEFAULT_LOG_LEVELS)
    log.register_options(CONF)
    CONF(
        args=args,
        project='mistral',
        version=version,
        usage=usage,
        default_config_files=default_config_files
    )
<|endoftext|>


<commit_before># Copyright (c) 2015, The MITRE Corporation. All rights reserved.
# See LICENSE.txt for complete terms.

__version__ = "0.0.5"
<commit_msg>Bump version to 0.0.6<commit_after># Copyright (c) 2015, The MITRE Corporation. All rights reserved.
# See LICENSE.txt for complete terms.

__version__ = "0.0.6"
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding:utf-8
# A tiny Domain name Parse Ip script
"""
Copyright (c) 2017 LandGrey (https://github.com/LandGrey/taoman)
License: MIT
"""

import os
import sys
import time
import socket
import Queue
import threading

class Parser(threading.Thread):
    global born

    def __init__(self, queue):
        threading.Thread.__init__(self)
        self._queue = queue

    def run(self):
        while True:
            element = self._queue.get()
            if isinstance(element, str) and element == 'quit':
                    break
            try:
                ip = socket.gethostbyname(element)
                born.append(ip)
                sys.stdout.write("\r[+] Parsing {0:20} -> {1:15}".format(element, ip))
                sys.stdout.flush()
            except Exception as e:
                sys.stdout.write("\r[-] Parsing %s  error" % element)
                sys.stdout.flush()

def build_worker_pool(queue, size):
    workers = []
    for _ in range(size):
        worker = Parser(queue)
        worker.start()
        workers.append(worker)
    return workers

def urlfillter(url):
    if url.startswith('http') and '//' in url:
        url = (url.split('/')[2]).split(':')[0]
    else:
        url = (url.split('/')[0]).split(':')[0]
    return url

def start():
    global domains, threads, domain_file_path

    with open(domain_file_path, 'r') as f:
        for dom in f.readlines():
            domains.append(urlfillter(dom.strip()))

    queue = Queue.Queue()
    worker_threads = build_worker_pool(queue, threads)
    for url in domains:
            queue.put(url)
    for worker in worker_threads:
            queue.put('quit')
    for worker in worker_threads:
            worker.join()

# order preserving
def unique(seq, idfun=None):
    if idfun is None:
        def idfun(x): return x
    seen = {}
    results = []
    for item in seq:
        marker = idfun(item)
        if marker in seen:
            continue
        seen[marker] = 1
        results.append(item)
    return results

if __name__ == '__main__':
    start_time = time.time()
    if len(sys.argv) != 2:
        exit("[-] Usage: python domain_to_ip.py domain_lists_path \n")
    else:
        if not os.path.isfile(sys.argv[1]):
            exit('[-] No file: %s \n' % sys.argv[1])
    print("[+] A tiny Domain name To Ip script start \n"
          "                                    Build By LandGrey\n")
    threads = 50
    born = []
    domains = []
    domain_file_path = sys.argv[1]
    store_file_path = r'ip_lists.txt'

    start()
    count = 0
    with open(store_file_path, 'a') as _:
        for x in unique(born):
            try:
                _.write(str(x) + "\n")
                count += 1
            except:
                pass
    print("\n[+] Parsed   : {0}     domains\n"
          "[+] Total of : {1}     ip\n"
          "[+] Cost     : {2:.6} seconds".format(len(domains), count, time.time()-start_time))
<commit_msg>Update script<commit_after>#!/usr/bin/env python
# coding:utf-8
# A tiny Domain name Parse Ip script
"""
Copyright (c) 2017 LandGrey (https://github.com/LandGrey/taoman)
License: MIT
"""

import os
import sys
import time
import socket
import Queue
import threading

class Parser(threading.Thread):
    global borname

    def __init__(self, queue):
        threading.Thread.__init__(self)
        self._queue = queue

    def run(self):
        while True:
            element = self._queue.get()
            if isinstance(element, str) and element == 'quit':
                    break
            try:
                ip = socket.gethostbyname(element)
                borname.append(ip)
                sys.stdout.write("\r[+] Parsing {0:20} -> {1:15}".format(element, ip))
                sys.stdout.flush()
            except Exception as e:
                sys.stdout.write("\r[-] Parsing %s  error" % element)
                sys.stdout.flush()

def build_worker_pool(queue, size):
    workers = []
    for _ in range(size):
        worker = Parser(queue)
        worker.start()
        workers.append(worker)
    return workers

def urlfillter(url):
    if url.startswith('http') and '//' in url:
        url = (url.split('/')[2]).split(':')[0]
    else:
        url = (url.split('/')[0]).split(':')[0]
    return url

def start():
    global targets, threads, domain_file_path

    with open(domain_file_path, 'r') as f:
        for dom in f.readlines():
            domains.append(urlfillter(dom.strip()))

    queue = Queue.Queue()
    worker_threads = build_worker_pool(queue, threads)
    for url in domains:
            queue.put(url)
    for worker in worker_threads:
            queue.put('quit')
    for worker in worker_threads:
            worker.join()

# order preserving
def unique(seq, idfun=None):
    if idfun is None:
        def idfun(x): return x
    seen = {}
    results = []
    for item in seq:
        marker = idfun(item)
        if marker in seen:
            continue
        seen[marker] = 1
        results.append(item)
    return results

if __name__ == '__main__':
    start_time = time.time()
    if len(sys.argv) != 2:
        exit("[-] Usage: python domain_to_ip.py domain_lists_path \n")
    else:
        if not os.path.isfile(sys.argv[1]):
            exit('[-] No file: %s \n' % sys.argv[1])
    print("[+] A tiny Domain name To Ip script start \n"
          "                                    Build By LandGrey\n")
    threads = 50
    borname = []
    targets = []
    domain_file_path = sys.argv[1]
    store_file_path = r'ip_lists.txt'

    start()
    count = 0
    with open(store_file_path, 'a') as _:
        for x in unique(borname):
            try:
                _.write(str(x) + "\n")
                count += 1
            except:
                pass
    print("\n[+] Parsed   : {0}     targets\n"
          "[+] Total of : {1}     ip\n"
          "[+] Cost     : {2:.6} seconds".format(len(targets), count, time.time() - start_time))
<|endoftext|>


<commit_before>from __future__ import division

import numpy as np
import pandas as pd
from sklearn.cross_validation import train_test_split
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier, ExtraTreesClassifier
from sklearn.preprocessing import LabelEncoder

#np.random.seed(17411)

def logloss_mc(y_true, y_prob, epsilon=1e-15):
    """ Multiclass logloss

    This function is not officially provided by Kaggle, so there is no
    guarantee for its correctness.
    """
    # normalize
    y_prob = y_prob / y_prob.sum(axis=1).reshape(-1, 1)
    y_prob = np.maximum(epsilon, y_prob)
    y_prob = np.minimum(1 - epsilon, y_prob)
    # get probabilities
    y = [y_prob[i, j] for (i, j) in enumerate(y_true)]
    ll = - np.mean(np.log(y))
    return ll

def load_train_data(path=None, train_size=0.8):
    df = pd.read_csv('train.csv')
    X = df.values.copy()
    np.random.shuffle(X)
    X_train, X_valid, y_train, y_valid = train_test_split(
        X[:, 1:-1], X[:, -1], train_size=train_size,
    )

    print(" -- Loaded data.")
    return (X_train.astype(float), X_valid.astype(float),
            y_train.astype(str), y_valid.astype(str))

def encoder(model, validation_y):
    encoder = LabelEncoder()
    y_true = encoder.fit_transform(validation_y)
    assert (encoder.classes_ == model.classes_).all()
    return y_true

def load_test_data(path=None):
    df = pd.read_csv('test.csv')
    X = df.values
    X_test, ids = X[:, 1:], X[:, 0]
    return X_test.astype(float), ids.astype(str)

def validate_model(model, validation_x, validation_y):
    y_prob = model.predict_proba(validation_x)
    score = logloss_mc(validation_y, y_prob)
    print(" -- {} Multiclass logloss on validation set: {:.4f}.".format(type(model).__name__, score))
    return score

def train_rf(training_x, training_y, n_est=10):
    clf = RandomForestClassifier(n_jobs=-1, n_estimators=n_est)
    clf.fit(training_x, training_y)
    return clf

def train_ada(training_x, training_y, n_est=10):
    clf = AdaBoostClassifier(n_estimators=n_est)
    clf.fit(training_x, training_y)
    return clf

def train_grad(training_x, training_y, n_est=10):
    clf = GradientBoostingClassifier(n_estimators=n_est)
    clf.fit(training_x, training_y)
    return clf

def train_ex(training_x, training_y, n_est=10):
    clf = ExtraTreesClassifier(n_jobs=-1, n_estimators=n_est)
    clf.fit(training_x, training_y)
    return clf

def make_submission(clf, encoder, path='my_submission.csv'):
    X_test, ids = load_test_data()
    y_prob = clf.predict_proba(X_test)
    with open(path, 'w') as f:
        f.write('id,')
        f.write(','.join(encoder.classes_))
        f.write('\n')
        for id, probs in zip(ids, y_prob):
            probas = ','.join([id] + map(str, probs.tolist()))
            f.write(probas)
            f.write('\n')
    print(" -- Wrote submission to file {}.".format(path))

def main():
    print(" - Start.")
    X_train, X_valid, y_train, y_valid = load_train_data()

    results = []
    for i in range(100,1000,50):

        print 'i = {}'.format(i)
        rf_model = train_rf(X_train, y_train, i)
        y_true = encoder(rf_model, y_valid)
        score = validate_model(rf_model, X_valid, y_true)
        results.append({'score':score,'model':type(rf_model).__name__,'n_estimators':i})

        ex_model = train_ex(X_train, y_train, i)
        y_true = encoder(ex_model, y_valid)
        score = validate_model(ex_model, X_valid, y_true)
        results.append({'score':score,'model':type(ex_model).__name__,'n_estimators':i})

        ada_model = train_ada(X_train, y_train, i)
        y_true = encoder(ada_model, y_valid)
        score = validate_model(ada_model, X_valid, y_true)
        results.append({'score':score,'model':type(ada_model).__name__,'n_estimators':i})

        grad_model = train_grad(X_train, y_train, i)
        y_true = encoder(grad_model, y_valid)
        score = validate_model(grad_model, X_valid, y_true)
        results.append({'score':score,'model':type(grad_model).__name__,'n_estimators':i})

        pd.DataFrame(results).to_csv('results\\results.csv')

    #make_submission(rf, encoder)
    print(" - Finished.")

if __name__ == '__main__':
    main()
<commit_msg>trying out RF and extra trees parameters<commit_after>from __future__ import division

import numpy as np
import pandas as pd
from sklearn.cross_validation import train_test_split
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, ExtraTreesClassifier
from sklearn.preprocessing import LabelEncoder

#np.random.seed(17411)

def logloss_mc(y_true, y_prob, epsilon=1e-15):
    """ Multiclass logloss

    This function is not officially provided by Kaggle, so there is no
    guarantee for its correctness.
    """
    # normalize
    y_prob = y_prob / y_prob.sum(axis=1).reshape(-1, 1)
    y_prob = np.maximum(epsilon, y_prob)
    y_prob = np.minimum(1 - epsilon, y_prob)
    # get probabilities
    y = [y_prob[i, j] for (i, j) in enumerate(y_true)]
    ll = - np.mean(np.log(y))
    return ll

def load_train_data(path=None, train_size=0.8):
    df = pd.read_csv('train.csv')
    X = df.values.copy()
    np.random.shuffle(X)
    X_train, X_valid, y_train, y_valid = train_test_split(
        X[:, 1:-1], X[:, -1], train_size=train_size,
    )

    print(" -- Loaded data.")
    return (X_train.astype(float), X_valid.astype(float),
            y_train.astype(str), y_valid.astype(str))

def encoder(model, validation_y):
    encoder = LabelEncoder()
    y_true = encoder.fit_transform(validation_y)
    assert (encoder.classes_ == model.classes_).all()
    return y_true

def load_test_data(path=None):
    df = pd.read_csv('test.csv')
    X = df.values
    X_test, ids = X[:, 1:], X[:, 0]
    return X_test.astype(float), ids.astype(str)

def validate_model(model, validation_x, validation_y):
    y_prob = model.predict_proba(validation_x)
    score = logloss_mc(validation_y, y_prob)
    print(" -- {} Multiclass logloss on validation set: {:.4f}.".format(type(model).__name__, score))
    return score

def train_rf(training_x, training_y, n_est=10, max_d=5, max_f='auto'):
    clf = RandomForestClassifier(n_jobs=-1, n_estimators=n_est, max_depth=max_d, max_features=max_f)
    clf.fit(training_x, training_y)
    return clf

def train_ex(training_x, training_y, n_est=10, max_d=5, max_f='auto'):
    clf = ExtraTreesClassifier(n_jobs=-1, n_estimators=n_est, max_depth=max_d, max_features=max_f)
    clf.fit(training_x, training_y)
    return clf

#def train_ada(training_x, training_y, n_est=10):
#    clf = AdaBoostClassifier(n_estimators=n_est)
#    clf.fit(training_x, training_y)
#    return clf

def train_grad(training_x, training_y, n_est=10):
    clf = GradientBoostingClassifier(n_estimators=n_est)
    clf.fit(training_x, training_y)
    return clf

def make_submission(clf, encoder, path='my_submission.csv'):
    X_test, ids = load_test_data()
    y_prob = clf.predict_proba(X_test)
    with open(path, 'w') as f:
        f.write('id,')
        f.write(','.join(encoder.classes_))
        f.write('\n')
        for id, probs in zip(ids, y_prob):
            probas = ','.join([id] + map(str, probs.tolist()))
            f.write(probas)
            f.write('\n')
    print(" -- Wrote submission to file {}.".format(path))

def main():
    print(" - Start.")
    X_train, X_valid, y_train, y_valid = load_train_data()

    # set n_estimators to be 650, and now check max_depth and  max_features
    poss_max_features = ['sqrt']
    n_est = 650
    results = []
    for max_d in range(50,100,3):
        for idx, max_f in enumerate(poss_max_features):

            print 'max_depth = {}, max_features = {}'.format(max_d, max_f)
            rf_model = train_rf(X_train, y_train, n_est, max_d, max_f)
            y_true = encoder(rf_model, y_valid)
            score = validate_model(rf_model, X_valid, y_true)
            results.append({'score':score,'model':type(rf_model).__name__,'n_estimators':650,'max_depth':max_d,'max_features':max_f})

            ex_model = train_ex(X_train, y_train, n_est, max_d, max_f)
            y_true = encoder(ex_model, y_valid)
            score = validate_model(ex_model, X_valid, y_true)
            results.append({'score':score,'model':type(ex_model).__name__,'n_estimators':650,'max_depth':max_d,'max_features':max_f})

        #    grad_model = train_grad(X_train, y_train, i)
        #    y_true = encoder(grad_model, y_valid)
        #    score = validate_model(grad_model, X_valid, y_true)
        #    results.append({'score':score,'model':type(grad_model).__name__,'n_estimators':i})
        #
            pd.DataFrame(results).to_csv('results\\results4.csv')

    #make_submission(rf, encoder)
    print(" - Finished.")

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from flask import Flask, render_template, request, redirect, abort, url_for, \
                  make_response, session, flash

from mmmpaste import app, db, forms, filters, helpers
from mmmpaste.rest import rest

# Register the REST API module.
app.register_blueprint(rest, url_prefix = "/api")

@app.teardown_request
def shutdown_session(exception = None):
    db.session.remove()

@app.errorhandler(404)
def page_not_found(e):
    error = "Page not found :("
    return render_template("error.html", error = error), 404

@app.errorhandler(500)
def page_not_found(e):
    error = "The system is down. The system is down."
    return render_template("error.html", error = error), 500

@app.route("/")
def root():
    return redirect(url_for("new_paste"))

@app.route("/p/<id>")
@filters.cache
def get_paste(id):
    paste = db.get_paste(id)

    if paste is None:
        error = "This paste is no longer available."
        return render_template("error.html", error = error), 404

    return render_template("paste.html", paste = paste)

@app.route("/new", methods = ["POST", "GET"])
@filters.no_cache
def new_paste():
    form = forms.NewPaste(request.form)
    if request.method == "POST" and form.validate():
        id = db.new_paste(form.content.data, helpers.get_ip(),
                          form.filename.data, form.highlight.data,
                          form.convert_tabs.data)
        if "pastes" not in session:
            session["pastes"] = []

        pastes = session["pastes"]
        pastes.append(id)
        session["paste"] = pastes

        return redirect(url_for("get_paste", id = id))

    return render_template("new.html", form = form)

@app.route("/history")
def history():
    pass

@app.route("/about")
def about():
    pass

@app.route("/p/<id>/raw")
@filters.cache
def get_raw_paste(id):
    paste = db.get_paste(id)

    if paste is None:
        error = "This paste is no longer available."
        return render_template("error.html", error = error), 404

    response = make_response(str(paste.content))
    response.mimetype = "text/plain"
    return response

@app.route("/p/<id>/download")
@filters.cache
def download_paste(id):
    paste = db.get_paste(id)

    if paste is None:
        error = "This paste is no longer available."
        return render_template("error.html", error = error), 404

    filename = paste.filename

    if filename is None:
        filename = "paste-%s.txt" % paste.id_b62

    response = make_response(str(paste.content))
    response.headers["Content-Disposition"] = "attachment; filename=%s" % filename
    return response

@app.route("/p/<id>/clone")
def clone_paste(id):
    paste = db.get_paste(id)

    if paste is None:
        error = "This paste is no longer available."
        return render_template("error.html", error = error), 404

    form = forms.NewPaste()
    form.content.data = str(paste.content)
    return render_template("new.html", form = form)

@app.route("/p/<id>/delete")
def delete_paste(id):
    if id not in session["pastes"]:
        error = "You do not have permission to delete this paste."
        return render_template("error.html", error = error), 403

    db.deactivate_paste(id)
    flash("The paste has been deleted.")
    return redirect(url_for("new_paste"))

@app.route("/latest")
@filters.no_cache
def get_latest_paste():
    paste = db.get_paste()
    return redirect(url_for("get_paste", id = paste.id_b62), 307)
<commit_msg>If the pastebin is empty, /latest should return a 404.<commit_after>from flask import Flask, render_template, request, redirect, abort, url_for, \
                  make_response, session, flash

from mmmpaste import app, db, forms, filters, helpers
from mmmpaste.rest import rest

# Register the REST API module.
app.register_blueprint(rest, url_prefix = "/api")

@app.teardown_request
def shutdown_session(exception = None):
    db.session.remove()

@app.errorhandler(404)
def page_not_found(e):
    error = "Page not found :("
    return render_template("error.html", error = error), 404

@app.errorhandler(500)
def page_not_found(e):
    error = "The system is down. The system is down."
    return render_template("error.html", error = error), 500

@app.route("/")
def root():
    return redirect(url_for("new_paste"))

@app.route("/p/<id>")
@filters.cache
def get_paste(id):
    paste = db.get_paste(id)

    if paste is None:
        error = "This paste is no longer available."
        return render_template("error.html", error = error), 404

    return render_template("paste.html", paste = paste)

@app.route("/new", methods = ["POST", "GET"])
@filters.no_cache
def new_paste():
    form = forms.NewPaste(request.form)
    if request.method == "POST" and form.validate():
        id = db.new_paste(form.content.data, helpers.get_ip(),
                          form.filename.data, form.highlight.data,
                          form.convert_tabs.data)
        if "pastes" not in session:
            session["pastes"] = []

        pastes = session["pastes"]
        pastes.append(id)
        session["paste"] = pastes

        return redirect(url_for("get_paste", id = id))

    return render_template("new.html", form = form)

@app.route("/history")
def history():
    pass

@app.route("/about")
def about():
    pass

@app.route("/p/<id>/raw")
@filters.cache
def get_raw_paste(id):
    paste = db.get_paste(id)

    if paste is None:
        error = "This paste is no longer available."
        return render_template("error.html", error = error), 404

    response = make_response(str(paste.content))
    response.mimetype = "text/plain"
    return response

@app.route("/p/<id>/download")
@filters.cache
def download_paste(id):
    paste = db.get_paste(id)

    if paste is None:
        error = "This paste is no longer available."
        return render_template("error.html", error = error), 404

    filename = paste.filename

    if filename is None:
        filename = "paste-%s.txt" % paste.id_b62

    response = make_response(str(paste.content))
    response.headers["Content-Disposition"] = "attachment; filename=%s" % filename
    return response

@app.route("/p/<id>/clone")
def clone_paste(id):
    paste = db.get_paste(id)

    if paste is None:
        error = "This paste is no longer available."
        return render_template("error.html", error = error), 404

    form = forms.NewPaste()
    form.content.data = str(paste.content)
    return render_template("new.html", form = form)

@app.route("/p/<id>/delete")
def delete_paste(id):
    if id not in session["pastes"]:
        error = "You do not have permission to delete this paste."
        return render_template("error.html", error = error), 403

    db.deactivate_paste(id)
    flash("The paste has been deleted.")
    return redirect(url_for("new_paste"))

@app.route("/latest")
@filters.no_cache
def get_latest_paste():
    paste = db.get_paste()

    if paste is None:
        error = "The pastebin is empty."
        return render_template("error.html", error = error), 404

    return redirect(url_for("get_paste", id = paste.id_b62), 307)
<|endoftext|>


<commit_before>from __future__ import print_function
import contextlib
import datetime
import errno
import json
import shlex
import sys
import tempfile
import threading
import os

from subprocess import Popen
from subprocess import PIPE
from subprocess import STDOUT

import clog

DEPLOY_PIPELINE_NON_DEPLOY_STEPS = (
    'itest',
    'security-check',
    'performance-check',
    'push-to-registry'
)
# Default values for _log
ANY_CLUSTER = 'N/A'
ANY_INSTANCE = 'N/A'
DEFAULT_LOGLEVEL = 'event'

class PaastaColors:
    """Collection of static variables and methods to assist in coloring text."""
    # ANSI colour codes
    BLUE = '\033[34m'
    BOLD = '\033[1m'
    CYAN = '\033[36m'
    DEFAULT = '\033[0m'
    GREEN = '\033[32m'
    GREY = '\033[1m\033[30m'
    RED = '\033[31m'
    YELLOW = '\033[33m'

    @staticmethod
    def bold(text):
        """Return bolded text.

        :param text: a string
        :return: text colour coded with ANSI bold
        """
        return PaastaColors.color_text(PaastaColors.BOLD, text)

    @staticmethod
    def blue(text):
        """Return text that can be printed cyan.

        :param text: a string
        :return: text colour coded with ANSI blue
        """
        return PaastaColors.color_text(PaastaColors.BLUE, text)

    @staticmethod
    def green(text):
        """Return text that can be printed cyan.

        :param text: a string
        :return: text colour coded with ANSI green"""
        return PaastaColors.color_text(PaastaColors.GREEN, text)

    @staticmethod
    def red(text):
        """Return text that can be printed cyan.

        :param text: a string
        :return: text colour coded with ANSI red"""
        return PaastaColors.color_text(PaastaColors.RED, text)

    @staticmethod
    def color_text(color, text):
        """Return text that can be printed color.

        :param color: ANSI colour code
        :param text: a string
        :return: a string with ANSI colour encoding"""
        return color + text + PaastaColors.DEFAULT

    @staticmethod
    def cyan(text):
        """Return text that can be printed cyan.

        :param text: a string
        :return: text colour coded with ANSI cyan"""
        return PaastaColors.color_text(PaastaColors.CYAN, text)

    @staticmethod
    def yellow(text):
        """Return text that can be printed yellow.

        :param text: a string
        :return: text colour coded with ANSI yellow"""
        return PaastaColors.color_text(PaastaColors.YELLOW, text)

    @staticmethod
    def grey(text):
        return PaastaColors.color_text(PaastaColors.GREY, text)

    @staticmethod
    def default(text):
        return PaastaColors.color_text(PaastaColors.DEFAULT, text)

LOG_COMPONENTS = {
    'build': {
        'color': PaastaColors.blue,
        'help': 'Jenkins build jobs output, like the itest, promotion, security checks, etc.',
        'command': 'NA - TODO: tee jenkins build steps into scribe PAASTA-201',
        'source_env': 'env1',
    },
    'deploy': {
        'color': PaastaColors.cyan,
        'help': 'Output from the paasta deploy code. (setup_marathon_job, bounces, etc)',
        'command': 'NA - TODO: tee deploy logs into scribe PAASTA-201',
    },
    'app_output': {
        'color': PaastaColors.bold,
        'help': 'Stderr and stdout of the actual process spawned by Mesos',
        'command': 'NA - PAASTA-78',
    },
    'app_request': {
        'color': PaastaColors.bold,
        'help': 'The request log for the service. Defaults to "service_NAME_requests"',
        'command': 'scribe_reader -e ENV -f service_example_happyhour_requests',
    },
    'app_errors': {
        'color': PaastaColors.red,
        'help': 'Application error log, defaults to "stream_service_NAME_errors"',
        'command': 'scribe_reader -e ENV -f stream_service_SERVICE_errors',
    },
    'lb_requests': {
        'color': PaastaColors.bold,
        'help': 'All requests from Smartstack haproxy',
        'command': 'NA - TODO: SRV-1130',
    },
    'lb_errors': {
        'color': PaastaColors.red,
        'help': 'Logs from Smartstack haproxy that have 400-500 error codes',
        'command': 'scribereader -e ENV -f stream_service_errors | grep SERVICE.instance',
    },
    'monitoring': {
        'color': PaastaColors.green,
        'help': 'Logs from Sensu checks for the service',
        'command': 'NA - TODO log mesos healthcheck and sensu stuff.',
    },
}

class NoSuchLogComponent(Exception):
    pass

def validate_log_component(component):
    if component in LOG_COMPONENTS.keys():
        return True
    else:
        raise NoSuchLogComponent

def get_git_url(service):
    """Get the git url for a service. Assumes that the service's
    repo matches its name, and that it lives in services- i.e.
    if this is called with the string 'test', the returned
    url will be git@git.yelpcorp.com:services/test.git.

    :param service: The service name to get a URL for
    :returns: A git url to the service's repository"""
    return 'git@git.yelpcorp.com:services/%s.git' % service

class NoSuchLogLevel(Exception):
    pass

def configure_log():
    """We will log to the yocalhost binded scribe."""
    clog.config.configure(scribe_host='169.254.255.254', scribe_port=1463)

def _now():
    return datetime.datetime.utcnow().isoformat()

def format_log_line(level, cluster, instance, component, line):
    """Accepts a string 'line'.

    Returns an appropriately-formatted dictionary which can be serialized to
    JSON for logging and which contains 'line'.
    """
    validate_log_component(component)
    now = _now()
    message = json.dumps({
        'timestamp': now,
        'level': level,
        'cluster': cluster,
        'instance': instance,
        'component': component,
        'message': line,
    }, sort_keys=True)
    return message

def get_log_name_for_service(service_name):
    return 'stream_paasta_%s' % service_name

def _log(service_name, line, component, level=DEFAULT_LOGLEVEL, cluster=ANY_CLUSTER, instance=ANY_INSTANCE):
    """This expects someone (currently the paasta cli main()) to have already
    configured the log object. We'll just write things to it.
    """
    line = format_log_line(level, cluster, instance, component, line)
    if level == 'event':
        print(line, file=sys.stdout)
    elif level == 'debug':
        print(line, file=sys.stderr)
    else:
        raise NoSuchLogLevel
    line = str(line)
    log_name = get_log_name_for_service(service_name)
    clog.log_line(log_name, line)

def _timeout(process):
    """Helper function for _run. It terminates the process.
    Doesn't raise OSError, if we try to terminate a non-existing
    process as there can be a very small window between poll() and kill()
    """
    if process.poll() is None:
        try:
            # sending SIGKILL to the process
            process.kill()
            print('ERROR: Timeout running command: %s' % process.name)
        except OSError as e:
            # No such process error
            # The process could have been terminated meanwhile
            if e.errno != errno.ESRCH:
                raise

def _run(command, env=os.environ, timeout=None, log=False, **kwargs):
    """Given a command, run it. Return a tuple of the return code and any
    output.

    :param timeout: If specified, the command will be terminated after timeout
        seconds.
    :param log: If True, the _log will be handled by _run. If set, it is mandatory
        to pass at least a :service_name: and a :component: parameter. Optionally you
        can pass :cluster:, :instance: and :loglevel: parameters for logging.
    We wanted to use plumbum instead of rolling our own thing with
    subprocess.Popen but were blocked by
    https://github.com/tomerfiliba/plumbum/issues/162 and our local BASH_FUNC
    magic.
    """
    output = []
    if log:
        service_name = kwargs['service_name']
        component = kwargs['component']
        cluster = kwargs.get('cluster', ANY_CLUSTER)
        instance = kwargs.get('instance', ANY_INSTANCE)
        loglevel = kwargs.get('loglevel', DEFAULT_LOGLEVEL)
    try:
        process = Popen(shlex.split(command), stdout=PIPE, stderr=STDOUT, env=env)
        process.name = command
        # start the timer if we specified a timeout
        if timeout:
            proctimer = threading.Timer(timeout, _timeout, (process,))
            proctimer.start()
        for line in iter(process.stdout.readline, ''):
            if log:
                _log(
                    service_name=service_name,
                    line=line,
                    component=component,
                    level=loglevel,
                    cluster=cluster,
                    instance=instance,
                )
            output.append(line)
        # when finished, get the exit code
        returncode = process.wait()
    except OSError as e:
        output.append(e.strerror)
        returncode = e.errno
    # Stop the timer
    if timeout:
        proctimer.cancel()
    return returncode, ''.join(output)

def get_umask():
    """Get the current umask for this process. NOT THREAD SAFE."""
    old_umask = os.umask(0022)
    os.umask(old_umask)
    return old_umask

@contextlib.contextmanager
def atomic_file_write(target_path):
    dirname = os.path.dirname(target_path)
    basename = os.path.basename(target_path)

    with tempfile.NamedTemporaryFile(
        dir=dirname,
        prefix=('.%s-' % basename),
        delete=False
    ) as f:
        temp_target_path = f.name
        yield f

    mode = 0666 & (~get_umask())
    os.chmod(temp_target_path, mode)
    os.rename(temp_target_path, target_path)
<commit_msg>Log when _run has an Exception<commit_after>from __future__ import print_function
import contextlib
import datetime
import errno
import json
import shlex
import sys
import tempfile
import threading
import os

from subprocess import Popen
from subprocess import PIPE
from subprocess import STDOUT

import clog

DEPLOY_PIPELINE_NON_DEPLOY_STEPS = (
    'itest',
    'security-check',
    'performance-check',
    'push-to-registry'
)
# Default values for _log
ANY_CLUSTER = 'N/A'
ANY_INSTANCE = 'N/A'
DEFAULT_LOGLEVEL = 'event'

class PaastaColors:
    """Collection of static variables and methods to assist in coloring text."""
    # ANSI colour codes
    BLUE = '\033[34m'
    BOLD = '\033[1m'
    CYAN = '\033[36m'
    DEFAULT = '\033[0m'
    GREEN = '\033[32m'
    GREY = '\033[1m\033[30m'
    RED = '\033[31m'
    YELLOW = '\033[33m'

    @staticmethod
    def bold(text):
        """Return bolded text.

        :param text: a string
        :return: text colour coded with ANSI bold
        """
        return PaastaColors.color_text(PaastaColors.BOLD, text)

    @staticmethod
    def blue(text):
        """Return text that can be printed cyan.

        :param text: a string
        :return: text colour coded with ANSI blue
        """
        return PaastaColors.color_text(PaastaColors.BLUE, text)

    @staticmethod
    def green(text):
        """Return text that can be printed cyan.

        :param text: a string
        :return: text colour coded with ANSI green"""
        return PaastaColors.color_text(PaastaColors.GREEN, text)

    @staticmethod
    def red(text):
        """Return text that can be printed cyan.

        :param text: a string
        :return: text colour coded with ANSI red"""
        return PaastaColors.color_text(PaastaColors.RED, text)

    @staticmethod
    def color_text(color, text):
        """Return text that can be printed color.

        :param color: ANSI colour code
        :param text: a string
        :return: a string with ANSI colour encoding"""
        return color + text + PaastaColors.DEFAULT

    @staticmethod
    def cyan(text):
        """Return text that can be printed cyan.

        :param text: a string
        :return: text colour coded with ANSI cyan"""
        return PaastaColors.color_text(PaastaColors.CYAN, text)

    @staticmethod
    def yellow(text):
        """Return text that can be printed yellow.

        :param text: a string
        :return: text colour coded with ANSI yellow"""
        return PaastaColors.color_text(PaastaColors.YELLOW, text)

    @staticmethod
    def grey(text):
        return PaastaColors.color_text(PaastaColors.GREY, text)

    @staticmethod
    def default(text):
        return PaastaColors.color_text(PaastaColors.DEFAULT, text)

LOG_COMPONENTS = {
    'build': {
        'color': PaastaColors.blue,
        'help': 'Jenkins build jobs output, like the itest, promotion, security checks, etc.',
        'command': 'NA - TODO: tee jenkins build steps into scribe PAASTA-201',
        'source_env': 'env1',
    },
    'deploy': {
        'color': PaastaColors.cyan,
        'help': 'Output from the paasta deploy code. (setup_marathon_job, bounces, etc)',
        'command': 'NA - TODO: tee deploy logs into scribe PAASTA-201',
    },
    'app_output': {
        'color': PaastaColors.bold,
        'help': 'Stderr and stdout of the actual process spawned by Mesos',
        'command': 'NA - PAASTA-78',
    },
    'app_request': {
        'color': PaastaColors.bold,
        'help': 'The request log for the service. Defaults to "service_NAME_requests"',
        'command': 'scribe_reader -e ENV -f service_example_happyhour_requests',
    },
    'app_errors': {
        'color': PaastaColors.red,
        'help': 'Application error log, defaults to "stream_service_NAME_errors"',
        'command': 'scribe_reader -e ENV -f stream_service_SERVICE_errors',
    },
    'lb_requests': {
        'color': PaastaColors.bold,
        'help': 'All requests from Smartstack haproxy',
        'command': 'NA - TODO: SRV-1130',
    },
    'lb_errors': {
        'color': PaastaColors.red,
        'help': 'Logs from Smartstack haproxy that have 400-500 error codes',
        'command': 'scribereader -e ENV -f stream_service_errors | grep SERVICE.instance',
    },
    'monitoring': {
        'color': PaastaColors.green,
        'help': 'Logs from Sensu checks for the service',
        'command': 'NA - TODO log mesos healthcheck and sensu stuff.',
    },
}

class NoSuchLogComponent(Exception):
    pass

def validate_log_component(component):
    if component in LOG_COMPONENTS.keys():
        return True
    else:
        raise NoSuchLogComponent

def get_git_url(service):
    """Get the git url for a service. Assumes that the service's
    repo matches its name, and that it lives in services- i.e.
    if this is called with the string 'test', the returned
    url will be git@git.yelpcorp.com:services/test.git.

    :param service: The service name to get a URL for
    :returns: A git url to the service's repository"""
    return 'git@git.yelpcorp.com:services/%s.git' % service

class NoSuchLogLevel(Exception):
    pass

def configure_log():
    """We will log to the yocalhost binded scribe."""
    clog.config.configure(scribe_host='169.254.255.254', scribe_port=1463)

def _now():
    return datetime.datetime.utcnow().isoformat()

def format_log_line(level, cluster, instance, component, line):
    """Accepts a string 'line'.

    Returns an appropriately-formatted dictionary which can be serialized to
    JSON for logging and which contains 'line'.
    """
    validate_log_component(component)
    now = _now()
    message = json.dumps({
        'timestamp': now,
        'level': level,
        'cluster': cluster,
        'instance': instance,
        'component': component,
        'message': line,
    }, sort_keys=True)
    return message

def get_log_name_for_service(service_name):
    return 'stream_paasta_%s' % service_name

def _log(service_name, line, component, level=DEFAULT_LOGLEVEL, cluster=ANY_CLUSTER, instance=ANY_INSTANCE):
    """This expects someone (currently the paasta cli main()) to have already
    configured the log object. We'll just write things to it.
    """
    line = format_log_line(level, cluster, instance, component, line)
    if level == 'event':
        print(line, file=sys.stdout)
    elif level == 'debug':
        print(line, file=sys.stderr)
    else:
        raise NoSuchLogLevel
    line = str(line)
    log_name = get_log_name_for_service(service_name)
    clog.log_line(log_name, line)

def _timeout(process):
    """Helper function for _run. It terminates the process.
    Doesn't raise OSError, if we try to terminate a non-existing
    process as there can be a very small window between poll() and kill()
    """
    if process.poll() is None:
        try:
            # sending SIGKILL to the process
            process.kill()
            print('ERROR: Timeout running command: %s' % process.name)
        except OSError as e:
            # No such process error
            # The process could have been terminated meanwhile
            if e.errno != errno.ESRCH:
                raise

def _run(command, env=os.environ, timeout=None, log=False, **kwargs):
    """Given a command, run it. Return a tuple of the return code and any
    output.

    :param timeout: If specified, the command will be terminated after timeout
        seconds.
    :param log: If True, the _log will be handled by _run. If set, it is mandatory
        to pass at least a :service_name: and a :component: parameter. Optionally you
        can pass :cluster:, :instance: and :loglevel: parameters for logging.
    We wanted to use plumbum instead of rolling our own thing with
    subprocess.Popen but were blocked by
    https://github.com/tomerfiliba/plumbum/issues/162 and our local BASH_FUNC
    magic.
    """
    output = []
    if log:
        service_name = kwargs['service_name']
        component = kwargs['component']
        cluster = kwargs.get('cluster', ANY_CLUSTER)
        instance = kwargs.get('instance', ANY_INSTANCE)
        loglevel = kwargs.get('loglevel', DEFAULT_LOGLEVEL)
    try:
        process = Popen(shlex.split(command), stdout=PIPE, stderr=STDOUT, env=env)
        process.name = command
        # start the timer if we specified a timeout
        if timeout:
            proctimer = threading.Timer(timeout, _timeout, (process,))
            proctimer.start()
        for line in iter(process.stdout.readline, ''):
            if log:
                _log(
                    service_name=service_name,
                    line=line,
                    component=component,
                    level=loglevel,
                    cluster=cluster,
                    instance=instance,
                )
            output.append(line)
        # when finished, get the exit code
        returncode = process.wait()
    except OSError as e:
        _log(
            service_name=service_name,
            line=e.strerror,
            component=component,
            level=loglevel,
            cluster=cluster,
            instance=instance,
        )
        output.append(e.strerror)
        returncode = e.errno
    # Stop the timer
    if timeout:
        proctimer.cancel()
    return returncode, ''.join(output)

def get_umask():
    """Get the current umask for this process. NOT THREAD SAFE."""
    old_umask = os.umask(0022)
    os.umask(old_umask)
    return old_umask

@contextlib.contextmanager
def atomic_file_write(target_path):
    dirname = os.path.dirname(target_path)
    basename = os.path.basename(target_path)

    with tempfile.NamedTemporaryFile(
        dir=dirname,
        prefix=('.%s-' % basename),
        delete=False
    ) as f:
        temp_target_path = f.name
        yield f

    mode = 0666 & (~get_umask())
    os.chmod(temp_target_path, mode)
    os.rename(temp_target_path, target_path)
<|endoftext|>


<commit_before>"""
add popups and image links to timeseries_analyses.html

requires two javascript libraries in the same directory:
https://raw.github.com/nhoening/popup.js/master/dist/nhpup_1.1.js

"""

import os

basedir=os.environ['MYCONNECTOME_DIR']

def add_timeseries_links():
    infile=os.path.join(basedir,'timeseries/timeseries_analyses.html')
    outfile=os.path.join(basedir,'timeseries/timeseries_analyses_annot.html')

    # multiple entries for some networks because of divergent naming across
    # within- and between- network analyses

    netnames={'Default':['1','Default mode network'],
              'DMN':['1','Default mode network'],
              'Visual_2':['2','Secondary visual network'],
              'Frontoparietal_1':['3','Frontoparietal network'],
              'Fronto_parietal':['3','Frontoparietal network'],
              'Visual_1':['4.5','Primary visual network'],
              'Dorsal_Attention':['5','Dorsal attention network'],
              'Ventral_Attention':['7','Ventral attention network'],
              'Salience':['8','Salience network'],
              'Cingulo_Opercular':['9','Cingulo-opercular network'],
              'Cingulo_opercular':['9','Cingulo-opercular network'],
              'Somatomotor':['10','Somatomotor network'],
              'Frontoparietal_2':['11.5','Secondary fronto-parietal network'],
              'Medial_Parietal':['15','Medial parietal network'],
              'Parieto_occipital':['16','Parieto-occipital network'],
              'Parieto_Occipital':['16','Parieto-occipital network']}

    lines=open(infile).readlines()
    f=open(outfile,'w')
    reps={'panas.positive':'PANAS positive mood scale (after scan)',
          'afterscan.diastolic':'Diastolic blood pressure (after scan)',
          'afterscan.systolic':'Systolic blood pressure (after scan)',
          'afterscan.pulse':'Pulse (after scan)',
          'temp.mean':'Mean outdoor temperature (day of scan)',
          'email.LIWCnegemo':'LIWC Negative emotion scale on sent emails (day of scan)',
          'email.LIWCposemo':'LIWC Positive emotion scale on sent emails (day of scan)',
          'panas.fatigue':'PANAS fatigue scale (after scan)',
          'prevevening.Guthealth':'Self-rated gut health (previous evening)',
          'prevevening.Stress':'Self-rated psychological stress (previous evening)',
          'prevevening.Alcohol':'Amount of alcohol consumed (previous evening)',
          'prevevening.Psoriasisseverity':'Self-rated psoriasis severity (previous evening)',
          'TuesThurs':'Tuesdays (fasted/no caffeine) vs. Thursdays (fed/caffeinated)',
          'zeo.zq':'Overall sleep quality measured using Zeo sleep monitor (morning before scan)',
          'morning.Sleepquality':'Self-rated sleep quality (morning before scan)',
          'panas.negative':'PANAS negative mood scale (after scan)'}

    for l in lines:
        if l.find('</style>')>-1:
            l="""#pup {
      position:absolute;
      z-index:200; /* aaaalways on top*/
      padding: 3px;
      margin-left: 10px;
      margin-top: 5px;
      width: 250px;
      border: 1px solid black;
      background-color: #777;
      color: white;
      font-size: 0.95em;
    }

    </style>
    """

        if l.find('<title>')>-1:
            l+="""

    """
        for r in reps.keys():
            if l.find(r)>-1:
                l=l.replace(r,
            "%s"%(reps[r],r))

        for r in netnames:
               if l.find(r)>-1:
                   img=os.path.join(basedir,'rsfmri/network_plots/network%s.png'%netnames[r][0])
                   l=l.replace(r,
          "<a onmouseover=\"nhpup.popup('%s:

 ', {'width': 250});\">%s"%(netnames[r][1],img,r))
        if l.find('
Behavioral variables vs. each other:
')>-1:
            l="""
Mouse over behavioral variables for a more detailed description, and over network names for an image of the network.

            """+l
        f.write(l)

    f.close()

if __name__ == "__main__":
    add_timeseries_links()<commit_msg>Overwrite top and left position<commit_after>"""
add popups and image links to timeseries_analyses.html

requires two javascript libraries in the same directory:
https://raw.github.com/nhoening/popup.js/master/dist/nhpup_1.1.js

"""

import os

basedir=os.environ['MYCONNECTOME_DIR']

def add_timeseries_links():
    infile=os.path.join(basedir,'timeseries/timeseries_analyses.html')
    outfile=os.path.join(basedir,'timeseries/timeseries_analyses_annot.html')

    # multiple entries for some networks because of divergent naming across
    # within- and between- network analyses

    netnames={'Default':['1','Default mode network'],
              'DMN':['1','Default mode network'],
              'Visual_2':['2','Secondary visual network'],
              'Frontoparietal_1':['3','Frontoparietal network'],
              'Fronto_parietal':['3','Frontoparietal network'],
              'Visual_1':['4.5','Primary visual network'],
              'Dorsal_Attention':['5','Dorsal attention network'],
              'Ventral_Attention':['7','Ventral attention network'],
              'Salience':['8','Salience network'],
              'Cingulo_Opercular':['9','Cingulo-opercular network'],
              'Cingulo_opercular':['9','Cingulo-opercular network'],
              'Somatomotor':['10','Somatomotor network'],
              'Frontoparietal_2':['11.5','Secondary fronto-parietal network'],
              'Medial_Parietal':['15','Medial parietal network'],
              'Parieto_occipital':['16','Parieto-occipital network'],
              'Parieto_Occipital':['16','Parieto-occipital network']}

    lines=open(infile).readlines()
    f=open(outfile,'w')
    reps={'panas.positive':'PANAS positive mood scale (after scan)',
          'afterscan.diastolic':'Diastolic blood pressure (after scan)',
          'afterscan.systolic':'Systolic blood pressure (after scan)',
          'afterscan.pulse':'Pulse (after scan)',
          'temp.mean':'Mean outdoor temperature (day of scan)',
          'email.LIWCnegemo':'LIWC Negative emotion scale on sent emails (day of scan)',
          'email.LIWCposemo':'LIWC Positive emotion scale on sent emails (day of scan)',
          'panas.fatigue':'PANAS fatigue scale (after scan)',
          'prevevening.Guthealth':'Self-rated gut health (previous evening)',
          'prevevening.Stress':'Self-rated psychological stress (previous evening)',
          'prevevening.Alcohol':'Amount of alcohol consumed (previous evening)',
          'prevevening.Psoriasisseverity':'Self-rated psoriasis severity (previous evening)',
          'TuesThurs':'Tuesdays (fasted/no caffeine) vs. Thursdays (fed/caffeinated)',
          'zeo.zq':'Overall sleep quality measured using Zeo sleep monitor (morning before scan)',
          'morning.Sleepquality':'Self-rated sleep quality (morning before scan)',
          'panas.negative':'PANAS negative mood scale (after scan)'}

    for l in lines:
        if l.find('</style>')>-1:
            l="""#pup {
      position:absolute;
      z-index:200; /* aaaalways on top*/
      padding: 3px;
      margin-left: 10px;
      margin-top: 5px;
      width: 250px;
      border: 1px solid black;
      background-color: #777;
      color: white;
      font-size: 0.95em;
      top: 200px;
      left:200px;

    }

    </style>
    """

        if l.find('<title>')>-1:
            l+="""

    """
        for r in reps.keys():
            if l.find(r)>-1:
                l=l.replace(r,
            "%s"%(reps[r],r))

        for r in netnames:
               if l.find(r)>-1:
                   img=os.path.join(basedir,'rsfmri/network_plots/network%s.png'%netnames[r][0])
                   l=l.replace(r,
          "<a onmouseover=\"nhpup.popup('%s:

 ', {'width': 250});\">%s"%(netnames[r][1],img,r))
        if l.find('
Behavioral variables vs. each other:
')>-1:
            l="""
Mouse over behavioral variables for a more detailed description, and over network names for an image of the network.

            """+l
        f.write(l)

    f.close()

if __name__ == "__main__":
    add_timeseries_links()
<|endoftext|>


<commit_before>import re
import tarfile  # type: ignore
import lxml.etree  # type: ignore
from typing import Dict, Any, Tuple, List, Set

from nextcloudappstore.core.api.v1.release import ReleaseConfig
from nextcloudappstore.core.versioning import pad_max_version, pad_min_version
from rest_framework.exceptions import APIException  # type: ignore

class MaxSizeAppMetadataXmlException(APIException):
    pass

class InvalidAppMetadataXmlException(APIException):
    pass

class UnsupportedAppArchiveException(APIException):
    pass

class InvalidAppPackageStructureException(APIException):
    pass

class XMLSyntaxError(APIException):
    pass

class GunZipAppMetadataExtractor:
    def __init__(self, config: ReleaseConfig) -> None:
        """
        :argument config the config
        """
        self.config = config
        self.app_folder_regex = re.compile(r'^[a-z]+[a-z_]*(?:/.*)*$')

    def extract_app_metadata(self, archive_path: str) -> Tuple[str, str]:
        """
        Extracts the info.xml from an tar.gz archive
        :argument archive_path the path to the tar.gz archive
        :raises InvalidAppPackageStructureException if the first level folder
        does not equal the app_id or no info.xml file could be found in the
        appinfo folder
        :return the info.xml as string
        """
        if not tarfile.is_tarfile(archive_path):
            msg = '%s is not a valid tar.gz archive ' % archive_path
            raise UnsupportedAppArchiveException(msg)

        with tarfile.open(archive_path, 'r:gz') as tar:
            result = self._parse_archive(tar)
        return result

    def _parse_archive(self, tar: Any) -> Tuple[str, str]:
        folders = self._find_app_folders(tar.getnames())
        if len(folders) > 1:
            msg = 'More than one possible app folder found'
            raise InvalidAppPackageStructureException(msg)
        elif len(folders) == 0:
            msg = 'No possible app folder found. App folder must contain ' \
                  'only lowercase ASCII characters or underscores'
            raise InvalidAppPackageStructureException(msg)

        app_id = folders.pop()
        info_path = '%s/appinfo/info.xml' % app_id
        try:
            info_member = tar.getmember(info_path)
            possible_links = [info_member]
            # its complicated, sometimes there are single members, sometimes
            # there aren't
            try:
                possible_links.append(tar.getmember(app_id))
            except KeyError:
                pass
            try:
                possible_links.append(tar.getmember('%s/appinfo' % app_id))
            except KeyError:
                pass

            for possible_link in possible_links:
                if possible_link.issym() or possible_link.islnk():
                    msg = 'Symlinks and hard links can not be used for %s' % \
                          possible_link
                    raise InvalidAppPackageStructureException(msg)
            info_file = tar.extractfile(info_member)
            contents = self._stream_read_file(info_file,
                                              self.config.max_info_size)
            return contents, app_id
        except KeyError:
            msg = 'Could not find %s file inside the archive' % info_path
            raise InvalidAppPackageStructureException(msg)

    def _stream_read_file(self, info_file: Any, max_info_size: int) -> str:
        """
        Instead of reading everything in one go which is vulnerable to
        zip bombs, stream and accumulate the bytes
        :argument info_file: buffered io reader
        :argument max_info_size: maximum file size in bytes
        :raises MaxSizeAppMetadataXmlException if the maximum size was reached
        :return: the parsed info.xml
        """
        # FIXME: If someone finds a less ugly version, please feel free to
        #        improve it
        size = 0
        result = b''
        while True:
            size += 1024
            if size > max_info_size:
                msg = 'info.xml was bigger than allowed %i bytes' % \
                      max_info_size
                raise MaxSizeAppMetadataXmlException(msg)

            chunk = info_file.read(1024)
            if not chunk:
                break
            result += chunk

        return result.decode('utf-8')

    def _find_app_folders(self, members: List[str]) -> Set[str]:
        regex = self.app_folder_regex
        matching_members = filter(lambda f: re.match(regex, f), members)
        folders = map(lambda m: m.split('/')[0], matching_members)
        return set(folders)

def element_to_dict(element: Any) -> Dict:
    type = element.get('type')
    key = element.tag.replace('-', '_')
    if type == 'int':
        return {key: int(element.text)}
    elif type == 'list':
        return {key: list(map(element_to_dict, element.iterchildren()))}
    elif type == 'min-version':
        return {key: pad_min_version(element.text)}
    elif type == 'max-version':
        return {key: pad_max_version(element.text)}
    elif len(list(element)) > 0:
        contents = {}
        for child in element.iterchildren():
            contents.update(element_to_dict(child))
        return {key: contents}
    else:
        return {key: element.text}

def parse_app_metadata(xml: str, schema: str, pre_xslt: str,
                       xslt: str) -> Dict:
    """
    Parses, validates and maps the xml onto a dict
    :argument xml the info.xml string to parse
    :argument schema the schema xml as string
    :argument pre_xslt xslt which is run before validation to ensure that
    everything is in the correct order and that unknown elements are excluded
    :argument xslt the xslt to transform it to a matching structure
    :raises InvalidAppMetadataXmlException if the schema does not validate
    :return the parsed xml as dict
    """
    parser = lxml.etree.XMLParser(resolve_entities=False, no_network=True,
                                  remove_comments=True, load_dtd=False,
                                  remove_blank_text=True, dtd_validation=False
                                  )
    try:
        doc = lxml.etree.fromstring(bytes(xml, encoding='utf-8'), parser)
    except lxml.etree.XMLSyntaxError as e:
        msg = 'info.xml contains malformed xml: %s' % e
        raise XMLSyntaxError(msg)
    for _ in doc.iter(lxml.etree.Entity):
        raise InvalidAppMetadataXmlException('Must not contain entities')
    pre_transform = lxml.etree.XSLT(lxml.etree.XML(pre_xslt))
    pre_transformed_doc = pre_transform(doc)
    schema_doc = lxml.etree.fromstring(bytes(schema, encoding='utf-8'), parser)
    schema = lxml.etree.XMLSchema(schema_doc)
    try:
        schema.assertValid(pre_transformed_doc)  # type: ignore
    except lxml.etree.DocumentInvalid as e:
        msg = 'info.xml did not validate: %s' % e
        raise InvalidAppMetadataXmlException(msg)
    transform = lxml.etree.XSLT(lxml.etree.XML(xslt))
    transformed_doc = transform(pre_transformed_doc)
    mapped = element_to_dict(transformed_doc.getroot())
    validate_english_present(mapped)
    return mapped

def validate_english_present(info: Dict) -> None:
    """
    Validates that name, summary and description are present in english
    :param info: the parsed xml
    :raises: InvalidAppMetadataXmlException if at least one of the required
     fields is not present in english
    """
    app = info['app']
    translated_fields = ['name', 'summary', 'description']
    for field in translated_fields:
        if 'en' not in app[field]:
            msg = 'At least one element "%s" with lang "en" required ' % field
            raise InvalidAppMetadataXmlException(msg)
<commit_msg>remove useless space<commit_after>import re
import tarfile  # type: ignore
import lxml.etree  # type: ignore
from typing import Dict, Any, Tuple, List, Set

from nextcloudappstore.core.api.v1.release import ReleaseConfig
from nextcloudappstore.core.versioning import pad_max_version, pad_min_version
from rest_framework.exceptions import APIException  # type: ignore

class MaxSizeAppMetadataXmlException(APIException):
    pass

class InvalidAppMetadataXmlException(APIException):
    pass

class UnsupportedAppArchiveException(APIException):
    pass

class InvalidAppPackageStructureException(APIException):
    pass

class XMLSyntaxError(APIException):
    pass

class GunZipAppMetadataExtractor:
    def __init__(self, config: ReleaseConfig) -> None:
        """
        :argument config the config
        """
        self.config = config
        self.app_folder_regex = re.compile(r'^[a-z]+[a-z_]*(?:/.*)*$')

    def extract_app_metadata(self, archive_path: str) -> Tuple[str, str]:
        """
        Extracts the info.xml from an tar.gz archive
        :argument archive_path the path to the tar.gz archive
        :raises InvalidAppPackageStructureException if the first level folder
        does not equal the app_id or no info.xml file could be found in the
        appinfo folder
        :return the info.xml as string
        """
        if not tarfile.is_tarfile(archive_path):
            msg = '%s is not a valid tar.gz archive ' % archive_path
            raise UnsupportedAppArchiveException(msg)

        with tarfile.open(archive_path, 'r:gz') as tar:
            result = self._parse_archive(tar)
        return result

    def _parse_archive(self, tar: Any) -> Tuple[str, str]:
        folders = self._find_app_folders(tar.getnames())
        if len(folders) > 1:
            msg = 'More than one possible app folder found'
            raise InvalidAppPackageStructureException(msg)
        elif len(folders) == 0:
            msg = 'No possible app folder found. App folder must contain ' \
                  'only lowercase ASCII characters or underscores'
            raise InvalidAppPackageStructureException(msg)

        app_id = folders.pop()
        info_path = '%s/appinfo/info.xml' % app_id
        try:
            info_member = tar.getmember(info_path)
            possible_links = [info_member]
            # its complicated, sometimes there are single members, sometimes
            # there aren't
            try:
                possible_links.append(tar.getmember(app_id))
            except KeyError:
                pass
            try:
                possible_links.append(tar.getmember('%s/appinfo' % app_id))
            except KeyError:
                pass

            for possible_link in possible_links:
                if possible_link.issym() or possible_link.islnk():
                    msg = 'Symlinks and hard links can not be used for %s' % \
                          possible_link
                    raise InvalidAppPackageStructureException(msg)
            info_file = tar.extractfile(info_member)
            contents = self._stream_read_file(info_file,
                                              self.config.max_info_size)
            return contents, app_id
        except KeyError:
            msg = 'Could not find %s file inside the archive' % info_path
            raise InvalidAppPackageStructureException(msg)

    def _stream_read_file(self, info_file: Any, max_info_size: int) -> str:
        """
        Instead of reading everything in one go which is vulnerable to
        zip bombs, stream and accumulate the bytes
        :argument info_file: buffered io reader
        :argument max_info_size: maximum file size in bytes
        :raises MaxSizeAppMetadataXmlException if the maximum size was reached
        :return: the parsed info.xml
        """
        # FIXME: If someone finds a less ugly version, please feel free to
        #        improve it
        size = 0
        result = b''
        while True:
            size += 1024
            if size > max_info_size:
                msg = 'info.xml was bigger than allowed %i bytes' % \
                      max_info_size
                raise MaxSizeAppMetadataXmlException(msg)

            chunk = info_file.read(1024)
            if not chunk:
                break
            result += chunk

        return result.decode('utf-8')

    def _find_app_folders(self, members: List[str]) -> Set[str]:
        regex = self.app_folder_regex
        matching_members = filter(lambda f: re.match(regex, f), members)
        folders = map(lambda m: m.split('/')[0], matching_members)
        return set(folders)

def element_to_dict(element: Any) -> Dict:
    type = element.get('type')
    key = element.tag.replace('-', '_')
    if type == 'int':
        return {key: int(element.text)}
    elif type == 'list':
        return {key: list(map(element_to_dict, element.iterchildren()))}
    elif type == 'min-version':
        return {key: pad_min_version(element.text)}
    elif type == 'max-version':
        return {key: pad_max_version(element.text)}
    elif len(list(element)) > 0:
        contents = {}
        for child in element.iterchildren():
            contents.update(element_to_dict(child))
        return {key: contents}
    else:
        return {key: element.text}

def parse_app_metadata(xml: str, schema: str, pre_xslt: str,
                       xslt: str) -> Dict:
    """
    Parses, validates and maps the xml onto a dict
    :argument xml the info.xml string to parse
    :argument schema the schema xml as string
    :argument pre_xslt xslt which is run before validation to ensure that
    everything is in the correct order and that unknown elements are excluded
    :argument xslt the xslt to transform it to a matching structure
    :raises InvalidAppMetadataXmlException if the schema does not validate
    :return the parsed xml as dict
    """
    parser = lxml.etree.XMLParser(resolve_entities=False, no_network=True,
                                  remove_comments=True, load_dtd=False,
                                  remove_blank_text=True, dtd_validation=False
                                  )
    try:
        doc = lxml.etree.fromstring(bytes(xml, encoding='utf-8'), parser)
    except lxml.etree.XMLSyntaxError as e:
        msg = 'info.xml contains malformed xml: %s' % e
        raise XMLSyntaxError(msg)
    for _ in doc.iter(lxml.etree.Entity):
        raise InvalidAppMetadataXmlException('Must not contain entities')
    pre_transform = lxml.etree.XSLT(lxml.etree.XML(pre_xslt))
    pre_transformed_doc = pre_transform(doc)
    schema_doc = lxml.etree.fromstring(bytes(schema, encoding='utf-8'), parser)
    schema = lxml.etree.XMLSchema(schema_doc)
    try:
        schema.assertValid(pre_transformed_doc)  # type: ignore
    except lxml.etree.DocumentInvalid as e:
        msg = 'info.xml did not validate: %s' % e
        raise InvalidAppMetadataXmlException(msg)
    transform = lxml.etree.XSLT(lxml.etree.XML(xslt))
    transformed_doc = transform(pre_transformed_doc)
    mapped = element_to_dict(transformed_doc.getroot())
    validate_english_present(mapped)
    return mapped

def validate_english_present(info: Dict) -> None:
    """
    Validates that name, summary and description are present in english
    :param info: the parsed xml
    :raises: InvalidAppMetadataXmlException if at least one of the required
     fields is not present in english
    """
    app = info['app']
    translated_fields = ['name', 'summary', 'description']
    for field in translated_fields:
        if 'en' not in app[field]:
            msg = 'At least one element "%s" with lang "en" required' % field
            raise InvalidAppMetadataXmlException(msg)
<|endoftext|>